iicd/xautodl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update models

Browse Source
This commit is contained in:
D-X-Y
2021-03-23 11:13:51 +00:00
parent 01397660de
commit 379b904203
7 changed files with 175 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
lib/procedures/q_exps.py
View File

@@ -6,7 +6,7 @@ import inspect
import os
import pprint
import logging
from copy import deepcopy
import qlib
from qlib.utils import init_instance_by_config
from qlib.workflow import R
@@ -33,11 +33,14 @@ def set_log_basic_config(filename=None, format=None, level=None):
if format is None:
format = C.logging_config["formatters"]["logger_format"]["format"]
# Remove all handlers associated with the root logger object.
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=filename, format=format, level=level)
def update_gpu(config, gpu):
config = config.copy()
config = deepcopy(config)
if "task" in config and "model" in config["task"]:
if "GPU" in config["task"]["model"]:
config["task"]["model"]["GPU"] = gpu
@@ -59,13 +62,20 @@ def update_gpu(config, gpu):
def update_market(config, market):
config = config.copy()
config = deepcopy(config.copy())
config["market"] = market
config["data_handler_config"]["instruments"] = market
return config
def run_exp(task_config, dataset, experiment_name, recorder_name, uri):
def run_exp(
task_config,
dataset,
experiment_name,
recorder_name,
uri,
model_obj_name="model.pkl",
):
model = init_instance_by_config(task_config["model"])
model_fit_kwargs = dict(dataset=dataset)
@@ -80,6 +90,7 @@ def run_exp(task_config, dataset, experiment_name, recorder_name, uri):
# Setup log
recorder_root_dir = R.get_recorder().get_local_dir()
log_file = os.path.join(recorder_root_dir, "{:}.log".format(experiment_name))
set_log_basic_config(log_file)
logger = get_module_logger("q.run_exp")
logger.info("task_config::\n{:}".format(pprint.pformat(task_config, indent=2)))
@@ -87,20 +98,29 @@ def run_exp(task_config, dataset, experiment_name, recorder_name, uri):
logger.info("dataset={:}".format(dataset))
# Train model
R.log_params(**flatten_dict(task_config))
if "save_path" in inspect.getfullargspec(model.fit).args:
model_fit_kwargs["save_path"] = os.path.join(recorder_root_dir, "model.ckp")
elif "save_dir" in inspect.getfullargspec(model.fit).args:
model_fit_kwargs["save_dir"] = os.path.join(recorder_root_dir, "model-ckps")
model.fit(**model_fit_kwargs)
try:
model = R.load_object(model_obj_name)
logger.info("[Find existing object from {:}]".format(model_obj_name))
except OSError:
R.log_params(**flatten_dict(task_config))
if "save_path" in inspect.getfullargspec(model.fit).args:
model_fit_kwargs["save_path"] = os.path.join(
recorder_root_dir, "model.ckp"
)
elif "save_dir" in inspect.getfullargspec(model.fit).args:
model_fit_kwargs["save_dir"] = os.path.join(
recorder_root_dir, "model-ckps"
)
model.fit(**model_fit_kwargs)
R.save_objects(**{model_obj_name: model})
except:
raise ValueError("Something wrong.")
# Get the recorder
recorder = R.get_recorder()
R.save_objects(**{"model.pkl": model})
# Generate records: prediction, backtest, and analysis
import pdb; pdb.set_trace()
for record in task_config["record"]:
record = record.copy()
record = deepcopy(record)
if record["class"] == "SignalRecord":
srconf = {"model": model, "dataset": dataset, "recorder": recorder}
record["kwargs"].update(srconf)

14
lib/trade_models/transformers.py
View File

@@ -193,19 +193,15 @@ def get_transformer(config):
raise ValueError("Invalid Configuration: {:}".format(config))
name = config.get("name", "basic")
if name == "basic":
model = TransformerModel(
model = SuperTransformer(
d_feat=config.get("d_feat"),
embed_dim=config.get("embed_dim"),
depth=config.get("depth"),
stem_dim=config.get("stem_dim"),
embed_dims=config.get("embed_dims"),
num_heads=config.get("num_heads"),
mlp_ratio=config.get("mlp_ratio"),
mlp_hidden_multipliers=config.get("mlp_hidden_multipliers"),
qkv_bias=config.get("qkv_bias"),
qk_scale=config.get("qkv_scale"),
pos_drop=config.get("pos_drop"),
mlp_drop_rate=config.get("mlp_drop_rate"),
attn_drop_rate=config.get("attn_drop_rate"),
drop_path_rate=config.get("drop_path_rate"),
norm_layer=config.get("norm_layer", None),
other_drop=config.get("other_drop"),
)
else:
raise ValueError("Unknown model name: {:}".format(name))

7
lib/xlayers/super_module.py
View File

@@ -14,6 +14,13 @@ IntSpaceType = Union[int, spaces.Integer, spaces.Categorical]
BoolSpaceType = Union[bool, spaces.Categorical]
class LayerOrder(Enum):
"""This class defines the enumerations for order of operation in a residual or normalization-based layer."""
PreNorm = "pre-norm"
PostNorm = "post-norm"
class SuperRunMode(Enum):
"""This class defines the enumerations for Super Model Running Mode."""

55
lib/xlayers/super_transformer.py
View File

@@ -15,6 +15,7 @@ import torch.nn.functional as F
import spaces
from .super_module import IntSpaceType
from .super_module import BoolSpaceType
from .super_module import LayerOrder
from .super_module import SuperModule
from .super_linear import SuperMLPv2
from .super_norm import SuperLayerNorm1D
@@ -30,7 +31,8 @@ class SuperTransformerEncoderLayer(SuperModule):
- PyTorch Implementation: https://pytorch.org/docs/stable/_modules/torch/nn/modules/transformer.html#TransformerEncoderLayer
Details:
MHA -> residual -> norm -> MLP -> residual -> norm
the original post-norm version: MHA -> residual -> norm -> MLP -> residual -> norm
the pre-norm version: norm -> MHA -> residual -> norm -> MLP -> residual
"""
def __init__(
@@ -42,9 +44,10 @@ class SuperTransformerEncoderLayer(SuperModule):
mlp_hidden_multiplier: IntSpaceType = 4,
drop: Optional[float] = None,
act_layer: Callable[[], nn.Module] = nn.GELU,
order: LayerOrder = LayerOrder.PreNorm,
):
super(SuperTransformerEncoderLayer, self).__init__()
self.mha = SuperAttention(
mha = SuperAttention(
input_dim,
input_dim,
num_heads=num_heads,
@@ -52,17 +55,33 @@ class SuperTransformerEncoderLayer(SuperModule):
attn_drop=drop,
proj_drop=drop,
)
self.drop1 = nn.Dropout(drop or 0.0)
self.norm1 = SuperLayerNorm1D(input_dim)
self.mlp = SuperMLPv2(
drop1 = nn.Dropout(drop or 0.0)
norm1 = SuperLayerNorm1D(input_dim)
mlp = SuperMLPv2(
input_dim,
hidden_multiplier=mlp_hidden_multiplier,
out_features=output_dim,
act_layer=act_layer,
drop=drop,
)
self.drop2 = nn.Dropout(drop or 0.0)
self.norm2 = SuperLayerNorm1D(output_dim)
drop2 = nn.Dropout(drop or 0.0)
norm2 = SuperLayerNorm1D(output_dim)
if order is LayerOrder.PreNorm:
self.norm1 = norm1
self.mha = mha
self.drop1 = drop1
self.norm2 = norm2
self.mlp = mlp
self.drop2 = drop2
elif order is LayerOrder.PostNoem:
self.mha = mha
self.drop1 = drop1
self.norm1 = norm1
self.mlp = mlp
self.drop2 = drop2
self.norm2 = norm2
else:
raise ValueError("Unknown order: {:}".format(order))
@property
def abstract_search_space(self):
@@ -89,12 +108,18 @@ class SuperTransformerEncoderLayer(SuperModule):
return self.forward_raw(input)
def forward_raw(self, input: torch.Tensor) -> torch.Tensor:
# multi-head attention
x = self.mha(input)
x = x + self.drop1(x)
x = self.norm1(x)
# feed-forward layer
x = self.mlp(x)
x = x + self.drop2(x)
x = self.norm2(x)
if order is LayerOrder.PreNorm:
x = self.norm1(input)
x = x + self.drop1(self.mha(x))
x = self.norm2(x)
x = x + self.drop2(self.mlp(x))
elif order is LayerOrder.PostNoem:
# multi-head attention
x = x + self.drop1(self.mha(input))
x = self.norm1(x)
# feed-forward layer
x = x + self.drop2(self.mlp(x))
x = self.norm2(x)
else:
raise ValueError("Unknown order: {:}".format(order))
return x