liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-18 09:54:33 +08:00
Code Activity

Format

Browse Source
This commit is contained in:
Jactus
2020-11-26 15:50:42 +08:00
parent a108f753d5
commit 056951605b
9 changed files with 72 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/component/data.rst
View File

@@ -292,6 +292,7 @@ The ``Processor`` module in ``Qlib`` is designed to be learnable and it is respo
- ``Fillna``: `processor` that handles N/A values, which will fill the N/A value by 0 or other given number. - ``Fillna``: `processor` that handles N/A values, which will fill the N/A value by 0 or other given number.
- ``MinMaxNorm``: `processor` that applies min-max normalization. - ``MinMaxNorm``: `processor` that applies min-max normalization.
- ``ZscoreNorm``: `processor` that applies z-score normalization. - ``ZscoreNorm``: `processor` that applies z-score normalization.
- ``RobustZScoreNorm``: `processor` that applies robust z-score normalization.
- ``CSZScoreNorm``: `processor` that applies cross sectional z-score normalization. - ``CSZScoreNorm``: `processor` that applies cross sectional z-score normalization.
- ``CSRankNorm``: `processor` that applies cross sectional rank normalization. - ``CSRankNorm``: `processor` that applies cross sectional rank normalization.

1
examples/workflow_by_code_gats.py
View File

@@ -17,7 +17,6 @@ from qlib.utils import exists_qlib_data
from qlib.utils import init_instance_by_config from qlib.utils import init_instance_by_config
if __name__ == "__main__": if __name__ == "__main__":
# use default data # use default data

1
examples/workflow_by_code_hats.py
View File

@@ -90,7 +90,6 @@ if __name__ == "__main__":
# "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'], # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
} }
model = init_instance_by_config(task["model"]) model = init_instance_by_config(task["model"])
dataset = init_instance_by_config(task["dataset"]) dataset = init_instance_by_config(task["dataset"])
model.fit(dataset, save_path="benchmarks/HATS/model_hat.pkl") model.fit(dataset, save_path="benchmarks/HATS/model_hat.pkl")

20
qlib/contrib/model/catboost_model.py
View File

@@ -34,14 +34,14 @@ class CatBoostModel(Model):
def fit( def fit(
self, self,
dataset: DatasetH, dataset: DatasetH,
num_boost_round = 1000, num_boost_round=1000,
early_stopping_rounds = 50, early_stopping_rounds=50,
verbose_eval = 20, verbose_eval=20,
evals_result = dict(), evals_result=dict(),
**kwargs **kwargs
): ):
df_train, df_valid = dataset.prepare( df_train, df_valid = dataset.prepare(
["train", "valid"], col_set = ["feature", "label"], data_key = DataHandlerLP.DK_L ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
) )
x_train, y_train = df_train["feature"], df_train["label"] x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"] x_valid, y_valid = df_valid["feature"], df_valid["label"]
@@ -52,8 +52,8 @@ class CatBoostModel(Model):
else: else:
raise ValueError("CatBoost doesn't support multi-label training") raise ValueError("CatBoost doesn't support multi-label training")
train_pool = Pool(data = x_train, label = y_train_1d) train_pool = Pool(data=x_train, label=y_train_1d)
valid_pool = Pool(data = x_valid, label = y_valid_1d) valid_pool = Pool(data=x_valid, label=y_valid_1d)
# Initialize the catboost model # Initialize the catboost model
self._params["iterations"] = num_boost_round self._params["iterations"] = num_boost_round
@@ -63,7 +63,7 @@ class CatBoostModel(Model):
self.model = CatBoost(self._params, **kwargs) self.model = CatBoost(self._params, **kwargs)
# train the model # train the model
self.model.fit(train_pool, eval_set = valid_pool, use_best_model = True, **kwargs) self.model.fit(train_pool, eval_set=valid_pool, use_best_model=True, **kwargs)
evals_result = self.model.get_evals_result() evals_result = self.model.get_evals_result()
evals_result["train"] = list(evals_result["learn"].values())[0] evals_result["train"] = list(evals_result["learn"].values())[0]
@@ -72,8 +72,8 @@ class CatBoostModel(Model):
def predict(self, dataset): def predict(self, dataset):
if self.model is None: if self.model is None:
raise ValueError("model is not fitted yet!") raise ValueError("model is not fitted yet!")
x_test = dataset.prepare("test", col_set = "feature") x_test = dataset.prepare("test", col_set="feature")
return pd.Series(self.model.predict(x_test.values), index = x_test.index) return pd.Series(self.model.predict(x_test.values), index=x_test.index)
if __name__ == "__main__": if __name__ == "__main__":

6
qlib/contrib/model/pytorch_hats.py
View File

@@ -261,10 +261,12 @@ class HATS(Model):
self.logger.info("Loading pretrained model...") self.logger.info("Loading pretrained model...")
if self.base_model == "LSTM": if self.base_model == "LSTM":
from ...contrib.model.pytorch_lstm import LSTMModel from ...contrib.model.pytorch_lstm import LSTMModel
pretrained_model = LSTMModel() pretrained_model = LSTMModel()
pretrained_model.load_state_dict(torch.load("benchmarks/LSTM/model_lstm_csi300.pkl")) pretrained_model.load_state_dict(torch.load("benchmarks/LSTM/model_lstm_csi300.pkl"))
elif self.base_model == "GRU": elif self.base_model == "GRU":
from ...contrib.model.pytorch_gru import GRUModel from ...contrib.model.pytorch_gru import GRUModel
pretrained_model = GRUModel() pretrained_model = GRUModel()
pretrained_model.load_state_dict(torch.load("benchmarks/GRU/model_gru_csi300.pkl")) pretrained_model.load_state_dict(torch.load("benchmarks/GRU/model_gru_csi300.pkl"))
model_dict = self.HATS_model.state_dict() model_dict = self.HATS_model.state_dict()
@@ -461,7 +463,9 @@ class GraphAttention(nn.Module):
h = self.fcs[k](features) h = self.fcs[k](features)
nbr_h = torch.cat(tuple([h[row] for row in rows]), dim=0) nbr_h = torch.cat(tuple([h[row] for row in rows]), dim=0)
self_h = torch.cat(tuple([h[mappings[nodes[i]]].repeat(len(row), 1) for (i, row) in enumerate(rows)]), dim=0) self_h = torch.cat(
tuple([h[mappings[nodes[i]]].repeat(len(row), 1) for (i, row) in enumerate(rows)]), dim=0
)
cat_h = torch.cat((self_h, nbr_h), dim=1) cat_h = torch.cat((self_h, nbr_h), dim=1)
e = self.leakyrelu(self.a[k](cat_h)) e = self.leakyrelu(self.a[k](cat_h))

27
qlib/contrib/model/pytorch_sfm.py
View File

@@ -31,6 +31,7 @@ from ...model.base import Model
from ...data.dataset import DatasetH from ...data.dataset import DatasetH
from ...data.dataset.handler import DataHandlerLP from ...data.dataset.handler import DataHandlerLP
class SFM_Model(nn.Module): class SFM_Model(nn.Module):
def __init__(self, d_feat=6, output_dim=1, freq_dim=10, hidden_size=64, dropout_W=0.0, dropout_U=0.0, device="cpu"): def __init__(self, d_feat=6, output_dim=1, freq_dim=10, hidden_size=64, dropout_W=0.0, dropout_U=0.0, device="cpu"):
super().__init__() super().__init__()
@@ -75,13 +76,13 @@ class SFM_Model(nn.Module):
self.states = [] self.states = []
def forward(self, input): def forward(self, input):
input = input.reshape(len(input), self.input_dim, -1) # [N, F, T] input = input.reshape(len(input), self.input_dim, -1) # [N, F, T]
input = input.permute(0, 2, 1) # [N, T, F] input = input.permute(0, 2, 1) # [N, T, F]
time_step = input.shape[1] time_step = input.shape[1]
for ts in range(time_step): for ts in range(time_step):
x = input[:, ts,:] x = input[:, ts, :]
if len(self.states)==0: #hasn't initialized yet if len(self.states) == 0: # hasn't initialized yet
self.init_states(x) self.init_states(x)
self.get_constants(x) self.get_constants(x)
p_tm1 = self.states[0] p_tm1 = self.states[0]
@@ -99,8 +100,9 @@ class SFM_Model(nn.Module):
x_c = torch.matmul(x * B_W[0], self.W_c) + self.b_c x_c = torch.matmul(x * B_W[0], self.W_c) + self.b_c
x_o = torch.matmul(x * B_W[0], self.W_o) + self.b_o x_o = torch.matmul(x * B_W[0], self.W_o) + self.b_o
i = self.inner_activation(x_i + torch.matmul(h_tm1 * B_U[0], self.U_i)) # not sure whether I am doing in the right unsquuze i = self.inner_activation(
x_i + torch.matmul(h_tm1 * B_U[0], self.U_i)
) # not sure whether I am doing in the right unsquuze
ste = self.inner_activation(x_ste + torch.matmul(h_tm1 * B_U[0], self.U_ste)) ste = self.inner_activation(x_ste + torch.matmul(h_tm1 * B_U[0], self.U_ste))
fre = self.inner_activation(x_fre + torch.matmul(h_tm1 * B_U[0], self.U_fre)) fre = self.inner_activation(x_fre + torch.matmul(h_tm1 * B_U[0], self.U_fre))
@@ -201,7 +203,7 @@ class SFM(Model):
dropout_U=0.0, dropout_U=0.0,
n_epochs=200, n_epochs=200,
lr=0.001, lr=0.001,
metric = "", metric="",
batch_size=2000, batch_size=2000,
early_stop=20, early_stop=20,
eval_steps=5, eval_steps=5,
@@ -234,7 +236,7 @@ class SFM(Model):
self.lr_decay_steps = lr_decay_steps self.lr_decay_steps = lr_decay_steps
self.optimizer = optimizer.lower() self.optimizer = optimizer.lower()
self.loss = loss self.loss = loss
self.device = "cuda:%d"%(GPU) if torch.cuda.is_available() else "cpu" self.device = "cuda:%d" % (GPU) if torch.cuda.is_available() else "cpu"
self.use_gpu = torch.cuda.is_available() self.use_gpu = torch.cuda.is_available()
self.seed = seed self.seed = seed
@@ -292,8 +294,8 @@ class SFM(Model):
freq_dim=self.freq_dim, freq_dim=self.freq_dim,
dropout_W=self.dropout_W, dropout_W=self.dropout_W,
dropout_U=self.dropout_U, dropout_U=self.dropout_U,
device=self.device device=self.device,
) )
if optimizer.lower() == "adam": if optimizer.lower() == "adam":
self.train_optimizer = optim.Adam(self.sfm_model.parameters(), lr=self.lr) self.train_optimizer = optim.Adam(self.sfm_model.parameters(), lr=self.lr)
elif optimizer.lower() == "gd": elif optimizer.lower() == "gd":
@@ -436,6 +438,7 @@ class SFM(Model):
def cal_ic(self, pred, label): def cal_ic(self, pred, label):
return torch.mean(pred * label) return torch.mean(pred * label)
def predict(self, dataset): def predict(self, dataset):
if not self._fitted: if not self._fitted:
raise ValueError("model is not fitted yet!") raise ValueError("model is not fitted yet!")
@@ -447,7 +450,7 @@ class SFM(Model):
sample_num = x_values.shape[0] sample_num = x_values.shape[0]
preds = [] preds = []
for begin in range(sample_num)[::self.batch_size]: for begin in range(sample_num)[:: self.batch_size]:
if sample_num - begin < self.batch_size: if sample_num - begin < self.batch_size:
end = sample_num end = sample_num
else: else:
@@ -465,8 +468,10 @@ class SFM(Model):
return pd.Series(np.concatenate(preds), index=index) return pd.Series(np.concatenate(preds), index=index)
class AverageMeter(object): class AverageMeter(object):
"""Computes and stores the average and current value""" """Computes and stores the average and current value"""
def __init__(self): def __init__(self):
self.reset() self.reset()

30
qlib/contrib/model/xgboost.py
View File

@@ -30,15 +30,15 @@ class XGBModel(Model):
def fit( def fit(
self, self,
dataset: DatasetH, dataset: DatasetH,
num_boost_round = 1000, num_boost_round=1000,
early_stopping_rounds = 50, early_stopping_rounds=50,
verbose_eval = 20, verbose_eval=20,
evals_result = dict(), evals_result=dict(),
**kwargs **kwargs
): ):
df_train, df_valid = dataset.prepare( df_train, df_valid = dataset.prepare(
["train", "valid"], col_set = ["feature", "label"], data_key = DataHandlerLP.DK_L ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
) )
x_train, y_train = df_train["feature"], df_train["label"] x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"] x_valid, y_valid = df_valid["feature"], df_valid["label"]
@@ -49,16 +49,16 @@ class XGBModel(Model):
else: else:
raise ValueError("XGBoost doesn't support multi-label training") raise ValueError("XGBoost doesn't support multi-label training")
dtrain = xgb.DMatrix(x_train.values, label = y_train_1d) dtrain = xgb.DMatrix(x_train.values, label=y_train_1d)
dvalid = xgb.DMatrix(x_valid.values, label = y_valid_1d) dvalid = xgb.DMatrix(x_valid.values, label=y_valid_1d)
self.model = xgb.train( self.model = xgb.train(
self._params, self._params,
dtrain = dtrain, dtrain=dtrain,
num_boost_round = num_boost_round, num_boost_round=num_boost_round,
evals = [(dtrain, "train"), (dvalid, "valid")], evals=[(dtrain, "train"), (dvalid, "valid")],
early_stopping_rounds = early_stopping_rounds, early_stopping_rounds=early_stopping_rounds,
verbose_eval = verbose_eval, verbose_eval=verbose_eval,
evals_result = evals_result, evals_result=evals_result,
**kwargs **kwargs
) )
evals_result["train"] = list(evals_result["train"].values())[0] evals_result["train"] = list(evals_result["train"].values())[0]
@@ -67,5 +67,5 @@ class XGBModel(Model):
def predict(self, dataset): def predict(self, dataset):
if self.model is None: if self.model is None:
raise ValueError("model is not fitted yet!") raise ValueError("model is not fitted yet!")
x_test = dataset.prepare("test", col_set = "feature") x_test = dataset.prepare("test", col_set="feature")
return pd.Series(self.model.predict(xgb.DMatrix(x_test.values)), index = x_test.index) return pd.Series(self.model.predict(xgb.DMatrix(x_test.values)), index=x_test.index)