liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-14 16:26:55 +08:00
Code Activity

Fix processor bug and format

Browse Source
This commit is contained in:
Jactus
2020-11-11 14:24:04 +08:00
parent e2d89f44fb
commit 52c0c4b7a8
8 changed files with 114 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
examples/workflow_by_code_gru.py
View File

@@ -36,15 +36,14 @@ if __name__ == "__main__":
MARKET = "csi300" MARKET = "csi300"
BENCHMARK = "SH000300" BENCHMARK = "SH000300"
################################### ###################################
# train model # train model
################################### ###################################
DATA_HANDLER_CONFIG = { DATA_HANDLER_CONFIG = {
"start_time": "2008-01-01", "start_time": "2008-01-01",
"end_time": "2020-08-01", "end_time": "2020-08-01",
"fit_start_time":"2008-01-01", "fit_start_time": "2008-01-01",
"fit_end_time":"2014-12-31", "fit_end_time": "2014-12-31",
"instruments": MARKET, "instruments": MARKET,
} }
@@ -69,37 +68,43 @@ if __name__ == "__main__":
"n_epochs": 2000, "n_epochs": 2000,
"lr": 1e-1, "lr": 1e-1,
"early_stop": 200, "early_stop": 200,
"batch_size":800, "batch_size": 800,
"smooth_steps": 5, "smooth_steps": 5,
"metric": "mse", "metric": "mse",
"loss": "mse", "loss": "mse",
"seed": 0, "seed": 0,
"GPU": 0, "GPU": 0,
} },
}, },
"dataset": { "dataset": {
"class": "DatasetH", "class": "DatasetH",
"module_path": "qlib.data.dataset", "module_path": "qlib.data.dataset",
"kwargs": { "kwargs": {
'handler': { "handler": {
"class": "ALPHA360", "class": "ALPHA360",
"module_path": "qlib.contrib.data.handler", "module_path": "qlib.contrib.data.handler",
"kwargs": DATA_HANDLER_CONFIG "kwargs": DATA_HANDLER_CONFIG,
}, },
'segments': { "segments": {
'train': ("2008-01-01", "2014-12-31"), "train": ("2008-01-01", "2014-12-31"),
'valid': ("2015-01-01", "2016-12-31",), "valid": (
'test': ("2017-01-01", "2020-08-01",), "2015-01-01",
} "2016-12-31",
} ),
"test": (
"2017-01-01",
"2020-08-01",
),
},
},
} }
# You shoud record the data in specific sequence # You shoud record the data in specific sequence
# "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'], # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
} }
# model = train_model(task) # model = train_model(task)
model = init_instance_by_config(task['model']) model = init_instance_by_config(task["model"])
dataset = init_instance_by_config(task['dataset']) dataset = init_instance_by_config(task["dataset"])
model.fit(dataset) model.fit(dataset)

46
examples/workflow_by_code_xgboost.py
View File

@@ -21,7 +21,6 @@ from qlib.utils import init_instance_by_config
if __name__ == "__main__": if __name__ == "__main__":
# use default data # use default data
provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
if not exists_qlib_data(provider_uri): if not exists_qlib_data(provider_uri):
@@ -36,15 +35,14 @@ if __name__ == "__main__":
MARKET = "csi300" MARKET = "csi300"
BENCHMARK = "SH000300" BENCHMARK = "SH000300"
################################### ###################################
# train model # train model
################################### ###################################
DATA_HANDLER_CONFIG = { DATA_HANDLER_CONFIG = {
"start_time": "2008-01-01", "start_time": "2008-01-01",
"end_time": "2020-08-01", "end_time": "2020-08-01",
"fit_start_time":"2008-01-01", "fit_start_time": "2008-01-01",
"fit_end_time":"2014-12-31", "fit_end_time": "2014-12-31",
"instruments": MARKET, "instruments": MARKET,
} }
@@ -62,43 +60,49 @@ if __name__ == "__main__":
"class": "XGBModel", "class": "XGBModel",
"module_path": "qlib.contrib.model.xgboost", "module_path": "qlib.contrib.model.xgboost",
"kwargs": { "kwargs": {
"objective": 'reg:linear', "objective": "reg:linear",
"n_estimators":5000, "n_estimators": 5000,
"colsample_bytree": 0.85, "colsample_bytree": 0.85,
"learning_rate": 0.0421, "learning_rate": 0.0421,
"subsample": 0.8789, "subsample": 0.8789,
"max_depth": 8, "max_depth": 8,
"num_leaves": 210, "num_leaves": 210,
"num_threads": 20, "num_threads": 20,
"missing":-1, "missing": -1,
"min_child_weight":1, "min_child_weight": 1,
"nthread":4, "nthread": 4,
"tree_method":'hist', "tree_method": "hist",
} },
}, },
"dataset": { "dataset": {
"class": "DatasetH", "class": "DatasetH",
"module_path": "qlib.data.dataset", "module_path": "qlib.data.dataset",
"kwargs": { "kwargs": {
'handler': { "handler": {
"class": "Alpha158", "class": "Alpha158",
"module_path": "qlib.contrib.data.handler", "module_path": "qlib.contrib.data.handler",
"kwargs": DATA_HANDLER_CONFIG "kwargs": DATA_HANDLER_CONFIG,
}, },
'segments': { "segments": {
'train': ("2008-01-01", "2014-12-31"), "train": ("2008-01-01", "2014-12-31"),
'valid': ("2015-01-01", "2016-12-31",), "valid": (
'test': ("2017-01-01", "2020-08-01",), "2015-01-01",
} "2016-12-31",
} ),
"test": (
"2017-01-01",
"2020-08-01",
),
},
},
} }
# You shoud record the data in specific sequence # You shoud record the data in specific sequence
# "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'], # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
} }
# model = train_model(task) # model = train_model(task)
model = init_instance_by_config(task['model']) model = init_instance_by_config(task["model"])
dataset = init_instance_by_config(task['dataset']) dataset = init_instance_by_config(task["dataset"])
model.fit(dataset) model.fit(dataset)
pred_score = model.predict(dataset) pred_score = model.predict(dataset)

58
qlib/contrib/data/handler.py
View File

@@ -8,15 +8,9 @@ from ...data.dataset import processor as processor_module
from ...log import TimeInspector from ...log import TimeInspector
import copy import copy
class ALPHA360(DataHandlerLP): class ALPHA360(DataHandlerLP):
def __init__( def __init__(self, instruments="csi500", start_time=None, end_time=None, fit_start_time=None, fit_end_time=None):
self,
instruments="csi500",
start_time=None,
end_time=None,
fit_start_time=None,
fit_end_time=None
):
data_loader = { data_loader = {
"class": "QlibDataLoader", "class": "QlibDataLoader",
"kwargs": { "kwargs": {
@@ -28,22 +22,22 @@ class ALPHA360(DataHandlerLP):
} }
learn_processors = [ learn_processors = [
{"class": "DropnaLabel", "kwargs": {'group': 'label'}}, {"class": "DropnaLabel", "kwargs": {"group": "label"}},
{"class": "CSZScoreNorm", "kwargs": {"fields_group": "label"}}, {"class": "CSZScoreNorm", "kwargs": {"fields_group": "label"}},
] ]
infer_processors = [ infer_processors = [
{"class": "ProcessInf", "kwargs": {}}, {"class": "ProcessInf", "kwargs": {}},
{"class": "ZscoreNorm", "kwargs": {"fit_start_time": fit_start_time, "fit_end_time": fit_end_time}}, {"class": "ZscoreNorm", "kwargs": {"fit_start_time": fit_start_time, "fit_end_time": fit_end_time}},
{"class": "Fillna", "kwargs": {}}, {"class": "Fillna", "kwargs": {}},
] ]
super().__init__( super().__init__(
instruments, instruments,
start_time, start_time,
end_time, end_time,
data_loader=data_loader, data_loader=data_loader,
learn_processors=learn_processors, learn_processors=learn_processors,
infer_processors=infer_processors infer_processors=infer_processors,
) )
def get_label_config(self): def get_label_config(self):
@@ -54,19 +48,19 @@ class ALPHA360(DataHandlerLP):
fields = [] fields = []
names = [] names = []
for i in range(59,0,-1): for i in range(59, 0, -1):
fields += ["Ref($close, %d)/$close"%(i)] fields += ["Ref($close, %d)/$close" % (i)]
names += ["CLOSE%d"%(i)] names += ["CLOSE%d" % (i)]
fields += ["Ref($open, %d)/$close"%(i)] fields += ["Ref($open, %d)/$close" % (i)]
names += ["OPEN%d"%(i)] names += ["OPEN%d" % (i)]
fields += ["Ref($high, %d)/$close"%(i)] fields += ["Ref($high, %d)/$close" % (i)]
names += ["HIGH%d"%(i)] names += ["HIGH%d" % (i)]
fields += ["Ref($low, %d)/$close"%(i)] fields += ["Ref($low, %d)/$close" % (i)]
names += ["LOW%d"%(i)] names += ["LOW%d" % (i)]
fields += ["Ref($vwap, %d)/$close"%(i)] fields += ["Ref($vwap, %d)/$close" % (i)]
names += ["VWAP%d"%(i)] names += ["VWAP%d" % (i)]
fields += ["Ref($volume, %d)/$volume"%(i)] fields += ["Ref($volume, %d)/$volume" % (i)]
names += ["VOLUME%d"%(i)] names += ["VOLUME%d" % (i)]
fields += ["$close/$close"] fields += ["$close/$close"]
fields += ["$open/$close"] fields += ["$open/$close"]

14
qlib/contrib/model/pytorch_gru.py
View File

@@ -22,6 +22,7 @@ from ...model.base import Model
from ...data.dataset import DatasetH from ...data.dataset import DatasetH
from ...data.dataset.handler import DataHandlerLP from ...data.dataset.handler import DataHandlerLP
class GRU(Model): class GRU(Model):
"""GRU Model """GRU Model
@@ -127,7 +128,9 @@ class GRU(Model):
raise NotImplementedError("loss {} is not supported!".format(loss)) raise NotImplementedError("loss {} is not supported!".format(loss))
self._scorer = mean_squared_error if loss == "mse" else roc_auc_score self._scorer = mean_squared_error if loss == "mse" else roc_auc_score
self.gru_model = GRUModel(d_feat=self.d_feat, hidden_size=self.hidden_size, num_layers=self.num_layers, dropout=self.dropout) self.gru_model = GRUModel(
d_feat=self.d_feat, hidden_size=self.hidden_size, num_layers=self.num_layers, dropout=self.dropout
)
if optimizer.lower() == "adam": if optimizer.lower() == "adam":
self.train_optimizer = optim.Adam(self.gru_model.parameters(), lr=self.lr) self.train_optimizer = optim.Adam(self.gru_model.parameters(), lr=self.lr)
elif optimizer.lower() == "gd": elif optimizer.lower() == "gd":
@@ -262,7 +265,7 @@ class GRU(Model):
def get_loss(self, pred, target, loss_type): def get_loss(self, pred, target, loss_type):
if loss_type == "mse": if loss_type == "mse":
sqr_loss = (pred - target)**2 sqr_loss = (pred - target) ** 2
loss = sqr_loss.mean() loss = sqr_loss.mean()
return loss return loss
elif loss_type == "binary": elif loss_type == "binary":
@@ -307,6 +310,7 @@ class GRU(Model):
self.gru_model.load_state_dict(torch.load(_model_path)) self.gru_model.load_state_dict(torch.load(_model_path))
self._fitted = True self._fitted = True
class AverageMeter(object): class AverageMeter(object):
"""Computes and stores the average and current value""" """Computes and stores the average and current value"""
@@ -327,7 +331,6 @@ class AverageMeter(object):
class GRUModel(nn.Module): class GRUModel(nn.Module):
def __init__(self, d_feat=6, hidden_size=64, num_layers=2, dropout=0.0): def __init__(self, d_feat=6, hidden_size=64, num_layers=2, dropout=0.0):
super().__init__() super().__init__()
@@ -344,8 +347,7 @@ class GRUModel(nn.Module):
def forward(self, x): def forward(self, x):
# x: [N, F*T] # x: [N, F*T]
x = x.reshape(len(x), self.d_feat, -1) # [N, F, T] x = x.reshape(len(x), self.d_feat, -1) # [N, F, T]
x = x.permute(0, 2, 1) # [N, T, F] x = x.permute(0, 2, 1) # [N, T, F]
out, _ = self.rnn(x) out, _ = self.rnn(x)
return self.fc_out(out[:, -1, :]).squeeze() return self.fc_out(out[:, -1, :]).squeeze()

4
qlib/contrib/model/xgboost.py
View File

@@ -41,14 +41,14 @@ class XGBModel(Model):
y_train_1d, y_valid_1d = np.squeeze(y_train.values), np.squeeze(y_valid.values) y_train_1d, y_valid_1d = np.squeeze(y_train.values), np.squeeze(y_valid.values)
else: else:
raise ValueError("XGBoost doesn't support multi-label training") raise ValueError("XGBoost doesn't support multi-label training")
dtrain = xgb.DMatrix(x_train.values, label=y_train_1d) dtrain = xgb.DMatrix(x_train.values, label=y_train_1d)
dvalid = xgb.DMatrix(x_valid.values, label=y_valid_1d) dvalid = xgb.DMatrix(x_valid.values, label=y_valid_1d)
self.model = xgb.train( self.model = xgb.train(
self._params, self._params,
dtrain=dtrain, dtrain=dtrain,
num_boost_round=num_boost_round, num_boost_round=num_boost_round,
evals=[(dtrain, 'train'), (dvalid, 'valid')], evals=[(dtrain, "train"), (dvalid, "valid")],
early_stopping_rounds=early_stopping_rounds, early_stopping_rounds=early_stopping_rounds,
verbose_eval=verbose_eval, verbose_eval=verbose_eval,
evals_result=evals_result, evals_result=evals_result,

25
qlib/data/dataset/handler.py
View File

@@ -16,7 +16,7 @@ from ...data import D
from ...config import C from ...config import C
from ...utils import parse_config, transform_end_date, init_instance_by_config from ...utils import parse_config, transform_end_date, init_instance_by_config
from ...utils.serial import Serializable from ...utils.serial import Serializable
from .utils import get_level_index from .utils import get_level_index, fetch_df_by_index
from pathlib import Path from pathlib import Path
from .loader import DataLoader from .loader import DataLoader
@@ -99,25 +99,6 @@ class DataHandler(Serializable):
self._data = self.data_loader.load(self.instruments, self.start_time, self.end_time) self._data = self.data_loader.load(self.instruments, self.start_time, self.end_time)
# TODO: cache # TODO: cache
def _fetch_df_by_index(
self, df: pd.DataFrame, selector: Union[pd.Timestamp, slice, str, list], level: Union[str, int]
) -> pd.DataFrame:
"""
fetch data from `data` with `selector` and `level`
Parameters
----------
selector : Union[pd.Timestamp, slice, str, list]
selector
level : Union[int, str]
the level to use the selector
"""
# Try to get the right index
idx_slc = (selector, slice(None, None))
if get_level_index(df, level) == 1:
idx_slc = idx_slc[1], idx_slc[0]
return df.loc(axis=0)[idx_slc]
CS_ALL = "__all" CS_ALL = "__all"
def _fetch_df_by_col(self, df: pd.DataFrame, col_set: str) -> pd.DataFrame: def _fetch_df_by_col(self, df: pd.DataFrame, col_set: str) -> pd.DataFrame:
@@ -156,7 +137,7 @@ class DataHandler(Serializable):
------- -------
pd.DataFrame: pd.DataFrame:
""" """
df = self._fetch_df_by_index(self._data, selector, level) df = fetch_df_by_index(self._data, selector, level)
df = self._fetch_df_by_col(df, col_set) df = self._fetch_df_by_col(df, col_set)
if squeeze: if squeeze:
# squeeze columns # squeeze columns
@@ -414,7 +395,7 @@ class DataHandlerLP(DataHandler):
pd.DataFrame: pd.DataFrame:
""" """
df = self._get_df_by_key(data_key) df = self._get_df_by_key(data_key)
df = self._fetch_df_by_index(df, selector, level) df = fetch_df_by_index(df, selector, level)
return self._fetch_df_by_col(df, col_set) return self._fetch_df_by_col(df, col_set)
def get_cols(self, col_set=DataHandler.CS_ALL, data_key: str = DK_I) -> list: def get_cols(self, col_set=DataHandler.CS_ALL, data_key: str = DK_I) -> list:

9
qlib/data/dataset/processor.py
View File

@@ -7,6 +7,7 @@ import pandas as pd
import copy import copy
from ...log import TimeInspector from ...log import TimeInspector
from .utils import fetch_df_by_index
from ...utils.serial import Serializable from ...utils.serial import Serializable
from ...utils.paral import datetime_groupby_apply from ...utils.paral import datetime_groupby_apply
@@ -106,6 +107,7 @@ class ProcessInf(Processor):
return replace_inf(df) return replace_inf(df)
class Fillna(Processor): class Fillna(Processor):
"""Process infinity """ """Process infinity """
@@ -123,14 +125,15 @@ class Fillna(Processor):
return fill_na(df) return fill_na(df)
class MinMaxNorm(Processor): class MinMaxNorm(Processor):
def __init__(self, fit_start_time, fit_end_time, fields_group=None): def __init__(self, fit_start_time, fit_end_time, fields_group=None):
# FIXME: time is not used
self.fit_start_time = fit_start_time self.fit_start_time = fit_start_time
self.fit_end_time = fit_end_time self.fit_end_time = fit_end_time
self.fields_group = fields_group self.fields_group = fields_group
def fit(self, df): def fit(self, df):
df = fetch_df_by_index(df, slice(self.fit_start_time, self.fit_end_time), level="datetime")
cols = get_group_columns(df, self.fields_group) cols = get_group_columns(df, self.fields_group)
self.min_val = np.nanmin(df[cols].values, axis=0) self.min_val = np.nanmin(df[cols].values, axis=0)
self.max_val = np.nanmax(df[cols].values, axis=0) self.max_val = np.nanmax(df[cols].values, axis=0)
@@ -152,15 +155,15 @@ class MinMaxNorm(Processor):
class ZscoreNorm(Processor): class ZscoreNorm(Processor):
def __init__(self, fit_start_time, fit_end_time, fields_group=None): def __init__(self, fit_start_time, fit_end_time, fields_group=None):
# FIXME: time is not used
self.fit_start_time = fit_start_time self.fit_start_time = fit_start_time
self.fit_end_time = fit_end_time self.fit_end_time = fit_end_time
self.fields_group = fields_group self.fields_group = fields_group
def fit(self, df): def fit(self, df):
df = fetch_df_by_index(df, slice(self.fit_start_time, self.fit_end_time), level="datetime")
cols = get_group_columns(df, self.fields_group) cols = get_group_columns(df, self.fields_group)
self.mean_train = np.nanmean(df[cols].values, axis=0) self.mean_train = np.nanmean(df[cols].values, axis=0)
self.std_train = np.nanstd(df[cols].values, axis=0) self.std_train = np.nanstd(_df[cols].values, axis=0)
self.ignore = self.std_train == 0 self.ignore = self.std_train == 0
self.cols = cols self.cols = cols

24
qlib/data/dataset/utils.py
View File

@@ -29,3 +29,27 @@ def get_level_index(df: pd.DataFrame, level=Union[str, int]) -> int:
return level return level
else: else:
raise NotImplementedError(f"This type of input is not supported") raise NotImplementedError(f"This type of input is not supported")
def fetch_df_by_index(
df: pd.DataFrame, selector: Union[pd.Timestamp, slice, str, list], level: Union[str, int]
) -> pd.DataFrame:
"""
fetch data from `data` with `selector` and `level`
Parameters
----------
selector : Union[pd.Timestamp, slice, str, list]
selector
level : Union[int, str]
the level to use the selector
Returns
-------
Data of the given index.
"""
# Try to get the right index
idx_slc = (selector, slice(None, None))
if get_level_index(df, level) == 1:
idx_slc = idx_slc[1], idx_slc[0]
return df.loc(axis=0)[idx_slc]