liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-16 17:12:20 +08:00
Code Activity

Update black formatter

Browse Source
This commit is contained in:
Jactus
2020-11-11 09:34:10 +08:00
parent 1a8ef55dc7
commit 722655ad13
15 changed files with 111 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/test.yml vendored
View File

@@ -43,7 +43,7 @@ jobs:
- name: Lint with Black - name: Lint with Black
run: | run: |
cd .. cd ..
python -m black qlib -l 120 python -m black qlib -l 120 --check
- name: Unit tests with Pytest - name: Unit tests with Pytest
run: | run: |

34
examples/workflow_by_code.py
View File

@@ -22,7 +22,6 @@ from qlib.utils import init_instance_by_config
if __name__ == "__main__": if __name__ == "__main__":
# use default data # use default data
provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
if not exists_qlib_data(provider_uri): if not exists_qlib_data(provider_uri):
@@ -37,15 +36,14 @@ if __name__ == "__main__":
MARKET = "csi300" MARKET = "csi300"
BENCHMARK = "SH000300" BENCHMARK = "SH000300"
################################### ###################################
# train model # train model
################################### ###################################
DATA_HANDLER_CONFIG = { DATA_HANDLER_CONFIG = {
"start_time": "2008-01-01", "start_time": "2008-01-01",
"end_time": "2020-08-01", "end_time": "2020-08-01",
"fit_start_time":"2008-01-01", "fit_start_time": "2008-01-01",
"fit_end_time":"2014-12-31", "fit_end_time": "2014-12-31",
"instruments": MARKET, "instruments": MARKET,
} }
@@ -72,31 +70,37 @@ if __name__ == "__main__":
"max_depth": 8, "max_depth": 8,
"num_leaves": 210, "num_leaves": 210,
"num_threads": 20, "num_threads": 20,
} },
}, },
"dataset": { "dataset": {
"class": "DatasetH", "class": "DatasetH",
"module_path": "qlib.data.dataset", "module_path": "qlib.data.dataset",
"kwargs": { "kwargs": {
'handler': { "handler": {
"class": "Alpha158", "class": "Alpha158",
"module_path": "qlib.contrib.data.handler", "module_path": "qlib.contrib.data.handler",
"kwargs": DATA_HANDLER_CONFIG "kwargs": DATA_HANDLER_CONFIG,
}, },
'segments': { "segments": {
'train': ("2008-01-01", "2014-12-31"), "train": ("2008-01-01", "2014-12-31"),
'valid': ("2015-01-01", "2016-12-31",), "valid": (
'test': ("2017-01-01", "2020-08-01",), "2015-01-01",
} "2016-12-31",
} ),
"test": (
"2017-01-01",
"2020-08-01",
),
},
},
} }
# You shoud record the data in specific sequence # You shoud record the data in specific sequence
# "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'], # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
} }
# model = train_model(task) # model = train_model(task)
model = init_instance_by_config(task['model']) model = init_instance_by_config(task["model"])
dataset = init_instance_by_config(task['dataset']) dataset = init_instance_by_config(task["dataset"])
model.fit(dataset) model.fit(dataset)

1
qlib/contrib/data/handler.py
View File

@@ -281,7 +281,6 @@ class Alpha158(DataHandlerLP):
class Alpha158vwap(Alpha158): class Alpha158vwap(Alpha158):
def get_feature_config(self): def get_feature_config(self):
conf = { conf = {
"kbar": {}, "kbar": {},

84
qlib/contrib/model/catboost_model.py
View File

@@ -9,17 +9,17 @@ from ...data.dataset.handler import DataHandlerLP
class CatBoostModel(Model): class CatBoostModel(Model):
"""CatBoost Model""" """CatBoost Model"""
def __init__(self, loss="RMSE", **kwargs): def __init__(self, loss="RMSE", **kwargs):
# There are more options # There are more options
if loss not in {"RMSE", "Logloss"}: if loss not in {"RMSE", "Logloss"}:
raise NotImplementedError raise NotImplementedError
self._params = {"loss_function": loss} self._params = {"loss_function": loss}
self._params.update(kwargs) self._params.update(kwargs)
self.model = None self.model = None
def fit( def fit(
self, self,
dataset: DatasetH, dataset: DatasetH,
num_boost_round=1000, num_boost_round=1000,
@@ -28,47 +28,41 @@ class CatBoostModel(Model):
evals_result=dict(), evals_result=dict(),
**kwargs **kwargs
): ):
df_train, df_valid = dataset.prepare( df_train, df_valid = dataset.prepare(
["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
) )
x_train, y_train = df_train["feature"], df_train["label"] x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"] x_valid, y_valid = df_valid["feature"], df_valid["label"]
# CatBoost needs 1D array as its label # CatBoost needs 1D array as its label
if y_train.values.ndim == 2 and y_train.values.shape[1] == 1: if y_train.values.ndim == 2 and y_train.values.shape[1] == 1:
y_train_1d, y_valid_1d = np.squeeze(y_train.values), np.squeeze(y_valid.values) y_train_1d, y_valid_1d = np.squeeze(y_train.values), np.squeeze(y_valid.values)
else: else:
raise ValueError("CatBoost doesn't support multi-label training") raise ValueError("CatBoost doesn't support multi-label training")
train_pool = Pool(data = x_train, label = y_train_1d) train_pool = Pool(data=x_train, label=y_train_1d)
valid_pool = Pool(data = x_valid, label = y_valid_1d) valid_pool = Pool(data=x_valid, label=y_valid_1d)
#Initialize the catboost model # Initialize the catboost model
self._params['iterations'] = num_boost_round self._params["iterations"] = num_boost_round
self._params['early_stopping_rounds'] = early_stopping_rounds self._params["early_stopping_rounds"] = early_stopping_rounds
self._params['verbose_eval'] = verbose_eval self._params["verbose_eval"] = verbose_eval
self._params['task_type'] = "GPU" if get_gpu_device_count() > 0 else "CPU" self._params["task_type"] = "GPU" if get_gpu_device_count() > 0 else "CPU"
self.model = CatBoost(self._params, **kwargs) self.model = CatBoost(self._params, **kwargs)
#train the model # train the model
self.model.fit( self.model.fit(train_pool, eval_set=valid_pool, use_best_model=True, **kwargs)
train_pool,
eval_set = valid_pool,
use_best_model = True,
**kwargs
)
evals_result = self.model.get_evals_result() evals_result = self.model.get_evals_result()
evals_result["train"] = list(evals_result["learn"].values())[0] evals_result["train"] = list(evals_result["learn"].values())[0]
evals_result["valid"] = list(evals_result["validation"].values())[0] evals_result["valid"] = list(evals_result["validation"].values())[0]
def predict(self, dataset):
if self.model is None:
raise ValueError("model is not fitted yet!")
x_test = dataset.prepare("test", col_set="feature")
return pd.Series(self.model.predict(np.squeeze(x_test.values)), index=x_test.index)
def predict(self, dataset): if __name__ == "__main__":
if self.model is None: cat = CatBoostModel()
raise ValueError("model is not fitted yet!")
x_test = dataset.prepare("test", col_set="feature")
return pd.Series(self.model.predict(np.squeeze(x_test.values)), index=x_test.index)
if __name__ == '__main__':
cat = CatBoostModel()

4
qlib/contrib/model/pytorch_nn.py
View File

@@ -159,9 +159,7 @@ class DNNModelPytorch(Model):
x_valid, y_valid = df_valid["feature"], df_valid["label"] x_valid, y_valid = df_valid["feature"], df_valid["label"]
try: try:
wdf_train, wdf_valid = dataset.prepare( wdf_train, wdf_valid = dataset.prepare(["train", "valid"], col_set=["weight"], data_key=DataHandlerLP.DK_L)
["train", "valid"], col_set=["weight"], data_key=DataHandlerLP.DK_L
)
w_train, w_valid = wdf_train["weight"], wdf_valid["weight"] w_train, w_valid = wdf_train["weight"], wdf_valid["weight"]
except: except:
w_train = pd.DataFrame(np.ones_like(y_train.values), index=y_train.index) w_train = pd.DataFrame(np.ones_like(y_train.values), index=y_train.index)

5
qlib/data/dataset/handler.py
View File

@@ -65,8 +65,9 @@ class DataHandler(Serializable):
self.data_loader = init_instance_by_config( self.data_loader = init_instance_by_config(
data_loader, data_loader,
None if (isinstance(data_loader, dict) and 'module_path' in data_loader) else data_loader_module, None if (isinstance(data_loader, dict) and "module_path" in data_loader) else data_loader_module,
accept_types=DataLoader) accept_types=DataLoader,
)
self.instruments = instruments self.instruments = instruments
self.start_time = start_time self.start_time = start_time

6
qlib/data/dataset/loader.py
View File

@@ -14,6 +14,7 @@ class DataLoader(abc.ABC):
""" """
DataLoader is designed for loading raw data from original data source. DataLoader is designed for loading raw data from original data source.
""" """
@abc.abstractmethod @abc.abstractmethod
def load(self, instruments, start_time=None, end_time=None) -> pd.DataFrame: def load(self, instruments, start_time=None, end_time=None) -> pd.DataFrame:
""" """
@@ -53,6 +54,7 @@ class DLWParser(DataLoader):
Extracting this class so that QlibDataLoader and other dataloaders(such as QdbDataLoader) can share the fields Extracting this class so that QlibDataLoader and other dataloaders(such as QdbDataLoader) can share the fields
""" """
def __init__(self, config: Tuple[list, tuple, dict]): def __init__(self, config: Tuple[list, tuple, dict]):
""" """
Parameters Parameters
@@ -113,7 +115,8 @@ class DLWParser(DataLoader):
grp: self.load_group_df(instruments, exprs, names, start_time, end_time) grp: self.load_group_df(instruments, exprs, names, start_time, end_time)
for grp, (exprs, names) in self.fields.items() for grp, (exprs, names) in self.fields.items()
}, },
axis=1) axis=1,
)
else: else:
exprs, names = self.fields exprs, names = self.fields
df = self.load_group_df(instruments, exprs, names, start_time, end_time) df = self.load_group_df(instruments, exprs, names, start_time, end_time)
@@ -122,6 +125,7 @@ class DLWParser(DataLoader):
class QlibDataLoader(DLWParser): class QlibDataLoader(DLWParser):
"""Same as QlibDataLoader. The fields can be define by config""" """Same as QlibDataLoader. The fields can be define by config"""
def __init__(self, config: Tuple[list, tuple, dict], filter_pipe=None): def __init__(self, config: Tuple[list, tuple, dict], filter_pipe=None):
""" """
Parameters Parameters

2
qlib/utils/__init__.py
View File

@@ -195,7 +195,7 @@ def get_cls_kwargs(config: Union[dict, str], module) -> (type, dict):
def init_instance_by_config( def init_instance_by_config(
config: Union[str, dict, object], module=None, accept_types: Union[type, Tuple[type]] = tuple([]), **kwargs config: Union[str, dict, object], module=None, accept_types: Union[type, Tuple[type]] = tuple([]), **kwargs
) -> object: ) -> object:
""" """
get initialized instance with config get initialized instance with config

10
qlib/utils/paral.py
View File

@@ -5,8 +5,8 @@ from joblib import Parallel, delayed
import pandas as pd import pandas as pd
def datetime_groupby_apply(df, apply_func, axis=0, level='datetime', resample_rule="M", n_jobs=-1, skip_group=False): def datetime_groupby_apply(df, apply_func, axis=0, level="datetime", resample_rule="M", n_jobs=-1, skip_group=False):
""" datetime_groupby_apply """datetime_groupby_apply
This function will apply the `apply_func` on the datetime level index. This function will apply the `apply_func` on the datetime level index.
Parameters Parameters
@@ -26,12 +26,14 @@ def datetime_groupby_apply(df, apply_func, axis=0, level='datetime', resample_ru
Returns: Returns:
pd.DataFrame pd.DataFrame
""" """
def _naive_group_apply(df): def _naive_group_apply(df):
return df.groupby(axis=axis, level=level).apply(apply_func) return df.groupby(axis=axis, level=level).apply(apply_func)
if n_jobs != 1: if n_jobs != 1:
dfs = Parallel(n_jobs=n_jobs)(delayed(_naive_group_apply)(sub_df) dfs = Parallel(n_jobs=n_jobs)(
for idx, sub_df in df.resample(resample_rule, axis=axis, level=level)) delayed(_naive_group_apply)(sub_df) for idx, sub_df in df.resample(resample_rule, axis=axis, level=level)
)
return pd.concat(dfs, axis=axis).sort_index() return pd.concat(dfs, axis=axis).sort_index()
else: else:
return _naive_group_apply(df) return _naive_group_apply(df)

1
qlib/workflow/__init__.py
View File

@@ -6,6 +6,7 @@ from .expm import MLflowExpManager
from ..utils import Wrapper from ..utils import Wrapper
from ..config import C from ..config import C
class QlibRecorder: class QlibRecorder:
""" """
A global system that helps to manage the experiments. A global system that helps to manage the experiments.

22
qlib/workflow/exp.py
View File

@@ -8,6 +8,7 @@ from ..log import get_module_logger
logger = get_module_logger("workflow", "INFO") logger = get_module_logger("workflow", "INFO")
class Experiment: class Experiment:
""" """
Thie is the `Experiment` class for each experiment being run. The API is designed Thie is the `Experiment` class for each experiment being run. The API is designed
@@ -17,7 +18,7 @@ class Experiment:
self.name = None self.name = None
self.id = None self.id = None
self.active_recorder = None # only one recorder can running each time self.active_recorder = None # only one recorder can running each time
self.recorders = dict() # recorder id -> object self.recorders = dict() # recorder id -> object
def __repr__(self): def __repr__(self):
return str(self.info) return str(self.info)
@@ -28,11 +29,11 @@ class Experiment:
@property @property
def info(self): def info(self):
output = dict() output = dict()
output['class'] = "Experiment" output["class"] = "Experiment"
output['id'] = self.id output["id"] = self.id
output['name'] = self.name output["name"] = self.name
output['active_recorder'] = self.active_recorder.id output["active_recorder"] = self.active_recorder.id
output['recorders'] = list(self.recorders.keys()) output["recorders"] = list(self.recorders.keys())
def start(self): def start(self):
""" """
@@ -137,7 +138,6 @@ class MLflowExperiment(Experiment):
run = self.active_recorder.start_run() run = self.active_recorder.start_run()
# store the recorder # store the recorder
self.recorders[self.active_recorder.id] = recorder self.recorders[self.active_recorder.id] = recorder
return self.active_recorder return self.active_recorder
def end(self, status): def end(self, status):
@@ -147,7 +147,7 @@ class MLflowExperiment(Experiment):
def create_recorder(self): def create_recorder(self):
num = len(self.recorders) num = len(self.recorders)
name = "Recorder_{}".format(num+1) name = "Recorder_{}".format(num + 1)
recorder = MLflowRecorder(name, self.id) recorder = MLflowRecorder(name, self.id)
return recorder return recorder
@@ -170,9 +170,7 @@ class MLflowExperiment(Experiment):
if self.recorders[rid].name == recorder_name: if self.recorders[rid].name == recorder_name:
return self.recorders[rid] return self.recorders[rid]
elif self.active_recorder is None: elif self.active_recorder is None:
raise Exception('No valid active recorder exists. Please make sure the experiment is running.') raise Exception("No valid active recorder exists. Please make sure the experiment is running.")
else: else:
logger.info( logger.info("No experiment id or name is given. Return the current active experiment.")
"No experiment id or name is given. Return the current active experiment."
)
return self.active_recorder return self.active_recorder

10
qlib/workflow/expm.py
View File

@@ -184,9 +184,7 @@ class MLflowExpManager(ExpManager):
else: else:
if experiment_name not in self.experiments: if experiment_name not in self.experiments:
if mlflow.get_experiment_by_name(experiment_name) is not None: if mlflow.get_experiment_by_name(experiment_name) is not None:
logger.info( logger.info("The experiment has already been created before. Try to resume the experiment...")
"The experiment has already been created before. Try to resume the experiment..."
)
experiment_id = mlflow.get_experiment_by_name(experiment_name).experiment_id experiment_id = mlflow.get_experiment_by_name(experiment_name).experiment_id
else: else:
experiment_id = mlflow.create_experiment(experiment_name) experiment_id = mlflow.create_experiment(experiment_name)
@@ -216,11 +214,9 @@ class MLflowExpManager(ExpManager):
if self.experiments[name].id == experiment_id: if self.experiments[name].id == experiment_id:
return self.experiments[name] return self.experiments[name]
elif self.active_experiment is None: elif self.active_experiment is None:
raise Exception('No valid active experiment exists. Please make sure experiment manager is running.') raise Exception("No valid active experiment exists. Please make sure experiment manager is running.")
else: else:
logger.info( logger.info("No experiment id or name is given. Return the current active experiment.")
"No experiment id or name is given. Return the current active experiment."
)
return self.active_experiment return self.active_experiment
def delete_exp(self, experiment_id): def delete_exp(self, experiment_id):

4
qlib/workflow/record_temp.py
View File

@@ -116,8 +116,8 @@ class PortAnaRecord(SignalRecord):
def __init__(self, recorder, config, **kwargs): def __init__(self, recorder, config, **kwargs):
self.recorder = recorder self.recorder = recorder
self.strategy_config = config['strategy'] self.strategy_config = config["strategy"]
self.backtest_config = config['backtest'] self.backtest_config = config["backtest"]
self.strategy = init_instance_by_config(self.strategy_config) self.strategy = init_instance_by_config(self.strategy_config)
self.artifact_path = "portfolio_analysis" self.artifact_path = "portfolio_analysis"

20
qlib/workflow/recorder.py
View File

@@ -30,11 +30,11 @@ class Recorder:
@property @property
def info(self): def info(self):
output = dict() output = dict()
output['class'] = "Recorder" output["class"] = "Recorder"
output['id'] = self.id output["id"] = self.id
output['name'] = self.name output["name"] = self.name
output['experiment_id'] = self.experiment_id output["experiment_id"] = self.experiment_id
output['status'] = self.status output["status"] = self.status
def set_recorder_name(self, rname): def set_recorder_name(self, rname):
self.recorder_name = rname self.recorder_name = rname
@@ -188,16 +188,16 @@ class MLflowRecorder(Recorder):
client = mlflow.tracking.MlflowClient(tracking_uri=self._uri) client = mlflow.tracking.MlflowClient(tracking_uri=self._uri)
if local_path is not None: if local_path is not None:
client.log_artifacts(self.id, local_path, artifact_path) client.log_artifacts(self.id, local_path, artifact_path)
elif kwargs.get('data') is not None and kwargs.get('name') is not None: elif kwargs.get("data") is not None and kwargs.get("name") is not None:
data, name = kwargs.get('data'), kwargs.get('name') data, name = kwargs.get("data"), kwargs.get("name")
self.fm.save_obj(data, name) self.fm.save_obj(data, name)
client.log_artifact(self.id, self.fm.path / name, artifact_path) client.log_artifact(self.id, self.fm.path / name, artifact_path)
elif kwargs.get('data_name_list') is not None: elif kwargs.get("data_name_list") is not None:
data_name_list = kwargs.get('data_name_list') data_name_list = kwargs.get("data_name_list")
self.fm.save_objs(data_name_list) self.fm.save_objs(data_name_list)
client.log_artifacts(self.id, self.fm.path, artifact_path) client.log_artifacts(self.id, self.fm.path, artifact_path)
else: else:
raise Exception('Please provide valid arguments in order to save object properly.') raise Exception("Please provide valid arguments in order to save object properly.")
def load_object(self, name): def load_object(self, name):
client = mlflow.tracking.MlflowClient(tracking_uri=self._uri) client = mlflow.tracking.MlflowClient(tracking_uri=self._uri)

2
setup.py
View File

@@ -55,7 +55,7 @@ REQUIRED = [
"loguru", "loguru",
"lightgbm", "lightgbm",
"tornado", "tornado",
"joblib>=0.17.0" "joblib>=0.17.0",
] ]
# Numpy include # Numpy include