Update CI

.github/workflows/test.yml

@@ -50,9 +50,10 @@ jobs:
         cd tests
         pytest . --durations=0
 
-    - name: Test data downloads and examples
+    - name: Test data downloads
       run: |
         python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
-        # cd examples
+
-        # estimator -c estimator/estimator_config.yaml
+    - name: Test workflow by config
-        # jupyter nbconvert --execute estimator/analyze_from_estimator.ipynb --to html
+      run: |
+        workflow_by_config examples/benchmarks/GBDT/workflow_config_gbdt.yaml

examples/workflow_by_code_gats.py

@@ -72,7 +72,7 @@ if __name__ == "__main__":
                 "batch_size": 800,
                 "metric": "IC",
                 "loss": "mse",
-                "base_model":"GRU",
+                "base_model": "GRU",
                 "seed": 0,
                 "GPU": 0,
             },

qlib/utils/__init__.py

@@ -659,7 +659,7 @@ def flatten_dict(d, parent_key="", sep="."):
     items = []
     for k, v in d.items():
         new_key = parent_key + sep + k if parent_key else k
-        if isinstance(v, collections.MutableMapping):
+        if isinstance(v, collections.abc.MutableMapping):
             items.extend(flatten_dict(v, new_key, sep=sep).items())
         else:
             items.append((new_key, v))

tests/test_all_pipeline.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 import sys
+import shutil
 import unittest
 from pathlib import Path
 
@@ -10,7 +11,7 @@ import pandas as pd
 from scipy.stats import pearsonr
 
 import qlib
-from qlib.config import REG_CN
+from qlib.config import REG_CN, C
 from qlib.utils import drop_nan_by_y_index
 from qlib.contrib.model.gbdt import LGBModel
 from qlib.contrib.data.handler import Alpha158
@@ -19,51 +20,78 @@ from qlib.contrib.evaluate import (
     backtest as normal_backtest,
     risk_analysis,
 )
-from qlib.utils import exists_qlib_data
+from qlib.utils import exists_qlib_data, init_instance_by_config, flatten_dict
+from qlib.workflow import R
+from qlib.workflow.record_temp import SignalRecord, PortAnaRecord
 
 
-DATA_HANDLER_CONFIG = {
+market = "csi300"
-    "dropna_label": True,
+benchmark = "SH000300"
-    "start_date": "2008-01-01",
+
-    "end_date": "2020-08-01",
+###################################
-    "market": "CSI300",
+# train model
+###################################
+data_handler_config = {
+    "start_time": "2008-01-01",
+    "end_time": "2020-08-01",
+    "fit_start_time": "2008-01-01",
+    "fit_end_time": "2014-12-31",
+    "instruments": market,
 }
 
-MODEL_CONFIG = {
+task = {
-    "loss": "mse",
+    "model": {
-    "colsample_bytree": 0.8879,
+        "class": "LGBModel",
-    "learning_rate": 0.0421,
+        "module_path": "qlib.contrib.model.gbdt",
-    "subsample": 0.8789,
+        "kwargs": {
-    "lambda_l1": 205.6999,
+            "loss": "mse",
-    "lambda_l2": 580.9768,
+            "colsample_bytree": 0.8879,
-    "max_depth": 8,
+            "learning_rate": 0.0421,
-    "num_leaves": 210,
+            "subsample": 0.8789,
-    "num_threads": 20,
+            "lambda_l1": 205.6999,
+            "lambda_l2": 580.9768,
+            "max_depth": 8,
+            "num_leaves": 210,
+            "num_threads": 20,
+        },
+    },
+    "dataset": {
+        "class": "DatasetH",
+        "module_path": "qlib.data.dataset",
+        "kwargs": {
+            "handler": {
+                "class": "Alpha158",
+                "module_path": "qlib.contrib.data.handler",
+                "kwargs": data_handler_config,
+            },
+            "segments": {
+                "train": ("2008-01-01", "2014-12-31"),
+                "valid": ("2015-01-01", "2016-12-31"),
+                "test": ("2017-01-01", "2020-08-01"),
+            },
+        },
+    },
 }
 
-TRAINER_CONFIG = {
+port_analysis_config = {
-    "train_start_date": "2008-01-01",
+    "strategy": {
-    "train_end_date": "2014-12-31",
+        "class": "TopkDropoutStrategy",
-    "validate_start_date": "2015-01-01",
+        "module_path": "qlib.contrib.strategy.strategy",
-    "validate_end_date": "2016-12-31",
+        "kwargs": {
-    "test_start_date": "2017-01-01",
+            "topk": 50,
-    "test_end_date": "2020-08-01",
+            "n_drop": 5,
-}
+        },
-
+    },
-STRATEGY_CONFIG = {
+    "backtest": {
-    "topk": 50,
+        "verbose": False,
-    "n_drop": 5,
+        "limit_threshold": 0.095,
-}
+        "account": 100000000,
-
+        "benchmark": benchmark,
-BACKTEST_CONFIG = {
+        "deal_price": "close",
-    "verbose": False,
+        "open_cost": 0.0005,
-    "limit_threshold": 0.095,
+        "close_cost": 0.0015,
-    "account": 100000000,
+        "min_cost": 5,
-    "benchmark": "SH000300",
+    },
-    "deal_price": "close",
-    "open_cost": 0.0005,
-    "close_cost": 0.0015,
-    "min_cost": 5,
 }
 
 
@@ -78,34 +106,32 @@ def train():
         performance: dict
             model performance
     """
-    # get data
-    x_train, y_train, x_validate, y_validate, x_test, y_test = Alpha158(**DATA_HANDLER_CONFIG).get_split_data(
-        **TRAINER_CONFIG
-    )
 
-    # train
+    # model initiaiton
-    model = LGBModel(**MODEL_CONFIG)
+    model = init_instance_by_config(task["model"])
-    model.fit(x_train, y_train, x_validate, y_validate)
+    dataset = init_instance_by_config(task["dataset"])
-    _pred = model.predict(x_test)
-    _pred = pd.DataFrame(_pred, index=x_test.index, columns=y_test.columns)
-    pred_score = pd.DataFrame(index=_pred.index)
-    pred_score["score"] = _pred.iloc(axis=1)[0]
 
-    # get performance
+    # start exp
-    try:
+    with R.start(experiment_name="workflow"):
-        model_score = model.score(x_test, y_test)
+        R.log_params(**flatten_dict(task))
-    except NotImplementedError:
+        model.fit(dataset)
-        model_score = None
-    # Remove rows from x, y and w, which contain Nan in any columns in y_test.
-    x_test, y_test, __ = drop_nan_by_y_index(x_test, y_test)
-    pred_test = model.predict(x_test)
-    model_pearsonr = pearsonr(np.ravel(pred_test), np.ravel(y_test.values))[0]
 
-    return pred_score, {"model_score": model_score, "model_pearsonr": model_pearsonr}
+        # prediction
+        recorder = R.get_recorder()
+        rid = recorder.id
+        sr = SignalRecord(model, dataset, recorder)
+        sr.generate()
+        pred_score = sr.load()
+
+    y_test = dataset.prepare("test", col_set="label")
+    pred_score, y_test, __ = drop_nan_by_y_index(pred_score, y_test)
+    model_pearsonr = pearsonr(np.ravel(pred_score.values), np.ravel(y_test.values))[0]
+
+    return pred_score, {"model_pearsonr": model_pearsonr}, rid
 
 
-def backtest(pred):
+def backtest_analysis(pred, rid):
-    """backtest
+    """backtest and analysis
 
     Parameters
     ----------
@@ -114,23 +140,14 @@ def backtest(pred):
 
     Returns
     -------
-    report_normal: pandas.DataFrame
+    analysis result : pandas.DataFrame
-
-    positions_normal: dict
 
     """
-    strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)
+    recorder = R.get_recorder(experiment_name="workflow", recorder_id=rid)
-    _report_normal, _positions_normal = normal_backtest(pred, strategy=strategy, **BACKTEST_CONFIG)
+    # backtest
-    return _report_normal, _positions_normal
+    par = PortAnaRecord(recorder, port_analysis_config)
-
+    par.generate()
-
+    analysis_df = par.load("port_analysis.pkl")
-def analyze(report_normal):
-    _analysis = dict()
-    _analysis["excess_return_without_cost"] = risk_analysis(report_normal["return"] - report_normal["bench"])
-    _analysis["excess_return_with_cost"] = risk_analysis(
-        report_normal["return"] - report_normal["bench"] - report_normal["cost"]
-    )
-    analysis_df = pd.concat(_analysis)  # type: pd.DataFrame
     print(analysis_df)
     return analysis_df
 
@@ -139,6 +156,7 @@ class TestAllFlow(unittest.TestCase):
     PRED_SCORE = None
     REPORT_NORMAL = None
     POSITIONS = None
+    RID = None
 
     @classmethod
     def setUpClass(cls) -> None:
@@ -154,13 +172,16 @@ class TestAllFlow(unittest.TestCase):
             )
         qlib.init(provider_uri=provider_uri, region=REG_CN)
 
+    @classmethod
+    def tearDownClass(cls) -> None:
+        shutil.rmtree(str(Path(C["exp_manager"]["kwargs"]["uri"].strip("file:")).resolve()))
+
     def test_0_train(self):
-        TestAllFlow.PRED_SCORE, model_pearsonr = train()
+        TestAllFlow.PRED_SCORE, model_pearsonr, TestAllFlow.RID = train()
         self.assertGreaterEqual(model_pearsonr["model_pearsonr"], 0, "train failed")
 
     def test_1_backtest(self):
-        TestAllFlow.REPORT_NORMAL, TestAllFlow.POSITIONS = backtest(TestAllFlow.PRED_SCORE)
+        analyze_df = backtest_analysis(TestAllFlow.PRED_SCORE, TestAllFlow.RID)
-        analyze_df = analyze(TestAllFlow.REPORT_NORMAL)
         self.assertGreaterEqual(
             analyze_df.loc(axis=0)["excess_return_with_cost", "annualized_return"].values[0],
             0.10,