Merge pull request #650 from microsoft/backtest_improve

Improve the backtest design and APIs
2026-07-04 19:41:00 +08:00 · 2021-11-08 09:10:33 +08:00
parent b18132dce1 3fa48d7017
commit 7800dd4ec9
65 changed files with 947 additions and 286 deletions
--- a/examples/benchmarks/ALSTM/workflow_config_alstm_Alpha158.yaml
+++ b/examples/benchmarks/ALSTM/workflow_config_alstm_Alpha158.yaml
@@ -36,8 +36,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/ALSTM/workflow_config_alstm_Alpha360.yaml
+++ b/examples/benchmarks/ALSTM/workflow_config_alstm_Alpha360.yaml
@@ -28,8 +28,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
@@ -86,4 +87,4 @@ task:
        - class: PortAnaRecord
          module_path: qlib.workflow.record_temp
          kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config
--- a/examples/benchmarks/CatBoost/workflow_config_catboost_Alpha158.yaml
+++ b/examples/benchmarks/CatBoost/workflow_config_catboost_Alpha158.yaml
@@ -14,8 +14,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/CatBoost/workflow_config_catboost_Alpha360.yaml
+++ b/examples/benchmarks/CatBoost/workflow_config_catboost_Alpha360.yaml
@@ -21,8 +21,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha158.yaml
+++ b/examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha158.yaml
@@ -14,8 +14,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha360.yaml
+++ b/examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha360.yaml
@@ -21,8 +21,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
@@ -100,4 +101,4 @@ task:
        - class: PortAnaRecord
          module_path: qlib.workflow.record_temp
          kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config
--- a/examples/benchmarks/GATs/workflow_config_gats_Alpha158.yaml
+++ b/examples/benchmarks/GATs/workflow_config_gats_Alpha158.yaml
@@ -35,8 +35,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
@@ -94,4 +95,4 @@ task:
        - class: PortAnaRecord
          module_path: qlib.workflow.record_temp
          kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config
--- a/examples/benchmarks/GATs/workflow_config_gats_Alpha360.yaml
+++ b/examples/benchmarks/GATs/workflow_config_gats_Alpha360.yaml
@@ -28,8 +28,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/GRU/workflow_config_gru_Alpha158.yaml
+++ b/examples/benchmarks/GRU/workflow_config_gru_Alpha158.yaml
@@ -36,8 +36,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/GRU/workflow_config_gru_Alpha360.yaml
+++ b/examples/benchmarks/GRU/workflow_config_gru_Alpha360.yaml
@@ -28,8 +28,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
@@ -85,4 +86,4 @@ task:
        - class: PortAnaRecord
          module_path: qlib.workflow.record_temp
          kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config
--- a/examples/benchmarks/LSTM/workflow_config_lstm_Alpha158.yaml
+++ b/examples/benchmarks/LSTM/workflow_config_lstm_Alpha158.yaml
@@ -36,8 +36,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/LSTM/workflow_config_lstm_Alpha360.yaml
+++ b/examples/benchmarks/LSTM/workflow_config_lstm_Alpha360.yaml
@@ -28,8 +28,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
@@ -85,4 +86,4 @@ task:
        - class: PortAnaRecord
          module_path: qlib.workflow.record_temp
          kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config
--- a/examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+++ b/examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
@@ -14,7 +14,7 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
+            model: <MODEL> 
            dataset: <DATASET>
            topk: 50
            n_drop: 5
--- a/examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_multi_freq.yaml
+++ b/examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_multi_freq.yaml
@@ -33,6 +33,9 @@ port_analysis_config: &port_analysis_config
        kwargs:
            topk: 50
            n_drop: 5
+            signal:
+                - <MODEL> 
+                - <DATASET>
    backtest:
        verbose: False
        limit_threshold: 0.095
@@ -80,4 +83,4 @@ task:
        - class: PortAnaRecord
          module_path: qlib.workflow.record_temp
          kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config
--- a/examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha360.yaml
+++ b/examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha360.yaml
@@ -21,8 +21,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
@@ -76,4 +77,4 @@ task:
        - class: PortAnaRecord
          module_path: qlib.workflow.record_temp
          kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config
--- a/examples/benchmarks/LightGBM/workflow_config_lightgbm_configurable_dataset.yaml
+++ b/examples/benchmarks/LightGBM/workflow_config_lightgbm_configurable_dataset.yaml
@@ -29,8 +29,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/LightGBM/workflow_config_lightgbm_multi_freq.yaml
+++ b/examples/benchmarks/LightGBM/workflow_config_lightgbm_multi_freq.yaml
@@ -31,8 +31,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/Linear/workflow_config_linear_Alpha158.yaml
+++ b/examples/benchmarks/Linear/workflow_config_linear_Alpha158.yaml
@@ -28,8 +28,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/Localformer/workflow_config_localformer_Alpha158.yaml
+++ b/examples/benchmarks/Localformer/workflow_config_localformer_Alpha158.yaml
@@ -36,8 +36,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml
+++ b/examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml
@@ -28,8 +28,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/MLP/workflow_config_mlp_Alpha158.yaml
+++ b/examples/benchmarks/MLP/workflow_config_mlp_Alpha158.yaml
@@ -41,8 +41,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
@@ -98,4 +99,4 @@ task:
        - class: PortAnaRecord
          module_path: qlib.workflow.record_temp
          kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config
--- a/examples/benchmarks/MLP/workflow_config_mlp_Alpha360.yaml
+++ b/examples/benchmarks/MLP/workflow_config_mlp_Alpha360.yaml
@@ -29,8 +29,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
@@ -85,4 +86,4 @@ task:
        - class: PortAnaRecord
          module_path: qlib.workflow.record_temp
          kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config
--- a/examples/benchmarks/README.md
+++ b/examples/benchmarks/README.md
@@ -13,6 +13,10 @@ The numbers shown below demonstrate the performance of the entire `workflow` of
 >
 > In the new version of qlib, the default dataset is **v2**. Since the data is collected from the YahooFinance API (which is not very stable), the results of *v2* and *v1* may differ -->

+> NOTE:
+> The backtest start from 0.8.0 is quite different from previous version. Please check out the changelog for the difference.
+
+
 ## Alpha158 dataset

 | Model Name                               | Dataset                             | IC          | ICIR        | Rank IC     | Rank ICIR   | Annualized Return | Information Ratio | Max Drawdown |
--- a/examples/benchmarks/SFM/workflow_config_sfm_Alpha360.yaml
+++ b/examples/benchmarks/SFM/workflow_config_sfm_Alpha360.yaml
@@ -28,8 +28,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml
+++ b/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml
@@ -30,8 +30,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
@@ -94,4 +95,4 @@ task:
        - class: PortAnaRecord
          module_path: qlib.workflow.record_temp
          kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config
--- a/examples/benchmarks/TFT/workflow_config_tft_Alpha158.yaml
+++ b/examples/benchmarks/TFT/workflow_config_tft_Alpha158.yaml
@@ -16,8 +16,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/TRA/workflow_config_tra_Alpha158.yaml
+++ b/examples/benchmarks/TRA/workflow_config_tra_Alpha158.yaml
@@ -57,8 +57,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/TRA/workflow_config_tra_Alpha158_full.yaml
+++ b/examples/benchmarks/TRA/workflow_config_tra_Alpha158_full.yaml
@@ -51,8 +51,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/TRA/workflow_config_tra_Alpha360.yaml
+++ b/examples/benchmarks/TRA/workflow_config_tra_Alpha360.yaml
@@ -51,8 +51,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/TabNet/workflow_config_TabNet_Alpha158.yaml
+++ b/examples/benchmarks/TabNet/workflow_config_TabNet_Alpha158.yaml
@@ -28,8 +28,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/TabNet/workflow_config_TabNet_Alpha360.yaml
+++ b/examples/benchmarks/TabNet/workflow_config_TabNet_Alpha360.yaml
@@ -28,8 +28,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml
+++ b/examples/benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml
@@ -36,8 +36,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml
+++ b/examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml
@@ -28,8 +28,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/XGBoost/workflow_config_xgboost_Alpha158.yaml
+++ b/examples/benchmarks/XGBoost/workflow_config_xgboost_Alpha158.yaml
@@ -14,8 +14,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/benchmarks/XGBoost/workflow_config_xgboost_Alpha360.yaml
+++ b/examples/benchmarks/XGBoost/workflow_config_xgboost_Alpha360.yaml
@@ -21,8 +21,9 @@ port_analysis_config: &port_analysis_config
        class: TopkDropoutStrategy
        module_path: qlib.contrib.strategy
        kwargs:
-            model: <MODEL>
-            dataset: <DATASET>
+            signal:
+                - <MODEL> 
+                - <DATASET>
            topk: 50
            n_drop: 5
    backtest:
--- a/examples/nested_decision_execution/workflow.py
+++ b/examples/nested_decision_execution/workflow.py
@@ -151,10 +151,9 @@ class NestedDecisionExecutionWorkflow:
        self._train_model(model, dataset)
        strategy_config = {
            "class": "TopkDropoutStrategy",
-            "module_path": "qlib.contrib.strategy.model_strategy",
+            "module_path": "qlib.contrib.strategy.signal_strategy",
            "kwargs": {
-                "model": model,
-                "dataset": dataset,
+                "signal": (model, dataset),
                "topk": 50,
                "n_drop": 5,
            },
@@ -189,10 +188,9 @@ class NestedDecisionExecutionWorkflow:
        backtest_config["benchmark"] = self.benchmark
        strategy_config = {
            "class": "TopkDropoutStrategy",
-            "module_path": "qlib.contrib.strategy.model_strategy",
+            "module_path": "qlib.contrib.strategy.signal_strategy",
            "kwargs": {
-                "model": model,
-                "dataset": dataset,
+                "signal": (model, dataset),
                "topk": 50,
                "n_drop": 5,
            },
--- a/examples/workflow_by_code.py
+++ b/examples/workflow_by_code.py
@@ -31,10 +31,9 @@ if __name__ == "__main__":
        },
        "strategy": {
            "class": "TopkDropoutStrategy",
-            "module_path": "qlib.contrib.strategy.model_strategy",
+            "module_path": "qlib.contrib.strategy.signal_strategy",
            "kwargs": {
-                "model": model,
-                "dataset": dataset,
+                "signal": (model, dataset),
                "topk": 50,
                "n_drop": 5,
            },