temporarily fix create exp conflicts for remote mlflow

* Auto injecting model and dataset for Recorder

* Support using Feature in expression

.github/workflows/python-publish.yml

@@ -12,8 +12,9 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, macos-latest]
-        python-version: [3.6, 3.7, 3.8]
+        os: [windows-latest, macos-latest, macos-11]
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-version: [3.7, 3.8]
 
     steps:
     - uses: actions/checkout@v2
@@ -44,7 +45,8 @@ jobs:
     - name: Build wheel on Linux
       uses: RalfG/python-wheels-manylinux-build@v0.3.1-manylinux2010_x86_64
       with:
-        python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38'
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-versions: 'cp37-cp37m cp38-cp38'
         build-requirements: 'numpy cython'
     - name: Set up Python
       uses: actions/setup-python@v2

.github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: Test 
+name: Test
 
 on:
   push:
@@ -12,8 +12,9 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-16.04, ubuntu-18.04, ubuntu-20.04, macos-latest]
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        os: [windows-latest, ubuntu-18.04, ubuntu-20.04]
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-version: [3.7, 3.8]
 
     steps:
     - uses: actions/checkout@v2
@@ -25,96 +26,41 @@ jobs:
 
     - name: Lint with Black
       run: |
-        cd ..
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe -m pip install black
-          $CONDA\\python.exe -m black qlib -l 120 --check --diff
-        else
-          sudo $CONDA/bin/python -m pip install black
-          $CONDA/bin/python -m black qlib -l 120 --check --diff
-        fi
-      shell: bash
+        pip install --upgrade pip
+        pip install black wheel
+        black qlib -l 120 --check --diff
 
-    # Test Qlib installed with pip
     - name: Install Qlib with pip
       run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe -m pip install numpy==1.19.5
-          $CONDA\\python.exe -m pip install pyqlib --ignore-installed ruamel.yaml numpy --user
-        else
-          sudo $CONDA/bin/python -m pip install numpy==1.19.5
-          sudo $CONDA/bin/python -m pip install pyqlib --ignore-installed ruamel.yaml numpy
-        fi
-      shell: bash
-    
-    - name: Install Lightgbm for MacOS
-      if: runner.os == 'macOS'
-      run: |
-        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
-        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
+        pip install numpy==1.19.5 ruamel.yaml
+        pip install pyqlib --ignore-installed 
 
     - name: Test data downloads
       run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
-        else
-          $CONDA/bin/python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
-        fi
-      shell: bash
+        python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
 
     - name: Test workflow by config (install from pip)
       run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe qlib\\workflow\\cli.py examples\\benchmarks\\LightGBM\\workflow_config_lightgbm_Alpha158.yaml 
-          $CONDA\\python.exe -m pip uninstall -y pyqlib
-        else
-          $CONDA/bin/python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml 
-          sudo $CONDA/bin/python -m pip uninstall -y pyqlib
-        fi
-      shell: bash
-      
-    # Test Qlib installed from source
+        python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+        python -m pip uninstall -y pyqlib
+
+     # Test Qlib installed from source
     - name: Install Qlib from source
       run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe -m pip install --upgrade cython
-          $CONDA\\python.exe -m pip install numpy jupyter jupyter_contrib_nbextensions
-          $CONDA\\python.exe -m pip install -U scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't
-          $CONDA\\python.exe setup.py install
-        else
-          sudo $CONDA/bin/python -m pip install --upgrade cython
-          sudo $CONDA/bin/python -m pip install numpy jupyter jupyter_contrib_nbextensions
-          sudo $CONDA/bin/python -m pip install -U scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't
-          sudo $CONDA/bin/python setup.py install
-        fi
-      shell: bash
+        pip install --upgrade cython jupyter jupyter_contrib_nbextensions numpy scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't
+        pip install -e .
 
     - name: Install test dependencies
       run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe -m pip install --upgrade pip
-          $CONDA\\python.exe -m pip install black pytest
-        else
-          sudo $CONDA/bin/python -m pip install --upgrade pip
-          sudo $CONDA/bin/python -m pip install black pytest
-        fi
-      shell: bash 
+        pip install --upgrade pip
+        pip install black pytest
 
     - name: Unit tests with Pytest
       run: |
         cd tests
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe -m pytest . --durations=0
-        else
-          $CONDA/bin/python -m pytest . --durations=0
-        fi
-      shell: bash
+        python -m pytest . --durations=10
 
     - name: Test workflow by config (install from source)
       run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe qlib\\workflow\\cli.py examples\\benchmarks\\LightGBM\\workflow_config_lightgbm_Alpha158.yaml 
-        else
-          $CONDA/bin/python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml 
-        fi
-      shell: bash
+        python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+      

.github/workflows/test_macos.yml

@@ -0,0 +1,75 @@
+# There are some issues (in the downloading data phase) on MacOS when running with other tests. So we split it into an individual config.
+name: Test MacOS
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-11, macos-latest]
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-version: [3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Lint with Black
+      run: |
+        cd ..
+        python -m pip install pip --upgrade
+        python -m pip install wheel --upgrade
+        python -m pip install black
+        python -m black qlib -l 120 --check --diff
+    # Test Qlib installed with pip
+
+    - name: Install Qlib with pip
+      run: |
+          python -m pip install numpy==1.19.5
+          python -m pip install pyqlib --ignore-installed ruamel.yaml numpy
+    - name: Install Lightgbm for MacOS
+      run: |
+        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
+        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
+        # FIX MacOS error: Segmentation fault
+        # reference: https://github.com/microsoft/LightGBM/issues/4229
+        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
+        brew unlink libomp
+        brew install libomp.rb
+    - name: Test data downloads
+      run: |
+        python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
+    - name: Test workflow by config (install from pip)
+      run: |
+        python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+        python -m pip uninstall -y pyqlib
+    # Test Qlib installed from source
+    - name: Install Qlib from source
+      run: |
+        python -m pip install --upgrade cython
+        python -m pip install numpy jupyter jupyter_contrib_nbextensions
+        python -m pip install -U scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't
+        python setup.py install
+    - name: Install test dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -U pyopenssl idna
+        python -m pip install black pytest
+    - name: Unit tests with Pytest
+      run: |
+        cd tests
+        python -m pytest . --durations=0
+    - name: Test workflow by config (install from source)
+      run: |
+          python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml

.gitignore

@@ -20,6 +20,7 @@ dist/
 .nvimrc
 .vscode
 
+qlib/VERSION.txt
 qlib/data/_libs/expanding.cpp
 qlib/data/_libs/rolling.cpp
 examples/estimator/estimator_example/

CHANGES.rst

@@ -159,6 +159,21 @@ Version 0.5.0
     - Add baselines
     - public data crawler
 
-Version greater than Version 0.5.0
+
+Version 0.8.0
+--------------------
+- The backtest is greatly refactored.
+    - Nested decision execution framework is supported
+    - There are lots of changes for daily trading, it is hard to list all of them. But a few important changes could be noticed
+        - The trading limitation is more accurate;
+            - In `previous version <https://github.com/microsoft/qlib/blob/v0.7.2/qlib/contrib/backtest/exchange.py#L160>`_, longing and shorting actions share the same action.
+            - In `current verison <https://github.com/microsoft/qlib/blob/7c31012b507a3823117bddcc693fc64899460b2a/qlib/backtest/exchange.py#L304>`_, the trading limitation is different between loging and shorting action.
+        - The constant is different when calculating annualized metrics.
+            - `Current version <https://github.com/microsoft/qlib/blob/7c31012b507a3823117bddcc693fc64899460b2a/qlib/contrib/evaluate.py#L42>`_ uses more accurate constant than `previous version <https://github.com/microsoft/qlib/blob/v0.7.2/qlib/contrib/evaluate.py#L22>`_
+        - `A new version <https://github.com/microsoft/qlib/blob/7c31012b507a3823117bddcc693fc64899460b2a/qlib/tests/data.py#L17>`_ of data is released. Due to the unstability of Yahoo data source, the data may be different after downloading data again.
+        - Users could chec kout the backtesting results between  `Current version <https://github.com/microsoft/qlib/tree/7c31012b507a3823117bddcc693fc64899460b2a/examples/benchmarks>`_ and `previous version <https://github.com/microsoft/qlib/tree/v0.7.2/examples/benchmarks>`_
+
+
+Other Versions
 ----------------------------------
 Please refer to `Github release Notes <https://github.com/microsoft/qlib/releases>`_

MANIFEST.in

@@ -0,0 +1 @@
+include qlib/VERSION.txt

README.md

@@ -11,6 +11,9 @@
 Recent released features
 | Feature | Status |
 | --                      | ------    |
+|Temporal Routing Adaptor (TRA) | [Released](https://github.com/microsoft/qlib/pull/531) on July 30, 2021 |
+| Transformer & Localformer | [Released](https://github.com/microsoft/qlib/pull/508) on July 22, 2021 |
+| Release Qlib v0.7.0 | [Released](https://github.com/microsoft/qlib/releases/tag/v0.7.0) on July 12, 2021 |
 | TCTS Model | [Released](https://github.com/microsoft/qlib/pull/491) on July 1, 2021 |
 | Online serving and automatic model rolling | :star: [Released](https://github.com/microsoft/qlib/pull/290) on May 17, 2021 | 
 | DoubleEnsemble Model | [Released](https://github.com/microsoft/qlib/pull/286) on Mar 2, 2021 | 
@@ -21,10 +24,8 @@ Recent released features
 
 Features released before 2021 are not listed here.
 
-
-
 <p align="center">
-  <img src="http://fintech.msra.cn/images_v060/logo/1.png" />
+  <img src="http://fintech.msra.cn/images_v070/logo/1.png" />
 </p>
 
 
@@ -43,7 +44,7 @@ For more details, please refer to our paper ["Qlib: An AI-oriented Quantitative
   - [Data Preparation](#data-preparation)
   - [Auto Quant Research Workflow](#auto-quant-research-workflow)
   - [Building Customized Quant Research Workflow by Code](#building-customized-quant-research-workflow-by-code)
-- [**Quant Model Zoo**](#quant-model-zoo)
+- [**Quant Model(Paper) Zoo**](#quant-model-paper-zoo)
   - [Run a single model](#run-a-single-model)
   - [Run multiple models](#run-multiple-models)
 - [**Quant Dataset Zoo**](#quant-dataset-zoo)
@@ -69,7 +70,7 @@ Your feedbacks about the features are very important.
 # Framework of Qlib
 
 <div style="align: center">
-<img src="http://fintech.msra.cn/images_v060/framework.png?v=0.2" />
+<img src="docs/_static/img/framework.svg" />
 </div>
 
 
@@ -99,14 +100,14 @@ Here is a quick **[demo](https://terminalizer.com/view/3f24561a4470)** shows how
 This table demonstrates the supported Python version of `Qlib`:
 |               | install with pip           | install from source  | plot |
 | ------------- |:---------------------:|:--------------------:|:----:|
-| Python 3.6    | :heavy_check_mark:    | :heavy_check_mark: (only with `Anaconda`)                  | :heavy_check_mark: |
 | Python 3.7    | :heavy_check_mark:    | :heavy_check_mark:   | :heavy_check_mark: |
 | Python 3.8    | :heavy_check_mark:    | :heavy_check_mark:   | :heavy_check_mark: |
 | Python 3.9    | :x:                   | :heavy_check_mark:   | :x: |
 
 **Note**: 
+1. **Conda** is suggested for managing your Python environment.
 1. Please pay attention that installing cython in Python 3.6 will raise some error when installing ``Qlib`` from source. If users use Python 3.6 on their machines, it is recommended to *upgrade* Python to version 3.7 or use `conda`'s Python to install ``Qlib`` from source.
-2. For Python 3.9, `Qlib` supports running workflows such as training models, doing backtest and plot most of the related figures (those included in [notebook](examples/workflow_by_code.ipynb)). However, plotting for the *model performance* is not supported for now and we will fix this when the dependent packages are upgraded in the future.
+1. For Python 3.9, `Qlib` supports running workflows such as training models, doing backtest and plot most of the related figures (those included in [notebook](examples/workflow_by_code.ipynb)). However, plotting for the *model performance* is not supported for now and we will fix this when the dependent packages are upgraded in the future.
 
 ### Install with pip
 Users can easily install ``Qlib`` by pip according to the following command.
@@ -160,7 +161,7 @@ Users could create the same dataset with it.
 *Please pay **ATTENTION** that the data is collected from [Yahoo Finance](https://finance.yahoo.com/lookup), and the data might not be perfect.
 We recommend users to prepare their own data if they have a high-quality dataset. For more information, users can refer to the [related document](https://qlib.readthedocs.io/en/latest/component/data.html#converting-csv-format-into-qlib-format)*.
 
-### Automatic update of daily frequency data(from yahoo finance)
+### Automatic update of daily frequency data (from yahoo finance)
   > It is recommended that users update the data manually once (--trading_date 2021-05-25) and then set it to update automatically.
 
   > For more information refer to: [yahoo collector](https://github.com/microsoft/qlib/tree/main/scripts/data_collector/yahoo#automatic-update-of-daily-frequency-datafrom-yahoo-finance)
@@ -245,19 +246,19 @@ Qlib provides a tool named `qrun` to run the whole workflow automatically (inclu
 2. Graphical Reports Analysis: Run `examples/workflow_by_code.ipynb` with `jupyter notebook` to get graphical reports
     - Forecasting signal (model prediction) analysis
       - Cumulative Return of groups
-      ![Cumulative Return](http://fintech.msra.cn/images_v060/analysis/analysis_model_cumulative_return.png?v=0.1)
+      ![Cumulative Return](http://fintech.msra.cn/images_v070/analysis/analysis_model_cumulative_return.png?v=0.1)
       - Return distribution
-      ![long_short](http://fintech.msra.cn/images_v060/analysis/analysis_model_long_short.png?v=0.1)
+      ![long_short](http://fintech.msra.cn/images_v070/analysis/analysis_model_long_short.png?v=0.1)
       - Information Coefficient (IC)
-      ![Information Coefficient](http://fintech.msra.cn/images_v060/analysis/analysis_model_IC.png?v=0.1)        
-      ![Monthly IC](http://fintech.msra.cn/images_v060/analysis/analysis_model_monthly_IC.png?v=0.1)
-      ![IC](http://fintech.msra.cn/images_v060/analysis/analysis_model_NDQ.png?v=0.1)
+      ![Information Coefficient](http://fintech.msra.cn/images_v070/analysis/analysis_model_IC.png?v=0.1)
+      ![Monthly IC](http://fintech.msra.cn/images_v070/analysis/analysis_model_monthly_IC.png?v=0.1)
+      ![IC](http://fintech.msra.cn/images_v070/analysis/analysis_model_NDQ.png?v=0.1)
       - Auto Correlation of forecasting signal (model prediction)
-      ![Auto Correlation](http://fintech.msra.cn/images_v060/analysis/analysis_model_auto_correlation.png?v=0.1)
+      ![Auto Correlation](http://fintech.msra.cn/images_v070/analysis/analysis_model_auto_correlation.png?v=0.1)
 
     - Portfolio analysis
       - Backtest return
-      ![Report](http://fintech.msra.cn/images_v060/analysis/report.png?v=0.1)
+      ![Report](http://fintech.msra.cn/images_v070/analysis/report.png?v=0.1)
       <!-- 
       - Score IC
       ![Score IC](docs/_static/img/score_ic.png)
@@ -274,7 +275,7 @@ Qlib provides a tool named `qrun` to run the whole workflow automatically (inclu
 The automatic workflow may not suit the research workflow of all Quant researchers. To support a flexible Quant research workflow, Qlib also provides a modularized interface to allow researchers to build their own workflow by code. [Here](examples/workflow_by_code.ipynb) is a demo for customized Quant research workflow by code.
 
 
-# [Quant Model Zoo](examples/benchmarks)
+# [Quant Model (Paper) Zoo](examples/benchmarks)
 
 Here is a list of models built on `Qlib`.
 - [GBDT based on XGBoost (Tianqi Chen, et al. KDD 2016)](qlib/contrib/model/xgboost.py)
@@ -290,6 +291,9 @@ Here is a list of models built on `Qlib`.
 - [TabNet based on pytorch (Sercan O. Arik, et al. AAAI 2019)](qlib/contrib/model/pytorch_tabnet.py)
 - [DoubleEnsemble based on LightGBM (Chuheng Zhang, et al. ICDM 2020)](qlib/contrib/model/double_ensemble.py)
 - [TCTS based on pytorch (Xueqing Wu, et al. ICML 2021)](qlib/contrib/model/pytorch_tcts.py)
+- [Transformer based on pytorch (Ashish Vaswani, et al. NeurIPS 2017)](qlib/contrib/model/pytorch_transformer.py)
+- [Localformer based on pytorch (Juyong Jiang, et al.)](qlib/contrib/model/pytorch_localformer.py)
+- [TRA based on pytorch (Hengxu, Dong, et al. KDD 2021)](qlib/contrib/model/pytorch_tra.py)
 
 Your PR of new Quant models is highly welcomed.
 
@@ -302,16 +306,17 @@ All the models listed above are runnable with ``Qlib``. Users can find the confi
 - Users can use the tool `qrun` mentioned above to run a model's workflow based from a config file.
 - Users can create a `workflow_by_code` python script based on the [one](examples/workflow_by_code.py) listed in the `examples` folder.
 
-- Users can use the script [`run_all_model.py`](examples/run_all_model.py) listed in the `examples` folder to run a model. Here is an example of the specific shell command to be used: `python run_all_model.py --models=lightgbm`, where the `--models` arguments can take any number of models listed above(the available models can be found  in [benchmarks](examples/benchmarks/)). For more use cases, please refer to the file's [docstrings](examples/run_all_model.py).
+- Users can use the script [`run_all_model.py`](examples/run_all_model.py) listed in the `examples` folder to run a model. Here is an example of the specific shell command to be used: `python run_all_model.py run --models=lightgbm`, where the `--models` arguments can take any number of models listed above(the available models can be found  in [benchmarks](examples/benchmarks/)). For more use cases, please refer to the file's [docstrings](examples/run_all_model.py).
+    - **NOTE**: Each baseline has different environment dependencies, please make sure that your python version aligns with the requirements(e.g. TFT only supports Python 3.6~3.7 due to the limitation of `tensorflow==1.15.0`)
 
 ## Run multiple models
-`Qlib` also provides a script [`run_all_model.py`](examples/run_all_model.py) which can run multiple models for several iterations. (**Note**: the script only support *Linux* for now. Other OS will be supported in the future. Besides, it doesn't support parrallel running the same model for multiple times as well, and this will be fixed in the future development too.)
+`Qlib` also provides a script [`run_all_model.py`](examples/run_all_model.py) which can run multiple models for several iterations. (**Note**: the script only support *Linux* for now. Other OS will be supported in the future. Besides, it doesn't support parallel running the same model for multiple times as well, and this will be fixed in the future development too.)
 
 The script will create a unique virtual environment for each model, and delete the environments after training. Thus, only experiment results such as `IC` and `backtest` results will be generated and stored.
 
 Here is an example of running all the models for 10 iterations:
 ```python
-python run_all_model.py 10
+python run_all_model.py run 10
 ```
 
 It also provides the API to run specific models at once. For more use cases, please refer to the file's [docstrings](examples/run_all_model.py). 
@@ -370,9 +375,7 @@ Such overheads greatly slow down the data loading process.
 Qlib data are stored in a compact format, which is efficient to be combined into arrays for scientific computation.
 
 # Related Reports
-- [【华泰金工林晓明团队】图神经网络选股与Qlib实践——华泰人工智能系列之四十二](https://mp.weixin.qq.com/s/w5fDB6oAv9dO6vlhf1kmhA)
 - [Guide To Qlib: Microsoft’s AI Investment Platform](https://analyticsindiamag.com/qlib/)
-- [【华泰金工林晓明团队】微软AI量化投资平台Qlib体验——华泰人工智能系列之四十](https://mp.weixin.qq.com/s/Brcd7im4NibJOJzZfMn6tQ)
 - [微软也搞AI量化平台？还是开源的！](https://mp.weixin.qq.com/s/47bP5YwxfTp2uTHjUBzJQQ)
 - [微矿Qlib：业内首个AI量化投资开源平台](https://mp.weixin.qq.com/s/vsJv7lsgjEi-ALYUz4CvtQ)
 
@@ -385,11 +388,21 @@ Qlib data are stored in a compact format, which is efficient to be combined into
 Join IM discussion groups:
 |[Gitter](https://gitter.im/Microsoft/qlib)|
 |----|
-|![image](http://fintech.msra.cn/images_v060/qrcode/gitter_qr.png)|
+|![image](http://fintech.msra.cn/images_v070/qrcode/gitter_qr.png)|
 
 # Contributing
 
-This project welcomes contributions and suggestions.  Most contributions require you to agree to a
+This project welcomes contributions and suggestions.  
+**Here are some 
+[code standards](docs/developer/code_standard.rst) when you submit a pull request.**
+
+If you want to contribute to Qlib's document, you can follow the steps in the figure below.
+<p align="center">
+  <img src="https://github.com/demon143/qlib/blob/main/docs/_static/img/change%20doc.gif" />
+</p>
+
+
+Most contributions require you to agree to a
 Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
 the right to use your contribution. For details, visit https://cla.opensource.microsoft.com.
 

VERSION.txt

@@ -0,0 +1 @@
+0.7.2.99

docs/FAQ/FAQ.rst

@@ -97,4 +97,57 @@ Also, feel free to post a new issue in our GitHub repository. We always check ea
 
         python setup.py build_ext --inplace
 
-- If the error occurs when importing ``qlib`` package with command ``python`` , users need to change the running directory to ensure that the script does not run in the project directory.
+- If the error occurs when importing ``qlib`` package with command ``python`` , users need to change the running directory to ensure that the script does not run in the project directory.
+
+
+4. BadNamespaceError: / is not a connected namespace
+------------------------------------------------------------------------------------------------------------------------------------
+
+.. code-block:: python
+
+      File "qlib_online.py", line 35, in <module>
+        cal = D.calendar()
+      File "e:\code\python\microsoft\qlib_latest\qlib\qlib\data\data.py", line 973, in calendar
+        return Cal.calendar(start_time, end_time, freq, future=future)
+      File "e:\code\python\microsoft\qlib_latest\qlib\qlib\data\data.py", line 798, in calendar
+        self.conn.send_request(
+      File "e:\code\python\microsoft\qlib_latest\qlib\qlib\data\client.py", line 101, in send_request
+        self.sio.emit(request_type + "_request", request_content)
+      File "G:\apps\miniconda\envs\qlib\lib\site-packages\python_socketio-5.3.0-py3.8.egg\socketio\client.py", line 369, in emit
+        raise exceptions.BadNamespaceError(
+      BadNamespaceError: / is not a connected namespace.
+
+- The version of ``python-socketio`` in qlib needs to be the same as the version of ``python-socketio`` in qlib-server:
+
+    .. code-block:: bash
+
+        pip install -U python-socketio==<qlib-server python-socketio version>
+
+
+5. TypeError: send() got an unexpected keyword argument 'binary'
+------------------------------------------------------------------------------------------------------------------------------------
+
+.. code-block:: python
+
+      File "qlib_online.py", line 35, in <module>
+        cal = D.calendar()
+      File "e:\code\python\microsoft\qlib_latest\qlib\qlib\data\data.py", line 973, in calendar
+        return Cal.calendar(start_time, end_time, freq, future=future)
+      File "e:\code\python\microsoft\qlib_latest\qlib\qlib\data\data.py", line 798, in calendar
+        self.conn.send_request(
+      File "e:\code\python\microsoft\qlib_latest\qlib\qlib\data\client.py", line 101, in send_request
+        self.sio.emit(request_type + "_request", request_content)
+      File "G:\apps\miniconda\envs\qlib\lib\site-packages\socketio\client.py", line 263, in emit
+        self._send_packet(packet.Packet(packet.EVENT, namespace=namespace,
+      File "G:\apps\miniconda\envs\qlib\lib\site-packages\socketio\client.py", line 339, in _send_packet
+        self.eio.send(ep, binary=binary)
+      TypeError: send() got an unexpected keyword argument 'binary'
+
+
+- The ``python-engineio`` version needs to be compatible with the ``python-socketio`` version, reference: https://github.com/miguelgrinberg/python-socketio#version-compatibility
+
+    .. code-block:: bash
+
+        pip install -U python-engineio==<compatible python-socketio version>
+        # or
+        pip install -U python-socketio==3.1.2 python-engineio==3.13.2

docs/component/backtest.rst

@@ -30,7 +30,7 @@ The simple example of the default strategy is as follows.
 
     from qlib.contrib.evaluate import backtest
     # pred_score is the prediction score
-    report, positions = backtest(pred_score, topk=50, n_drop=0.5, verbose=False, limit_threshold=0.0095)
+    report, positions = backtest(pred_score, topk=50, n_drop=0.5, limit_threshold=0.0095)
 
 To know more about backtesting with a specific ``Strategy``, please refer to `Portfolio Strategy <strategy.html>`_.
 

docs/component/data.rst

@@ -179,6 +179,7 @@ After conversion, users can find their Qlib format data in the directory `~/.qli
         The Restoration factor. Normally, ``factor = adjusted_price / original_price``, `adjusted price` reference: `split adjusted <https://www.investopedia.com/terms/s/splitadjusted.asp>`_
 
     In the convention of `Qlib` data processing, `open, close, high, low, volume, money and factor` will be set to NaN if the stock is suspended. 
+    If you want to use your own alpha-factor which can't be calculate by OCHLV, like PE, EPS and so on, you could add it to the CSV files with OHCLV together and then dump it to the Qlib format data.
 
 Stock Pool (Market)
 --------------------------------

docs/component/highfreq.rst

@@ -0,0 +1,120 @@
+.. _highfreq:
+
+============================================
+Design of hierarchical order execution framework
+============================================
+.. currentmodule:: qlib
+
+Introduction
+===================
+
+In order to support reinforcement learning algorithms for high-frequency trading, a corresponding framework is required. None of the publicly available high-frequency trading frameworks now consider multi-layer trading mechanisms, and the currently designed algorithms cannot directly use existing frameworks.
+In addition to supporting the basic intraday multi-layer trading, the linkage with the day-ahead strategy is also a factor that affects the performance evaluation of the strategy. Different day strategies generate different order distributions and different patterns on different stocks. To verify that high-frequency trading strategies perform well on real trading orders, it is necessary to support day-frequency and high-frequency multi-level linkage trading. In addition to more accurate backtesting of high-frequency trading algorithms, if the distribution of day-frequency orders is considered when training a high-frequency trading model, the algorithm can also be optimized more for product-specific day-frequency orders.
+Therefore, innovation in the high-frequency trading framework is necessary to solve the various problems mentioned above, for which we designed a hierarchical order execution framework that can link daily-frequency and intra-day trading at different granularities.
+
+.. image:: ../_static/img/framework.svg
+
+The design of the framework is shown in the figure above. At each layer consists of Trading Agent and Execution Env. The Trading Agent has its own data processing module (Information Extractor), forecasting module (Forecast Model) and decision generator (Decision Generator). The trading algorithm generates the corresponding decisions by the Decision Generator based on the forecast signals output by the Forecast Module, and the decisions generated by the trading algorithm are passed to the Execution Env, which returns the execution results. Here the frequency of trading algorithm, decision content and execution environment can be customized by users (e.g. intra-day trading, daily-frequency trading, weekly-frequency trading), and the execution environment can be nested with finer-grained trading algorithm and execution environment inside (i.e. sub-workflow in the figure, e.g. daily-frequency orders can be turned into finer-grained decisions by splitting orders within the day). The hierarchical order execution framework is user-defined in terms of hierarchy division and decision frequency, making it easy for users to explore the effects of combining different levels of trading algorithms and breaking down the barriers between different levels of trading algorithm optimization.
+In addition to the innovation in the framework, the hierarchical order execution framework also takes into account various details of the real backtesting environment, minimizing the differences with the final real environment as much as possible. At the same time, the framework is designed to unify the interface between online and offline (e.g. data pre-processing level supports using the same set of code to process both offline and online data) to reduce the cost of strategy go-live as much as possible.
+
+Prepare Data
+===================
+.. _data:: ../../examples/highfreq/README.md
+
+
+Example
+===========================
+
+Here is an example of highfreq execution.
+
+.. code-block:: python
+
+    import qlib
+    # init qlib
+    provider_uri_day = "~/.qlib/qlib_data/cn_data"
+    provider_uri_1min = "~/.qlib/qlib_data/cn_data_1min"
+    provider_uri_map = {"1min": provider_uri_1min, "day": provider_uri_day}
+    qlib.init(provider_uri=provider_uri_day, expression_cache=None, dataset_cache=None)
+
+    # data freq and backtest time
+    freq = "1min"
+    inst_list = D.list_instruments(D.instruments("all"), as_list=True)
+    start_time = "2020-01-01"
+    start_time = "2020-01-31"
+
+When initializing qlib, if the default data is used, then both daily and minute frequency data need to be passed in.
+
+.. code-block:: python
+
+    # random order strategy config
+    strategy_config = {
+        "class": "RandomOrderStrategy",
+        "module_path": "qlib.contrib.strategy.rule_strategy",
+        "kwargs": {
+            "trade_range": TradeRangeByTime("9:30", "15:00"),
+            "sample_ratio": 1.0,
+            "volume_ratio": 0.01,
+            "market": market,
+        },
+    }
+
+.. code-block:: python
+    # backtest config
+    backtest_config = {
+        "start_time": start_time,
+        "end_time": end_time,
+        "account": 100000000,
+        "benchmark": None,
+        "exchange_kwargs": {
+            "freq": freq,
+            "limit_threshold": 0.095,
+            "deal_price": "close",
+            "open_cost": 0.0005,
+            "close_cost": 0.0015,
+            "min_cost": 5,
+            "codes": market,
+        },
+        "pos_type": "InfPosition",  # Position with infinitive position
+    }
+
+please refer to "../../qlib/backtest".
+
+.. code-block:: python
+    # excutor config
+    executor_config = {
+        "class": "NestedExecutor",
+        "module_path": "qlib.backtest.executor",
+        "kwargs": {
+            "time_per_step": "day",
+            "inner_executor": {
+                "class": "SimulatorExecutor",
+                "module_path": "qlib.backtest.executor",
+                "kwargs": {
+                    "time_per_step": freq,
+                    "generate_portfolio_metrics": True,
+                    "verbose": False,
+                    # "verbose": True,
+                    "indicator_config": {
+                        "show_indicator": False,
+                    },
+                },
+            },
+            "inner_strategy": {
+                "class": "TWAPStrategy",
+                "module_path": "qlib.contrib.strategy.rule_strategy",
+            },
+            "track_data": True,
+            "generate_portfolio_metrics": True,
+            "indicator_config": {
+                "show_indicator": True,
+            },
+        },
+    }
+
+NestedExecutor represents not the innermost layer, the initialization parameters should contain inner_executor and inner_strategy. simulatorExecutor represents the current excutor is the innermost layer, the innermost strategy used here is the TWAP strategy, the framework currently also supports the VWAP strategy
+
+.. code-block:: python
+    # backtest
+    portfolio_metrics_dict, indicator_dict = backtest(executor=executor_config, strategy=strategy_config, **backtest_config)
+
+The metrics of backtest are included in the portfolio_metrics_dict and indicator_dict.

docs/component/online.rst

@@ -21,6 +21,8 @@ which including `Online Manager <#Online Manager>`_, `Online Strategy <#Online S
 If you have many models or `task` needs to be managed, please consider `Task Management <../advanced/task_management.html>`_.
 The `examples <https://github.com/microsoft/qlib/tree/main/examples/online_srv>`_ are based on some components in `Task Management <../advanced/task_management.html>`_ such as ``TrainerRM`` or ``Collector``.
 
+**NOTE**: User should keep his data source updated to support online serving. For example, Qlib provides `a batch of scripts <https://github.com/microsoft/qlib/blob/main/scripts/data_collector/yahoo/README.md#automatic-update-of-daily-frequency-datafrom-yahoo-finance>`_ to help users update Yahoo daily data.
+
 Online Manager
 =============
 
@@ -43,4 +45,4 @@ Updater
 =============
 
 .. automodule:: qlib.workflow.online.update
-    :members:
+    :members:

docs/component/recorder.rst

@@ -123,7 +123,6 @@ Here is a simple exampke of what is done in ``PortAnaRecord``, which users can r
         "n_drop": 5,
     }
     BACKTEST_CONFIG = {
-        "verbose": False,
         "limit_threshold": 0.095,
         "account": 100000000,
         "benchmark": BENCHMARK,

docs/component/strategy.rst

@@ -93,7 +93,6 @@ Usage & Example
         "n_drop": 5,
     }
     BACKTEST_CONFIG = {
-        "verbose": False,
         "limit_threshold": 0.095,
         "account": 100000000,
         "benchmark": BENCHMARK,

docs/component/workflow.rst

@@ -53,8 +53,10 @@ Below is a typical config file of ``qrun``.
             kwargs:
                 topk: 50
                 n_drop: 5
+                signal:
+                    - <MODEL> 
+                    - <DATASET>
         backtest:
-            verbose: False
             limit_threshold: 0.095
             account: 100000000
             benchmark: *benchmark
@@ -241,8 +243,10 @@ The following script is the configuration of `backtest` and the `strategy` used
             kwargs:
                 topk: 50
                 n_drop: 5
+                signal:
+                    - <MODEL> 
+                    - <DATASET>
         backtest:
-            verbose: False
             limit_threshold: 0.095
             account: 100000000
             benchmark: *benchmark

docs/developer/code_standard.rst

@@ -0,0 +1,22 @@
+.. _code_standard:
+
+=================================
+Code Standard
+=================================
+
+Docstring
+=================================
+Please use the `Numpydoc Style <https://stackoverflow.com/a/24385103>`_.
+
+Continuous Integration
+=================================
+Continuous Integration (CI) tools help you stick to the quality standards by running tests every time you push a new commit and reporting the results to a pull request. 
+
+When you submit a PR request, you can check whether your code passes the CI tests in the "check" section at the bottom of the web page.
+
+A common error is the mixed use of space and tab. You can fix the bug by inputing the following code in the command line.
+
+.. code-block:: python
+
+    pip install black
+    python -m black . -l 120

docs/hidden/tuner.rst

@@ -93,7 +93,6 @@ We write a simple configuration example as following,
                   fend_time: 2018-12-11
     backtest:
         normal_backtest_args:
-            verbose: False
             limit_threshold: 0.095
             account: 500000
             benchmark: SH000905
@@ -306,7 +305,6 @@ About the data and backtest
                   fend_time: 2018-12-11
     backtest:
         normal_backtest_args:
-            verbose: False
             limit_threshold: 0.095
             account: 500000
             benchmark: SH000905

docs/introduction/introduction.rst

@@ -15,7 +15,7 @@ With ``Qlib``, users can easily try their ideas to create better Quant investmen
 Framework
 ===================
    
-.. image:: ../_static/img/framework.png
+.. image:: ../_static/img/framework.svg
     :align: center
 
 

docs/reference/api.rst

@@ -241,6 +241,7 @@ Online Tool
 .. automodule:: qlib.workflow.online.utils
     :members:
 
+
 RecordUpdater
 --------------------
 .. automodule:: qlib.workflow.online.update
@@ -257,4 +258,4 @@ Serializable
     :members:
 
 
-    
+    

docs/start/initialization.rst

@@ -77,7 +77,8 @@ Besides `provider_uri` and `region`, `qlib.init` has other parameters. The follo
         })
 - `mongo`
     Type: dict, optional parameter, the setting of `MongoDB <https://www.mongodb.com/>`_ which will be used in some features such as `Task Management <../advanced/task_management.html>`_, with high performance and clustered processing. 
-    Users need finished `installation <https://www.mongodb.com/try/download/community>`_ firstly, and run it in a fixed URL.
+    Users need to follow the steps in  `installation <https://www.mongodb.com/try/download/community>`_  to install MongoDB firstly and then access it via a URI.
+    Users can access mongodb with credential by setting "task_url"  to a string like `"mongodb://%s:%s@%s" % (user, pwd, host + ":" + port)`.
 
     .. code-block:: Python
 

examples/benchmarks/ALSTM/workflow_config_alstm_Alpha158.yaml

@@ -34,19 +34,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: ALSTM
@@ -81,7 +86,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/ALSTM/workflow_config_alstm_Alpha360.yaml

@@ -26,19 +26,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: ALSTM
@@ -71,7 +76,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -80,4 +87,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config

examples/benchmarks/CatBoost/workflow_config_catboost_Alpha158.yaml

@@ -12,19 +12,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: CatBoostModel
@@ -53,7 +58,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/CatBoost/workflow_config_catboost_Alpha360.yaml

@@ -19,19 +19,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: CatBoostModel
@@ -60,7 +65,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha158.yaml

@@ -12,19 +12,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: DEnsembleModel
@@ -75,16 +80,18 @@ task:
                 train: [2008-01-01, 2014-12-31]
                 valid: [2015-01-01, 2016-12-31]
                 test: [2017-01-01, 2020-08-01]
-    record:
+    record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
-          kwargs:
+          kwargs: 
             ana_long_short: False
             ann_scaler: 252
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
-          kwargs:
+          kwargs: 
             config: *port_analysis_config

examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha360.yaml

@@ -19,19 +19,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: DEnsembleModel
@@ -82,10 +87,12 @@ task:
                 train: [2008-01-01, 2014-12-31]
                 valid: [2015-01-01, 2016-12-31]
                 test: [2017-01-01, 2020-08-01]
-    record:
+    record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs:
@@ -93,5 +100,5 @@ task:
             ann_scaler: 252
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
-          kwargs:
-            config: *port_analysis_config
+          kwargs: 
+            config: *port_analysis_config

examples/benchmarks/GATs/workflow_config_gats_Alpha158.yaml

@@ -33,19 +33,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: GATs
@@ -79,7 +84,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -88,4 +95,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config

examples/benchmarks/GATs/workflow_config_gats_Alpha360.yaml

@@ -26,19 +26,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: GATs
@@ -71,7 +76,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/GRU/workflow_config_gru_Alpha158.yaml

@@ -34,19 +34,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: GRU
@@ -80,7 +85,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/GRU/workflow_config_gru_Alpha360.yaml

@@ -26,19 +26,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: GRU
@@ -70,7 +75,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -79,4 +86,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config

examples/benchmarks/LSTM/workflow_config_lstm_Alpha158.yaml

@@ -34,19 +34,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: LSTM
@@ -80,7 +85,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/LSTM/workflow_config_lstm_Alpha360.yaml

@@ -26,19 +26,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: LSTM
@@ -70,7 +75,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -79,4 +86,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config

examples/benchmarks/LightGBM/features_resample_N.py

@@ -0,0 +1,18 @@
+#  Copyright (c) Microsoft Corporation.
+#  Licensed under the MIT License.
+
+import pandas as pd
+
+from qlib.data.inst_processor import InstProcessor
+from qlib.utils.resam import resam_calendar
+
+
+class ResampleNProcessor(InstProcessor):
+    def __init__(self, target_frq: str, **kwargs):
+        self.target_frq = target_frq
+
+    def __call__(self, df: pd.DataFrame, *args, **kwargs):
+        df.index = pd.to_datetime(df.index)
+        res_index = resam_calendar(df.index, "1min", self.target_frq)
+        df = df.resample(self.target_frq).last().reindex(res_index)
+        return df

examples/benchmarks/LightGBM/features_sample.py

@@ -0,0 +1,16 @@
+import datetime
+import pandas as pd
+
+from qlib.data.inst_processor import InstProcessor
+
+
+class Resample1minProcessor(InstProcessor):
+    def __init__(self, hour: int, minute: int, **kwargs):
+        self.hour = hour
+        self.minute = minute
+
+    def __call__(self, df: pd.DataFrame, *args, **kwargs):
+        df.index = pd.to_datetime(df.index)
+        df = df.loc[df.index.time == datetime.time(self.hour, self.minute)]
+        df.index = df.index.normalize()
+        return df

examples/benchmarks/LightGBM/multi_freq_handler.py

@@ -0,0 +1,135 @@
+#  Copyright (c) Microsoft Corporation.
+#  Licensed under the MIT License.
+
+import pandas as pd
+
+from qlib.data.dataset.loader import QlibDataLoader
+from qlib.contrib.data.handler import DataHandlerLP, _DEFAULT_LEARN_PROCESSORS, check_transform_proc
+
+
+class Avg15minLoader(QlibDataLoader):
+    def load(self, instruments=None, start_time=None, end_time=None) -> pd.DataFrame:
+        df = super(Avg15minLoader, self).load(instruments, start_time, end_time)
+        if self.is_group:
+            # feature_day(day freq) and feature_15min(1min freq, Average every 15 minutes) renamed feature
+            df.columns = df.columns.map(lambda x: ("feature", x[1]) if x[0].startswith("feature") else x)
+        return df
+
+
+class Avg15minHandler(DataHandlerLP):
+    def __init__(
+        self,
+        instruments="csi500",
+        start_time=None,
+        end_time=None,
+        freq="day",
+        infer_processors=[],
+        learn_processors=_DEFAULT_LEARN_PROCESSORS,
+        fit_start_time=None,
+        fit_end_time=None,
+        process_type=DataHandlerLP.PTYPE_A,
+        filter_pipe=None,
+        inst_processor=None,
+        **kwargs,
+    ):
+        infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time)
+        learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time)
+        data_loader = Avg15minLoader(
+            config=self.loader_config(), filter_pipe=filter_pipe, freq=freq, inst_processor=inst_processor
+        )
+        super().__init__(
+            instruments=instruments,
+            start_time=start_time,
+            end_time=end_time,
+            data_loader=data_loader,
+            infer_processors=infer_processors,
+            learn_processors=learn_processors,
+            process_type=process_type,
+        )
+
+    def loader_config(self):
+
+        # Results for dataset: df: pd.DataFrame
+        #   len(df.columns) == 6 + 6 * 16, len(df.index.get_level_values(level="datetime").unique()) == T
+        #   df.columns: close0, close1, ..., close16, open0, ..., open16, ..., vwap16
+        #       freq == day:
+        #           close0, open0, low0, high0, volume0, vwap0
+        #       freq == 1min:
+        #           close1, ..., close16, ..., vwap1, ..., vwap16
+        #   df.index.name == ["datetime", "instrument"]: pd.MultiIndex
+        # Example:
+        #                          feature                        ...                  label
+        #                           close0      open0       low0  ... vwap1 vwap16    LABEL0
+        # datetime   instrument                                   ...
+        # 2020-10-09 SH600000    11.794546  11.819587  11.769505  ...   NaN    NaN -0.005214
+        # 2020-10-15 SH600000    12.044961  11.944795  11.932274  ...   NaN    NaN -0.007202
+        # ...                          ...        ...        ...  ...   ...    ...       ...
+        # 2021-05-28 SZ300676     6.369684   6.495406   6.306568  ...   NaN    NaN -0.001321
+        # 2021-05-31 SZ300676     6.601626   6.465643   6.465130  ...   NaN    NaN -0.023428
+
+        # features day: len(columns) == 6, freq = day
+        # $close is the closing price of the current trading day:
+        #   if the user needs to get the `close` before the last T days, use Ref($close, T-1), for example:
+        #                                    $close  Ref($close, 1)  Ref($close, 2)  Ref($close, 3)  Ref($close, 4)
+        #         instrument datetime
+        #         SH600519   2021-06-01  244.271530
+        #                    2021-06-02  242.205917      244.271530
+        #                    2021-06-03  242.229889      242.205917      244.271530
+        #                    2021-06-04  245.421524      242.229889      242.205917      244.271530
+        #                    2021-06-07  247.547089      245.421524      242.229889      242.205917      244.271530
+
+        # WARNING: Ref($close, N), if N == 0, Ref($close, N) ==> $close
+
+        fields = ["$close", "$open", "$low", "$high", "$volume", "$vwap"]
+        # names: close0, open0, ..., vwap0
+        names = list(map(lambda x: x.strip("$") + "0", fields))
+
+        config = {"feature_day": (fields, names)}
+
+        # features 15min: len(columns) == 6 * 16, freq = 1min
+        #   $close is the closing price of the current trading day:
+        #       if the user gets 'close' for the i-th 15min of the last T days, use `Ref(Mean($close, 15), (T-1) * 240 + i * 15)`, for example:
+        #                                    Ref(Mean($close, 15), 225)  Ref(Mean($close, 15), 465)  Ref(Mean($close, 15), 705)
+        #             instrument datetime
+        #             SH600519   2021-05-31                  241.769897                  243.077942                  244.712997
+        #                        2021-06-01                  244.271530                  241.769897                  243.077942
+        #                        2021-06-02                  242.205917                  244.271530                  241.769897
+
+        # WARNING: Ref(Mean($close, 15), N), if N == 0, Ref(Mean($close, 15), N) ==> Mean($close, 15)
+
+        # Results of the current script:
+        #   time:   09:00 --> 09:14,            ..., 14:45 --> 14:59
+        #   fields: Ref(Mean($close, 15), 225), ..., Mean($close, 15)
+        #   name:   close1,                     ..., close16
+        #
+
+        # Expression description: take close as an example
+        #   Mean($close, 15) ==> df["$close"].rolling(15, min_periods=1).mean()
+        #   Ref(Mean($close, 15), 15) ==> df["$close"].rolling(15, min_periods=1).mean().shift(15)
+
+        #   NOTE: The last data of each trading day, which is the average of the i-th 15 minutes
+
+        # Average:
+        #   Average of the i-th 15-minute period of each trading day: 1 <= i <= 250 // 16
+        #       Avg(15minutes): Ref(Mean($close, 15), 240 - i * 15)
+        #
+        #   Average of the first 15 minutes of each trading day; i = 1
+        #       Avg(09:00 --> 09:14), df.index.loc["09:14"]: Ref(Mean($close, 15), 240- 1 * 15) ==> Ref(Mean($close, 15), 225)
+        #   Average of the last 15 minutes of each trading day; i = 16
+        #       Avg(14:45 --> 14:59), df.index.loc["14:59"]: Ref(Mean($close, 15), 240 - 16 * 15) ==> Ref(Mean($close, 15), 0) ==> Mean($close, 15)
+
+        # 15min resample to day
+        #   df.resample("1d").last()
+        tmp_fields = []
+        tmp_names = []
+        for i, _f in enumerate(fields):
+            _fields = [f"Ref(Mean({_f}, 15), {j * 15})" for j in range(1, 240 // 15)]
+            _names = [f"{names[i][:-1]}{int(names[i][-1])+j}" for j in range(240 // 15 - 1, 0, -1)]
+            _fields.append(f"Mean({_f}, 15)")
+            _names.append(f"{names[i][:-1]}{int(names[i][-1])+240 // 15}")
+            tmp_fields += _fields
+            tmp_names += _names
+        config["feature_15min"] = (tmp_fields, tmp_names)
+        # label
+        config["label"] = (["Ref($close, -2)/Ref($close, -1) - 1"], ["LABEL0"])
+        return config

examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml

@@ -12,19 +12,23 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            model: <MODEL> 
+            dataset: <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: LGBModel
@@ -54,7 +58,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -63,4 +69,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config

examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_multi_freq.yaml

@@ -0,0 +1,86 @@
+qlib_init:
+    provider_uri:
+        day: "~/.qlib/qlib_data/cn_data"
+        1min: "~/.qlib/qlib_data/cn_data_1min"
+    region: cn
+    dataset_cache: null
+    maxtasksperchild: 1
+market: &market csi300
+benchmark: &benchmark SH000300
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    # 1min closing time is 15:00:00
+    end_time: "2020-08-01 15:00:00"
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    freq:
+        label: day
+        feature: 1min
+    # with label as reference
+    inst_processor:
+        feature:
+            - class: Resample1minProcessor
+              module_path: features_sample.py
+              kwargs:
+                  hour: 14
+                  minute: 56
+
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy.strategy
+        kwargs:
+            topk: 50
+            n_drop: 5
+            signal:
+                - <MODEL> 
+                - <DATASET>
+    backtest:
+        verbose: False
+        limit_threshold: 0.095
+        account: 100000000
+        benchmark: *benchmark
+        deal_price: close
+        open_cost: 0.0005
+        close_cost: 0.0015
+        min_cost: 5
+task:
+    model:
+        class: LGBModel
+        module_path: qlib.contrib.model.gbdt
+        kwargs:
+            loss: mse
+            colsample_bytree: 0.8879
+            learning_rate: 0.2
+            subsample: 0.8789
+            lambda_l1: 205.6999
+            lambda_l2: 580.9768
+            max_depth: 8
+            num_leaves: 210
+            num_threads: 20
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha158
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: {}
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha360.yaml

@@ -19,19 +19,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: LGBModel
@@ -61,7 +66,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -70,4 +77,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config

examples/benchmarks/LightGBM/workflow_config_lightgbm_configurable_dataset.yaml

@@ -27,19 +27,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: LGBModel
@@ -69,7 +74,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -78,4 +85,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config

examples/benchmarks/LightGBM/workflow_config_lightgbm_multi_freq.yaml

@@ -0,0 +1,90 @@
+qlib_init:
+    provider_uri:
+        day: "~/.qlib/qlib_data/cn_data"
+        1min: "~/.qlib/qlib_data/cn_data_1min"
+    region: cn
+    dataset_cache: null
+    maxtasksperchild: null
+market: &market csi300
+benchmark: &benchmark SH000300
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    # 1min closing time is 15:00:00
+    end_time: "2020-08-01 15:00:00"
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    freq:
+        label: day
+        feature_15min: 1min
+        feature_day: day
+    # with label as reference
+    inst_processor:
+        feature_15min:
+            - class: ResampleNProcessor
+              module_path: features_resample_N.py
+              kwargs:
+                  target_frq: 1d
+
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: LGBModel
+        module_path: qlib.contrib.model.gbdt
+        kwargs:
+            loss: mse
+            colsample_bytree: 0.8879
+            learning_rate: 0.2
+            subsample: 0.8789
+            lambda_l1: 205.6999
+            lambda_l2: 580.9768
+            max_depth: 8
+            num_leaves: 210
+            num_threads: 20
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Avg15minHandler
+                module_path: multi_freq_handler.py
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record:
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs:
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs:
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs:
+            config: *port_analysis_config

examples/benchmarks/Linear/workflow_config_linear_Alpha158.yaml

@@ -26,19 +26,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: LinearModel
@@ -57,16 +62,18 @@ task:
                 train: [2008-01-01, 2014-12-31]
                 valid: [2015-01-01, 2016-12-31]
                 test: [2017-01-01, 2020-08-01]
-    record:
+    record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
-          kwargs:
+          kwargs: 
             ana_long_short: True
             ann_scaler: 252
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
-          kwargs:
+          kwargs: 
             config: *port_analysis_config

examples/benchmarks/Localformer/requirements.txt

@@ -0,0 +1,3 @@
+numpy==1.17.4
+pandas==1.1.2
+torch==1.2.0

examples/benchmarks/Localformer/workflow_config_localformer_Alpha158.yaml

@@ -0,0 +1,89 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market csi300
+benchmark: &benchmark SH000300
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    infer_processors:
+        - class: FilterCol
+          kwargs:
+              fields_group: feature
+              col_list: ["RESI5", "WVMA5", "RSQR5", "KLEN", "RSQR10", "CORR5", "CORD5", "CORR10", 
+                            "ROC60", "RESI10", "VSTD5", "RSQR60", "CORR60", "WVMA60", "STD5", 
+                            "RSQR20", "CORD60", "CORD10", "CORR20", "KLOW"
+                        ]
+        - class: RobustZScoreNorm
+          kwargs:
+              fields_group: feature
+              clip_outlier: true
+        - class: Fillna
+          kwargs:
+              fields_group: feature
+    learn_processors:
+        - class: DropnaLabel
+        - class: CSRankNorm
+          kwargs:
+              fields_group: label
+    label: ["Ref($close, -2) / Ref($close, -1) - 1"] 
+
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: LocalformerModel
+        module_path: qlib.contrib.model.pytorch_localformer_ts
+        kwargs:
+            seed: 0
+            n_jobs: 20
+    dataset:
+        class: TSDatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha158
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+            step_len: 20
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+              model: <MODEL>
+              dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+              ana_long_short: False
+              ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+              config: *port_analysis_config

examples/benchmarks/Localformer/workflow_config_localformer_Alpha360.yaml

@@ -0,0 +1,80 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market csi300
+benchmark: &benchmark SH000300
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    infer_processors:
+        - class: RobustZScoreNorm
+          kwargs:
+              fields_group: feature
+              clip_outlier: true
+        - class: Fillna
+          kwargs:
+              fields_group: feature
+    learn_processors:
+        - class: DropnaLabel
+        - class: CSRankNorm
+          kwargs:
+              fields_group: label
+    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: LocalformerModel
+        module_path: qlib.contrib.model.pytorch_localformer
+        kwargs:
+            d_feat: 6
+            seed: 0
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha360
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+    - class: SignalRecord
+      module_path: qlib.workflow.record_temp
+      kwargs: 
+        model: <MODEL>
+        dataset: <DATASET>
+    - class: SigAnaRecord
+      module_path: qlib.workflow.record_temp
+      kwargs: 
+        ana_long_short: False
+        ann_scaler: 252
+    - class: PortAnaRecord
+      module_path: qlib.workflow.record_temp
+      kwargs: 
+        config: *port_analysis_config

examples/benchmarks/MLP/workflow_config_mlp_Alpha158.yaml

@@ -39,19 +39,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: DNNModelPytorch
@@ -83,7 +88,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -92,4 +99,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config

examples/benchmarks/MLP/workflow_config_mlp_Alpha360.yaml

@@ -27,19 +27,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: DNNModelPytorch
@@ -70,7 +75,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -79,4 +86,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config

examples/benchmarks/README.md

@@ -1,44 +1,60 @@
 # Benchmarks Performance
 
-Here are the results of each benchmark model running on Qlib's `Alpha360` and `Alpha158` dataset with China's A shared-stock & CSI300 data respectively. The values of each metric are the mean and std calculated based on 20 runs.
+Here are the results of each benchmark model running on Qlib's `Alpha360` and `Alpha158` dataset with China's A shared-stock & CSI300 data respectively. The values of each metric are the mean and std calculated based on 20 runs with different random seeds.
 
 The numbers shown below demonstrate the performance of the entire `workflow` of each model. We will update the `workflow` as well as models in the near future for better results.
-
+<!-- 
 > If you need to reproduce the results below, please use the **v1** dataset: `python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/qlib_cn_1d --region cn --version v1`
 >
-> In the new version of qlib, the default dataset is **v2**. Since the data is collected from the YahooFinance API (which is not very stable), the results of *v2* and *v1* may differ
+> In the new version of qlib, the default dataset is **v2**. Since the data is collected from the YahooFinance API (which is not very stable), the results of *v2* and *v1* may differ -->
+
+> NOTE:
+> The backtest start from 0.8.0 is quite different from previous version. Please check out the changelog for the difference.
 
-## Alpha360 dataset
-| Model Name | Dataset | IC | ICIR | Rank IC | Rank ICIR | Annualized Return | Information Ratio | Max Drawdown |
-|---|---|---|---|---|---|---|---|---|
-| Linear | Alpha360 | 0.0150±0.00 | 0.1049±0.00| 0.0284±0.00 | 0.1970±0.00 | -0.0659±0.00 | -0.7072±0.00| -0.2955±0.00 |
-| CatBoost (Liudmila Prokhorenkova, et al.) | Alpha360 | 0.0397±0.00 | 0.2878±0.00| 0.0470±0.00 | 0.3703±0.00 | 0.0342±0.00 | 0.4092±0.00| -0.1057±0.00 |
-| XGBoost (Tianqi Chen, et al.) | Alpha360 | 0.0400±0.00 | 0.3031±0.00| 0.0461±0.00 | 0.3862±0.00 | 0.0528±0.00 | 0.6307±0.00| -0.1113±0.00 |
-| LightGBM (Guolin Ke, et al.) | Alpha360 | 0.0399±0.00 | 0.3075±0.00| 0.0492±0.00 | 0.4019±0.00 | 0.0323±0.00 | 0.4370±0.00| -0.0917±0.00 |
-| MLP | Alpha360 | 0.0285±0.00 | 0.1981±0.02| 0.0402±0.00 | 0.2993±0.02 | 0.0073±0.02 | 0.0880±0.22| -0.1446±0.03 |
-| GRU (Kyunghyun Cho, et al.) | Alpha360 | 0.0490±0.01 | 0.3787±0.05| 0.0581±0.00 | 0.4664±0.04 | 0.0726±0.02 | 0.9817±0.34| -0.0902±0.03 |
-| LSTM (Sepp Hochreiter, et al.) | Alpha360 | 0.0443±0.01 | 0.3401±0.05| 0.0536±0.01 | 0.4248±0.05 | 0.0627±0.03 | 0.8441±0.48| -0.0882±0.03 |
-| ALSTM (Yao Qin, et al.) | Alpha360 | 0.0493±0.01 | 0.3778±0.06| 0.0585±0.00 | 0.4606±0.04 | 0.0513±0.03 | 0.6727±0.38| -0.1085±0.02 |
-| GATs (Petar Velickovic, et al.) | Alpha360 | 0.0475±0.00 | 0.3515±0.02| 0.0592±0.00 | 0.4585±0.01 | 0.0876±0.02 | 1.1513±0.27| -0.0795±0.02 |
-| DoubleEnsemble (Chuheng Zhang, et al.) | Alpha360 | 0.0407±0.00| 0.3053±0.00 | 0.0490±0.00 | 0.3840±0.00 | 0.0380±0.02 | 0.5000±0.21 | -0.0984±0.02 |
-| TabNet (Sercan O. Arik, et al.)| Alpha360 | 0.0192±0.00 | 0.1401±0.00| 0.0291±0.00 | 0.2163±0.00 | -0.0258±0.00 | -0.2961±0.00| -0.1429±0.00 |
-| TCTS (Xueqing Wu, et al.)| Alpha360 | 0.0485±0.00 | 0.3689±0.04| 0.0586±0.00 | 0.4669±0.02 | 0.0816±0.02 | 1.1572±0.30| -0.0689±0.02 |
 
 ## Alpha158 dataset
-| Model Name | Dataset | IC | ICIR | Rank IC | Rank ICIR | Annualized Return | Information Ratio | Max Drawdown |
-|---|---|---|---|---|---|---|---|---|
-| Linear | Alpha158 | 0.0393±0.00 | 0.2980±0.00| 0.0475±0.00 | 0.3546±0.00 | 0.0795±0.00 | 1.0712±0.00| -0.1449±0.00 |
-| CatBoost (Liudmila Prokhorenkova, et al.) | Alpha158 | 0.0503±0.00 | 0.3586±0.00| 0.0483±0.00 | 0.3667±0.00 | 0.1080±0.00 | 1.1561±0.00| -0.0787±0.00 |
-| XGBoost (Tianqi Chen, et al.) | Alpha158 | 0.0481±0.00 | 0.3659±0.00| 0.0495±0.00 | 0.4033±0.00 | 0.1111±0.00 | 1.2915±0.00| -0.0893±0.00 |
-| LightGBM (Guolin Ke, et al.) | Alpha158 | 0.0475±0.00 | 0.3979±0.00| 0.0485±0.00 | 0.4123±0.00 | 0.1143±0.00 | 1.2744±0.00| -0.0800±0.00 |
-| MLP | Alpha158 | 0.0358±0.00 | 0.2738±0.03| 0.0425±0.00 | 0.3221±0.01 | 0.0836±0.02 | 1.0323±0.25| -0.1127±0.02 |
-| TFT (Bryan Lim, et al.) | Alpha158 (with selected 20 features) | 0.0343±0.00 | 0.2071±0.02| 0.0107±0.00 | 0.0660±0.02 | 0.0623±0.02 | 0.5818±0.20| -0.1762±0.01 |
-| GRU (Kyunghyun Cho, et al.) | Alpha158 (with selected 20 features) | 0.0311±0.00 | 0.2418±0.04| 0.0425±0.00 | 0.3434±0.02 | 0.0330±0.02 | 0.4805±0.30| -0.1021±0.02 |
-| LSTM (Sepp Hochreiter, et al.) | Alpha158 (with selected 20 features) | 0.0312±0.00 | 0.2394±0.04| 0.0418±0.00 | 0.3324±0.03 | 0.0298±0.02 | 0.4198±0.33| -0.1348±0.03 |
-| ALSTM (Yao Qin, et al.) | Alpha158 (with selected 20 features) | 0.0385±0.01 | 0.3022±0.06| 0.0478±0.00 | 0.3874±0.04 | 0.0486±0.03 | 0.7141±0.45| -0.1088±0.03 |
-| GATs (Petar Velickovic, et al.) | Alpha158 (with selected 20 features) | 0.0349±0.00 | 0.2511±0.01| 0.0457±0.00 | 0.3537±0.01 | 0.0578±0.02 | 0.8221±0.25| -0.0824±0.02 |
-| DoubleEnsemble (Chuheng Zhang, et al.) | Alpha158 | 0.0544±0.00 | 0.4338±0.01 | 0.0523±0.00 | 0.4257±0.01 | 0.1253±0.01 | 1.4105±0.14 | -0.0902±0.01 |
-| TabNet (Sercan O. Arik, et al.)| Alpha158 | 0.0383±0.00 | 0.3414±0.00| 0.0388±0.00 | 0.3460±0.00 | 0.0226±0.00 | 0.2652±0.00| -0.1072±0.00 |
+
+| Model Name                               | Dataset                             | IC          | ICIR        | Rank IC     | Rank ICIR   | Annualized Return | Information Ratio | Max Drawdown |
+|------------------------------------------|-------------------------------------|-------------|-------------|-------------|-------------|-------------------|-------------------|--------------|
+| TabNet(Sercan O. Arik, et al.)           | Alpha158                            | 0.0204±0.01 | 0.1554±0.07 | 0.0333±0.00 | 0.2552±0.05 | 0.0227±0.04       | 0.3676±0.54       | -0.1089±0.08 |
+| Transformer(Ashish Vaswani, et al.)      | Alpha158                            | 0.0264±0.00 | 0.2053±0.02 | 0.0407±0.00 | 0.3273±0.02 | 0.0273±0.02       | 0.3970±0.26       | -0.1101±0.02 |
+| GRU(Kyunghyun Cho, et al.)               | Alpha158(with selected 20 features) | 0.0315±0.00 | 0.2450±0.04 | 0.0428±0.00 | 0.3440±0.03 | 0.0344±0.02       | 0.5160±0.25       | -0.1017±0.02 |
+| LSTM(Sepp Hochreiter, et al.)            | Alpha158(with selected 20 features) | 0.0318±0.00 | 0.2367±0.04 | 0.0435±0.00 | 0.3389±0.03 | 0.0381±0.03       | 0.5561±0.46       | -0.1207±0.04 |
+| Localformer(Juyong Jiang, et al.)        | Alpha158                            | 0.0356±0.00 | 0.2756±0.03 | 0.0468±0.00 | 0.3784±0.03 | 0.0438±0.02       | 0.6600±0.33       | -0.0952±0.02 |
+| SFM(Liheng Zhang, et al.)                | Alpha158                            | 0.0379±0.00 | 0.2959±0.04 | 0.0464±0.00 | 0.3825±0.04 | 0.0465±0.02       | 0.5672±0.29       | -0.1282±0.03 |
+| ALSTM (Yao Qin, et al.)                  | Alpha158(with selected 20 features) | 0.0362±0.01 | 0.2789±0.06 | 0.0463±0.01 | 0.3661±0.05 | 0.0470±0.03       | 0.6992±0.47       | -0.1072±0.03 |
+| GATs (Petar Velickovic, et al.)          | Alpha158(with selected 20 features) | 0.0349±0.00 | 0.2511±0.01 | 0.0462±0.00 | 0.3564±0.01 | 0.0497±0.01       | 0.7338±0.19       | -0.0777±0.02 |
+| TRA(Hengxu Lin, et al.)                  | Alpha158(with selected 20 features) | 0.0404±0.00 | 0.3197±0.05 | 0.0490±0.00 | 0.4047±0.04 | 0.0649±0.02       | 1.0091±0.30       | -0.0860±0.02 |
+| Linear                                   | Alpha158                            | 0.0397±0.00 | 0.3000±0.00 | 0.0472±0.00 | 0.3531±0.00 | 0.0692±0.00       | 0.9209±0.00       | -0.1509±0.00 |
+| TRA(Hengxu Lin, et al.)                  | Alpha158                            | 0.0440±0.00 | 0.3535±0.05 | 0.0540±0.00 | 0.4451±0.03 | 0.0718±0.02       | 1.0835±0.35       | -0.0760±0.02 |
+| CatBoost(Liudmila Prokhorenkova, et al.) | Alpha158                            | 0.0481±0.00 | 0.3366±0.00 | 0.0454±0.00 | 0.3311±0.00 | 0.0765±0.00       | 0.8032±0.01       | -0.1092±0.00 |
+| XGBoost(Tianqi Chen, et al.)             | Alpha158                            | 0.0498±0.00 | 0.3779±0.00 | 0.0505±0.00 | 0.4131±0.00 | 0.0780±0.00       | 0.9070±0.00       | -0.1168±0.00 |
+| TFT (Bryan Lim, et al.)                  | Alpha158(with selected 20 features) | 0.0358±0.00 | 0.2160±0.03 | 0.0116±0.01 | 0.0720±0.03 | 0.0847±0.02       | 0.8131±0.19       | -0.1824±0.03 |
+| MLP                                      | Alpha158                            | 0.0376±0.00 | 0.2846±0.02 | 0.0429±0.00 | 0.3220±0.01 | 0.0895±0.02       | 1.1408±0.23       | -0.1103±0.02 |
+| LightGBM(Guolin Ke, et al.)              | Alpha158                            | 0.0448±0.00 | 0.3660±0.00 | 0.0469±0.00 | 0.3877±0.00 | 0.0901±0.00       | 1.0164±0.00       | -0.1038±0.00 |
+| DoubleEnsemble(Chuheng Zhang, et al.)    | Alpha158                            | 0.0544±0.00 | 0.4340±0.00 | 0.0523±0.00 | 0.4284±0.01 | 0.1168±0.01       | 1.3384±0.12       | -0.1036±0.01 |
+
+
+
+## Alpha360 dataset
+
+| Model Name                                | Dataset  | IC          | ICIR        | Rank IC     | Rank ICIR   | Annualized Return | Information Ratio | Max Drawdown |
+|-------------------------------------------|----------|-------------|-------------|-------------|-------------|-------------------|-------------------|--------------|
+| Transformer(Ashish Vaswani, et al.)       | Alpha360 | 0.0114±0.00 | 0.0716±0.03 | 0.0327±0.00 | 0.2248±0.02 | -0.0270±0.03      | -0.3378±0.37      | -0.1653±0.05 |
+| TabNet(Sercan O. Arik, et al.)            | Alpha360 | 0.0099±0.00 | 0.0593±0.00 | 0.0290±0.00 | 0.1887±0.00 | -0.0369±0.00      | -0.3892±0.00      | -0.2145±0.00 |
+| MLP                                       | Alpha360 | 0.0273±0.00 | 0.1870±0.02 | 0.0396±0.00 | 0.2910±0.02 | 0.0029±0.02       | 0.0274±0.23       | -0.1385±0.03 |
+| Localformer(Juyong Jiang, et al.)         | Alpha360 | 0.0404±0.00 | 0.2932±0.04 | 0.0542±0.00 | 0.4110±0.03 | 0.0246±0.02       | 0.3211±0.21       | -0.1095±0.02 |
+| CatBoost((Liudmila Prokhorenkova, et al.) | Alpha360 | 0.0378±0.00 | 0.2714±0.00 | 0.0467±0.00 | 0.3659±0.00 | 0.0292±0.00       | 0.3781±0.00       | -0.0862±0.00 |
+| XGBoost(Tianqi Chen, et al.)              | Alpha360 | 0.0394±0.00 | 0.2909±0.00 | 0.0448±0.00 | 0.3679±0.00 | 0.0344±0.00       | 0.4527±0.02       | -0.1004±0.00 |
+| DoubleEnsemble(Chuheng Zhang, et al.)     | Alpha360 | 0.0404±0.00 | 0.3023±0.00 | 0.0495±0.00 | 0.3898±0.00 | 0.0468±0.01       | 0.6302±0.20       | -0.0860±0.01 |
+| LightGBM(Guolin Ke, et al.)               | Alpha360 | 0.0400±0.00 | 0.3037±0.00 | 0.0499±0.00 | 0.4042±0.00 | 0.0558±0.00       | 0.7632±0.00       | -0.0659±0.00 |
+| ALSTM (Yao Qin, et al.)                   | Alpha360 | 0.0497±0.00 | 0.3829±0.04 | 0.0599±0.00 | 0.4736±0.03 | 0.0626±0.02       | 0.8651±0.31       | -0.0994±0.03 |
+| LSTM(Sepp Hochreiter, et al.)             | Alpha360 | 0.0448±0.00 | 0.3474±0.04 | 0.0549±0.00 | 0.4366±0.03 | 0.0647±0.03       | 0.8963±0.39       | -0.0875±0.02 |
+| GRU(Kyunghyun Cho, et al.)                | Alpha360 | 0.0493±0.00 | 0.3772±0.04 | 0.0584±0.00 | 0.4638±0.03 | 0.0720±0.02       | 0.9730±0.33       | -0.0821±0.02 |
+| GATs (Petar Velickovic, et al.)           | Alpha360 | 0.0476±0.00 | 0.3508±0.02 | 0.0598±0.00 | 0.4604±0.01 | 0.0824±0.02       | 1.1079±0.26       | -0.0894±0.03 |
+| TCTS(Xueqing Wu, et al.)                  | Alpha360 | 0.0508±0.00 | 0.3931±0.04 | 0.0599±0.00 | 0.4756±0.03 | 0.0893±0.03       | 1.2256±0.36       | -0.0857±0.02 |
+| TRA(Hengxu Lin, et al.)                   | Alpha360 | 0.0485±0.00 | 0.3787±0.03 | 0.0587±0.00 | 0.4756±0.03 | 0.0920±0.03       | 1.2789±0.42       | -0.0834±0.02 |
 
 - The selected 20 features are based on the feature importance of a lightgbm-based model.
 - The base model of DoubleEnsemble is LGBM.
+- The base model of TCTS is GRU.

examples/benchmarks/SFM/requirements.txt

@@ -1,4 +1,4 @@
 pandas==1.1.2
 numpy==1.17.4
 scikit_learn==0.23.2
-torch==1.7.0
+torch==1.7.0

examples/benchmarks/SFM/workflow_config_sfm_Alpha360.yaml

@@ -26,19 +26,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: SFM
@@ -73,7 +78,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/TCTS/README.md

@@ -1,52 +1,38 @@
 # Temporally Correlated Task Scheduling for Sequence Learning
-We provide the [code](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py) for reproducing the stock trend forecasting experiments.
-
 ### Background
 Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. 
 
-<p align="center"> 
-<img src="task_description.png" width="600" height="200"/>
-</p>
-
-
 ### Method
-Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2.
+Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. This work introduces a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in the current minibatch) and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2.
 
 <p align="center"> 
 <img src="workflow.png"/>
 </p>
 
-At step <img src="https://render.githubusercontent.com/render/math?math=s">, with training data <img src="https://render.githubusercontent.com/render/math?math=x_s,y_s">, the scheduler <img src="https://render.githubusercontent.com/render/math?math=\varphi"> chooses a suitable task <img src="https://render.githubusercontent.com/render/math?math=T_{i_s}"> (green solid lines) to update the model <img src="https://render.githubusercontent.com/render/math?math=f"> (blue solid lines). After <img src="https://render.githubusercontent.com/render/math?math=S"> steps, we evaluate the model <img src="https://render.githubusercontent.com/render/math?math=f"> on the validation set and update the scheduler <img src="https://render.githubusercontent.com/render/math?math=\varphi"> (green dashed lines).
-
-### DataSet
-* We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. 
-* We split the data into training (01/01/2008-12/31/2013), validation (01/01/2014-12/31/2015), and test sets (01/01/2016-08/01/2020) based on the transaction time. 
+At step <img src="https://latex.codecogs.com/png.latex?s" title="s" />, with training data <img src="https://latex.codecogs.com/png.latex?x_s,y_s" title="x_s,y_s" />, the scheduler <img src="https://latex.codecogs.com/png.latex?\varphi" title="\varphi" /> chooses a suitable task <img src="https://latex.codecogs.com/png.latex?T_{i_s}" title="T_{i_s}" /> (green solid lines) to update the model <img src="https://latex.codecogs.com/png.latex?f" title="f" /> (blue solid lines). After <img src="https://latex.codecogs.com/png.latex?S" title="S" /> steps, we evaluate the model <img src="https://latex.codecogs.com/png.latex?f" title="f" /> on the validation set and update the scheduler <img src="https://latex.codecogs.com/png.latex?\varphi" title="\varphi" /> (green dashed lines).
 
 ### Experiments
-#### Task Description
-* The main tasks <img src="https://render.githubusercontent.com/render/math?math=T_k"> (<img src="https://render.githubusercontent.com/render/math?math=task_k"> in Figure1) refers to forecasting return of stock <img src="https://render.githubusercontent.com/render/math?math=i"> as following,
+Due to different data versions and different Qlib versions, the original data and data preprocessing methods of the experimental settings in the paper are different from those experimental settings in the existing Qlib version. Therefore, we provide two versions of the code according to the two kinds of settings, 1) the [code](https://github.com/lwwang1995/tcts) that can be used to reproduce the experimental results and 2) the [code](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py) in the current Qlib baseline.
+
+#### Setting1
+* Dataset: We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. We split the data into training (01/01/2008-12/31/2013), validation (01/01/2014-12/31/2015), and test sets (01/01/2016-08/01/2020) based on the transaction time. 
+
+* The main tasks <img src="https://latex.codecogs.com/png.latex?T_k" title="T_k" /> refers to forecasting return of stock <img src="https://latex.codecogs.com/png.latex?i" title="i" /> as following,
 <div align=center>
-<img src="https://render.githubusercontent.com/render/math?math=r_{i}^k = \frac{\price_i^{t+k}}{\price_i^{t+k-1}} - 1">
+<img src="https://latex.codecogs.com/png.image?\dpi{110}&space;r_{i}^{t,k}&space;=&space;\frac{price_i^{t&plus;k}}{price_i^{t&plus;k-1}}-1" title="r_{i}^{t,k} = \frac{price_i^{t+k}}{price_i^{t+k-1}}-1" />
 </div>
 
-* Temporally correlated task sets <img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_k = \{T_1, T_2, ... , T_k\}">, in this paper, <img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_3">, <img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_5"> and <img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_10"> are used.
-#### Baselines
-* GRU/MLP/LightGBM (LGB)/Graph Attention Networks (GAT)
-* Multi-task learning (MTL): In multi-task learning, multiple tasks are jointly trained and mutually boosted. Each task is treated equally, while in our setting, we focus on the main task.
-* Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task <img src="https://render.githubusercontent.com/render/math?math=T_1" >, then <img src="https://render.githubusercontent.com/render/math?math=T_2" >, and gradually move to the last one.
-#### Result
-| Methods | <img src="https://render.githubusercontent.com/render/math?math=T_1" > | <img src="https://render.githubusercontent.com/render/math?math=T_2"> | <img src="https://render.githubusercontent.com/render/math?math=T_3"> |
-| :----: | :----: | :----: | :----: |
-| GRU | 0.049 / 1.903 | 0.018 / 1.972 | 0.014 / 1.989 |
-| MLP | 0.023 / 1.961 | 0.022 / 1.962 | 0.015 / 1.978 |
-| LGB | 0.038 / 1.883 | 0.023 / 1.952 | 0.007 / 1.987 |
-| GAT | 0.052 / 1.898 | 0.024 / 1.954 | 0.015 / 1.973 |
-| MTL(<img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_3">)  | 0.061 / 1.862  | 0.023 / 1.942  | 0.012 / 1.956 |
-| CL(<img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_3">)  | 0.051 / 1.880  | 0.028 / 1.941  | 0.016 / 1.962 |
-| Ours(<img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_3">)  | 0.071 / 1.851  | 0.030 / 1.939  | 0.017 / 1.963 |
-| MTL(<img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_5">)  | 0.057 / 1.875  | 0.021 / 1.939  | 0.017 / 1.959 |
-| CL(<img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_5">)  | 0.056 / 1.877  | 0.028 / 1.942  | 0.015 / 1.962 |
-| Ours(<img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_5">)  | 0.075 / 1.849  | 0.032 /1.939  | 0.021 / 1.955  | 
-| MTL(<img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_{10}">)  | 0.052 / 1.882  | 0.020 / 1.947  | 0.019 / 1.952 |
-| CL(<img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_{10}">)  | 0.051 / 1.882  | 0.028 / 1.950  | 0.016 / 1.961 |
-| Ours(<img src="https://render.githubusercontent.com/render/math?math=\mathcal{T}_{10}">)  | 0.067 /  1.867  | 0.030 / 1.960  | 0.022 / 1.942|
+* Temporally correlated task sets <img src="https://latex.codecogs.com/png.latex?\mathcal{T}_k&space;=&space;\{T_1,&space;T_2,&space;...&space;,&space;T_k\}" title="\mathcal{T}_k = \{T_1, T_2, ... , T_k\}" />, in this paper, <img src="https://latex.codecogs.com/png.latex?\mathcal{T}_3" title="\mathcal{T}_3" />, <img src="https://latex.codecogs.com/png.latex?\mathcal{T}_5" title="\mathcal{T}_5" /> and <img src="https://latex.codecogs.com/png.latex?\mathcal{T}_{10}" title="\mathcal{T}_{10}" /> are used in <img src="https://latex.codecogs.com/png.latex?T_1" title="T_1" />, <img src="https://latex.codecogs.com/png.latex?T_2" title="T_2" />, and <img src="https://latex.codecogs.com/png.latex?T_3" title="T_3" />.
+
+#### Setting2
+* Dataset: We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. We split the data into training (01/01/2008-12/31/2014), validation (01/01/2015-12/31/2016), and test sets (01/01/2017-08/01/2020) based on the transaction time. 
+
+* The main tasks <img src="https://latex.codecogs.com/png.latex?T_k" title="T_k" /> refers to forecasting return of stock <img src="https://latex.codecogs.com/png.latex?i" title="i" /> as following,
+<div align=center>
+<img src="https://latex.codecogs.com/png.image?\dpi{110}&space;r_{i}^{t,k}&space;=&space;\frac{price_i^{t&plus;1&plus;k}}{price_i^{t&plus;1}}-1" title="r_{i}^{t,k} = \frac{price_i^{t+1+k}}{price_i^{t+1}}-1" />
+</div>
+
+* In Qlib baseline, <img src="https://latex.codecogs.com/png.latex?\mathcal{T}_3" title="\mathcal{T}_3" />, is used in  <img src="https://latex.codecogs.com/png.latex?T_1" title="T_1" />.
+
+### Experimental Result
+You can find the experimental result of setting1 in the [paper](http://proceedings.mlr.press/v139/wu21e/wu21e.pdf) and the experimental result of setting2 in this [page](https://github.com/microsoft/qlib/tree/main/examples/benchmarks).

examples/benchmarks/TCTS/requirements.txt

@@ -0,0 +1,4 @@
+pandas==1.1.2
+numpy==1.17.4
+scikit_learn==0.23.2
+torch==1.7.0

examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml

@@ -22,25 +22,30 @@ data_handler_config: &data_handler_config
         - class: CSRankNorm
           kwargs:
               fields_group: label
-    label: ["Ref($close, -1) / $close - 1",
-            "Ref($close, -2) / Ref($close, -1) - 1",
-            "Ref($close, -3) / Ref($close, -2) - 1"]
+    label: ["Ref($close, -2) / Ref($close, -1) - 1", 
+            "Ref($close, -3) / Ref($close, -1) - 1", 
+            "Ref($close, -4) / Ref($close, -1) - 1"]
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: TCTS
@@ -49,9 +54,8 @@ task:
             d_feat: 6
             hidden_size: 64
             num_layers: 2
-            dropout: 0.0
+            dropout: 0.3
             n_epochs: 200
-            lr: 1e-3
             early_stop: 20
             batch_size: 800
             metric: loss
@@ -60,10 +64,10 @@ task:
             fore_optimizer: adam
             weight_optimizer: adam
             output_dim: 3
-            fore_lr: 5e-4
-            weight_lr: 5e-4
+            fore_lr: 2e-3
+            weight_lr: 2e-3
             steps: 3
-            target_label: 1
+            target_label: 0
             lowest_valid_performance: 0.993
     dataset:
         class: DatasetH
@@ -80,13 +84,14 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
             ana_long_short: False
             ann_scaler: 252
-            label_col: 1
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/TFT/README.md

@@ -8,7 +8,7 @@
 Users can follow the ``workflow_by_code_tft.py`` to run the benchmark. 
 
 ### Notes
-1. Please be **aware** that this script can only support `Python 3.5 - 3.8`.
+1. Please be **aware** that this script can only support `Python 3.6 - 3.7`.
 2. If the CUDA version on your machine is not 10.0, please remember to run the following commands `conda install anaconda cudatoolkit=10.0` and `conda install cudnn` on your machine.
 3. The model must run in GPU, or an error will be raised.
 4. New datasets should be registered in ``data_formatters``, for detail please visit the source.

examples/benchmarks/TFT/data_formatters/qlib_Alpha158.py

@@ -195,7 +195,8 @@ class Alpha158Formatter(GenericDataFormatter):
 
         for col in column_names:
             if col not in {"forecast_time", "identifier"}:
-                output[col] = self._target_scaler.inverse_transform(predictions[col])
+                # Using [col] is for aligning with the format when fitting
+                output[col] = self._target_scaler.inverse_transform(predictions[[col]])
 
         return output
 

examples/benchmarks/TFT/requirements.txt

@@ -1,3 +1,2 @@
 tensorflow-gpu==1.15.0
-numpy == 1.19.4
-pandas==1.1.0
+pandas==1.1.0

examples/benchmarks/TFT/tft.py

@@ -1,6 +1,8 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
+from pathlib import Path
+from typing import Union
 import numpy as np
 import pandas as pd
 import tensorflow.compat.v1 as tf
@@ -243,7 +245,7 @@ class TFTModel(ModelFT):
             #    extract_numerical_data(targets), extract_numerical_data(p90_forecast),
             #    0.9)
             tf.keras.backend.set_session(default_keras_session)
-        print("Training completed.".format(dte.datetime.now()))
+        print("Training completed at {}.".format(dte.datetime.now()))
         # ===========================Training Process===========================
 
     def predict(self, dataset):
@@ -289,3 +291,31 @@ class TFTModel(ModelFT):
             dataset for finetuning
         """
         pass
+
+    def to_pickle(self, path: Union[Path, str]):
+        """
+        Tensorflow model can't be dumped directly.
+        So the data should be save seperatedly
+
+        **TODO**: Please implement the function to load the files
+
+        Parameters
+        ----------
+        path : Union[Path, str]
+            the target path to be dumped
+        """
+        # FIXME: implementing saving tensorflow models
+        # save tensorflow model
+        # path = Path(path)
+        # path.mkdir(parents=True)
+        # self.model.save(path)
+
+        # save qlib model wrapper
+        drop_attrs = ["model", "tf_graph", "sess", "data_formatter"]
+        orig_attr = {}
+        for attr in drop_attrs:
+            orig_attr[attr] = getattr(self, attr)
+            setattr(self, attr, None)
+        super(TFTModel, self).to_pickle(path)
+        for attr in drop_attrs:
+            setattr(self, attr, orig_attr[attr])

examples/benchmarks/TFT/workflow_config_tft_Alpha158.yaml

@@ -14,19 +14,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: TFTModel
@@ -46,7 +51,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/TRA/README.md

@@ -1,53 +1,78 @@
 # Learning Multiple Stock Trading Patterns with Temporal Routing Adaptor and Optimal Transport
 
-This code provides a PyTorch implementation for TRA (Temporal Routing Adaptor), as described in the paper [Learning Multiple Stock Trading Patterns with Temporal Routing Adaptor and Optimal Transport](http://arxiv.org/abs/2106.12950).
+Temporal Routing Adaptor (TRA) is designed to capture multiple trading patterns in the stock market data. Please refer to [our paper](http://arxiv.org/abs/2106.12950) for more details.
 
-* TRA (Temporal Routing Adaptor) is a lightweight module that consists of a set of independent predictors for learning multiple patterns as well as a router to dispatch samples to different predictors.
-* We also design a learning algorithm based on Optimal Transport (OT) to obtain the optimal sample to predictor assignment and effectively optimize the router with such assignment through an auxiliary loss term.
+If you find our work useful in your research, please cite:
+```
+@inproceedings{HengxuKDD2021,
+ author = {Hengxu Lin and Dong Zhou and Weiqing Liu and Jiang Bian},
+ title = {Learning Multiple Stock Trading Patterns with Temporal Routing Adaptor and Optimal Transport},
+ booktitle = {Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery \& Data Mining},
+ series = {KDD '21},
+ year = {2021},
+ publisher = {ACM},
+}
 
+@article{yang2020qlib,
+  title={Qlib: An AI-oriented Quantitative Investment Platform},
+  author={Yang, Xiao and Liu, Weiqing and Zhou, Dong and Bian, Jiang and Liu, Tie-Yan},
+  journal={arXiv preprint arXiv:2009.11189},
+  year={2020}
+}
+```
 
-# Running TRA 
+## Usage (Recommended)
 
-## Requirements
-- Install `Qlib` main branch
+**Update**: `TRA` has been moved to `qlib.contrib.model.pytorch_tra` to support other `Qlib` components like  `qlib.workflow` and `Alpha158/Alpha360` dataset.
 
-## Running 
+Please follow the official [doc](https://qlib.readthedocs.io/en/latest/component/workflow.html) to use `TRA` with `workflow`. Here we also provide several example config files:
+
+- `workflow_config_tra_Alpha360.yaml`: running `TRA` with `Alpha360` dataset
+- `workflow_config_tra_Alpha158.yaml`: running `TRA` with `Alpha158` dataset (with feature subsampling)
+- `workflow_config_tra_Alpha158_full.yaml`: running `TRA` with `Alpha158` dataset (without feature subsampling)
+
+The performances of `TRA` are reported in [Benchmarks](https://github.com/microsoft/qlib/tree/main/examples/benchmarks).
+
+## Usage (Not Maintained)
+
+This section is used to reproduce the results in the paper.
+
+### Running
 
 We attach our running scripts for the paper in `run.sh`.
 
 And here are two ways to run the model:
 
 * Running from scripts with default parameters
-    You can directly run from Qlib command `qrun`:
-    ```
-    qrun configs/config_alstm.yaml
-    ```
+
+  You can directly run from Qlib command `qrun`:
+  ```
+  qrun configs/config_alstm.yaml
+  ```
 
 * Running from code with self-defined parameters
-    Setting different parameters is also allowed. See codes in `example.py`:
-    ```
-    python example.py --config_file configs/config_alstm.yaml
-    ```
+
+  Setting different parameters is also allowed. See codes in `example.py`:
+  ```
+  python example.py --config_file configs/config_alstm.yaml
+  ```
 
 Here we trained TRA on a pretrained backbone model. Therefore we run `*_init.yaml` before TRA's scipts.
 
-# Results
-
-## Outputs
+### Results
 
 After running the scripts, you can find result files in path `./output`:
 
-`info.json` - config settings and result metrics.
+* `info.json` - config settings and result metrics.
+* `log.csv` - running logs.
+* `model.bin` - the model parameter dictionary.
+* `pred.pkl` - the prediction scores and output for inference.
 
-`log.csv` - running logs.
+Evaluation metrics reported in the paper:
+This result is generated by qlib==0.7.1.
 
-`model.bin` - the model parameter dictionary.
-
-`pred.pkl` - the prediction scores and output for inference.
-
-## Our Results
 | Methods | MSE| MAE| IC | ICIR | AR | AV | SR | MDD |
-|-------------------|-------------------|---------------------|--------------------|--------------------|--------------------|--------------------|--------------------|--------------------|
+|-------|-------|------|-----|-----|-----|-----|-----|-----|
 |Linear|0.163|0.327|0.020|0.132|-3.2%|16.8%|-0.191|32.1%|
 |LightGBM|0.160(0.000)|0.323(0.000)|0.041|0.292|7.8%|15.5%|0.503|25.7%|
 |MLP|0.160(0.002)|0.323(0.003)|0.037|0.273|3.7%|15.3%|0.264|26.2%|
@@ -61,21 +86,8 @@ After running the scripts, you can find result files in path `./output`:
 
 A more detailed demo for our experiment results in the paper can be found in `Report.ipynb`.
 
-# Common Issues
+## Common Issues
 
 For help or issues using TRA, please submit a GitHub issue.
 
-Sometimes we might encounter situation where the loss is `NaN`, please check the `epsilon` parameter in the sinkhorn algorithm, adjusting the `epsilon` according to input's scale is important. 
-
-# Citation
-If you find this repository useful in your research, please cite:
-```
-@inproceedings{HengxuKDD2021,
- author = {Hengxu Lin and Dong Zhou and Weiqing Liu and Jiang Bian},
- title = {Learning Multiple Stock Trading Patterns with Temporal Routing Adaptor and Optimal Transport},
- booktitle = {Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery \& Data Mining},
- series = {KDD '21},
- year = {2021},
- publisher = {ACM},
-}
-```
+Sometimes we might encounter situation where the loss is `NaN`, please check the `epsilon` parameter in the sinkhorn algorithm, adjusting the `epsilon` according to input's scale is important.

examples/benchmarks/TRA/requirements.txt

@@ -0,0 +1,5 @@
+pandas==1.1.2
+numpy==1.17.4
+scikit_learn==0.23.2
+torch==1.7.0
+seaborn

examples/benchmarks/TRA/src/model.py

@@ -38,7 +38,7 @@ class TRAModel(Model):
         model_init_state=None,
         lamb=0.0,
         rho=0.99,
-        seed=0,
+        seed=None,
         logdir=None,
         eval_train=True,
         eval_test=False,

examples/benchmarks/TRA/workflow_config_tra_Alpha158.yaml

@@ -0,0 +1,136 @@
+qlib_init:
+  provider_uri: "~/.qlib/qlib_data/cn_data"
+  region: cn
+
+market: &market csi300
+benchmark: &benchmark SH000300
+
+data_handler_config: &data_handler_config
+  start_time: 2008-01-01
+  end_time: 2020-08-01
+  fit_start_time: 2008-01-01
+  fit_end_time: 2014-12-31
+  instruments: *market
+  infer_processors:
+    - class: FilterCol
+      kwargs:
+        fields_group: feature
+        col_list: ["RESI5", "WVMA5", "RSQR5", "KLEN", "RSQR10", "CORR5", "CORD5", "CORR10",
+                   "ROC60", "RESI10", "VSTD5", "RSQR60", "CORR60", "WVMA60", "STD5",
+                   "RSQR20", "CORD60", "CORD10", "CORR20", "KLOW"]
+    - class: RobustZScoreNorm
+      kwargs:
+        fields_group: feature
+        clip_outlier: true
+    - class: Fillna
+      kwargs:
+        fields_group: feature
+  learn_processors:
+    - class: CSRankNorm
+      kwargs:
+        fields_group: label
+  label: ["Ref($close, -2) / Ref($close, -1) - 1"]
+
+num_states: &num_states 3
+
+memory_mode: &memory_mode sample
+
+tra_config: &tra_config
+  num_states: *num_states
+  rnn_arch: LSTM
+  hidden_size: 32
+  num_layers: 1
+  dropout: 0.0
+  tau: 1.0
+  src_info: LR_TPE
+
+model_config: &model_config
+  input_size: 20
+  hidden_size: 64
+  num_layers: 2
+  rnn_arch: LSTM
+  use_attn: True
+  dropout: 0.0
+
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+
+task:
+  model:
+    class: TRAModel
+    module_path: qlib.contrib.model.pytorch_tra
+    kwargs:
+      tra_config: *tra_config
+      model_config: *model_config
+      model_type: RNN
+      lr: 1e-3
+      n_epochs: 100
+      max_steps_per_epoch:
+      early_stop: 20
+      logdir: output/Alpha158
+      seed: 0
+      lamb: 1.0
+      rho: 0.99
+      alpha: 0.5
+      transport_method: router
+      memory_mode: *memory_mode
+      eval_train: False
+      eval_test: True
+      pretrain: True
+      init_state:
+      freeze_model: False
+      freeze_predictors: False
+  dataset:
+    class: MTSDatasetH
+    module_path: qlib.contrib.data.dataset
+    kwargs:
+      handler:
+        class: Alpha158
+        module_path: qlib.contrib.data.handler
+        kwargs: *data_handler_config
+      segments:
+        train: [2008-01-01, 2014-12-31]
+        valid: [2015-01-01, 2016-12-31]
+        test: [2017-01-01, 2020-08-01]
+      seq_len: 60
+      horizon: 2
+      input_size:
+      num_states: *num_states
+      batch_size: 1024
+      n_samples:
+      memory_mode: *memory_mode
+      drop_last: True
+  record:
+    - class: SignalRecord
+      module_path: qlib.workflow.record_temp
+      kwargs: 
+        model: <MODEL>
+        dataset: <DATASET>
+    - class: SigAnaRecord
+      module_path: qlib.workflow.record_temp
+      kwargs: 
+        ana_long_short: False
+        ann_scaler: 252
+    - class: PortAnaRecord
+      module_path: qlib.workflow.record_temp
+      kwargs: 
+        config: *port_analysis_config

examples/benchmarks/TRA/workflow_config_tra_Alpha158_full.yaml

@@ -0,0 +1,130 @@
+qlib_init:
+  provider_uri: "~/.qlib/qlib_data/cn_data"
+  region: cn
+
+market: &market csi300
+benchmark: &benchmark SH000300
+
+data_handler_config: &data_handler_config
+  start_time: 2008-01-01
+  end_time: 2020-08-01
+  fit_start_time: 2008-01-01
+  fit_end_time: 2014-12-31
+  instruments: *market
+  infer_processors:
+    - class: RobustZScoreNorm
+      kwargs:
+        fields_group: feature
+        clip_outlier: true
+    - class: Fillna
+      kwargs:
+        fields_group: feature
+  learn_processors:
+    - class: CSRankNorm
+      kwargs:
+        fields_group: label
+  label: ["Ref($close, -2) / Ref($close, -1) - 1"]
+
+num_states: &num_states 3
+
+memory_mode: &memory_mode sample
+
+tra_config: &tra_config
+  num_states: *num_states
+  rnn_arch: LSTM
+  hidden_size: 32
+  num_layers: 1
+  dropout: 0.0
+  tau: 1.0
+  src_info: LR_TPE
+
+model_config: &model_config
+  input_size: 158
+  hidden_size: 256
+  num_layers: 2
+  rnn_arch: LSTM
+  use_attn: True
+  dropout: 0.2
+
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+
+task:
+  model:
+    class: TRAModel
+    module_path: qlib.contrib.model.pytorch_tra
+    kwargs:
+      tra_config: *tra_config
+      model_config: *model_config
+      model_type: RNN
+      lr: 1e-3
+      n_epochs: 100
+      max_steps_per_epoch:
+      early_stop: 20
+      logdir: output/Alpha158_full
+      seed: 0
+      lamb: 1.0
+      rho: 0.99
+      alpha: 0.5
+      transport_method: router
+      memory_mode: *memory_mode
+      eval_train: False
+      eval_test: True
+      pretrain: True
+      init_state:
+      freeze_model: False
+      freeze_predictors: False
+  dataset:
+    class: MTSDatasetH
+    module_path: qlib.contrib.data.dataset
+    kwargs:
+      handler:
+        class: Alpha158
+        module_path: qlib.contrib.data.handler
+        kwargs: *data_handler_config
+      segments:
+        train: [2008-01-01, 2014-12-31]
+        valid: [2015-01-01, 2016-12-31]
+        test: [2017-01-01, 2020-08-01]
+      seq_len: 60
+      horizon: 2
+      input_size:
+      num_states: *num_states
+      batch_size: 1024
+      n_samples:
+      memory_mode: *memory_mode
+      drop_last: True
+  record:
+    - class: SignalRecord
+      module_path: qlib.workflow.record_temp
+      kwargs: 
+        model: <MODEL>
+        dataset: <DATASET>
+    - class: SigAnaRecord
+      module_path: qlib.workflow.record_temp
+      kwargs: 
+        ana_long_short: False
+        ann_scaler: 252
+    - class: PortAnaRecord
+      module_path: qlib.workflow.record_temp
+      kwargs:
+        config: *port_analysis_config

examples/benchmarks/TRA/workflow_config_tra_Alpha360.yaml

@@ -0,0 +1,130 @@
+qlib_init:
+  provider_uri: "~/.qlib/qlib_data/cn_data"
+  region: cn
+
+market: &market csi300
+benchmark: &benchmark SH000300
+
+data_handler_config: &data_handler_config
+  start_time: 2008-01-01
+  end_time: 2020-08-01
+  fit_start_time: 2008-01-01
+  fit_end_time: 2014-12-31
+  instruments: *market
+  infer_processors:
+    - class: RobustZScoreNorm
+      kwargs:
+        fields_group: feature
+        clip_outlier: true
+    - class: Fillna
+      kwargs:
+        fields_group: feature
+  learn_processors:
+    - class: CSRankNorm
+      kwargs:
+        fields_group: label
+  label: ["Ref($close, -2) / Ref($close, -1) - 1"]
+
+num_states: &num_states 3
+
+memory_mode: &memory_mode sample
+
+tra_config: &tra_config
+  num_states: *num_states
+  rnn_arch: LSTM
+  hidden_size: 32
+  num_layers: 1
+  dropout: 0.0
+  tau: 1.0
+  src_info: LR_TPE
+
+model_config: &model_config
+  input_size: 6
+  hidden_size: 64
+  num_layers: 2
+  rnn_arch: LSTM
+  use_attn: True
+  dropout: 0.0
+
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+
+task:
+  model:
+    class: TRAModel
+    module_path: qlib.contrib.model.pytorch_tra
+    kwargs:
+      tra_config: *tra_config
+      model_config: *model_config
+      model_type: RNN
+      lr: 1e-3
+      n_epochs: 100
+      max_steps_per_epoch:
+      early_stop: 20
+      logdir: output/Alpha360
+      seed: 0
+      lamb: 1.0
+      rho: 0.99
+      alpha: 0.5
+      transport_method: router
+      memory_mode: *memory_mode
+      eval_train: False
+      eval_test: True
+      pretrain: True
+      init_state:
+      freeze_model: False
+      freeze_predictors: False
+  dataset:
+    class: MTSDatasetH
+    module_path: qlib.contrib.data.dataset
+    kwargs:
+      handler:
+        class: Alpha360
+        module_path: qlib.contrib.data.handler
+        kwargs: *data_handler_config
+      segments:
+        train: [2008-01-01, 2014-12-31]
+        valid: [2015-01-01, 2016-12-31]
+        test: [2017-01-01, 2020-08-01]
+      seq_len: 60
+      horizon: 2
+      input_size: 6
+      num_states: *num_states
+      batch_size: 1024
+      n_samples:
+      memory_mode: *memory_mode
+      drop_last: True
+  record:
+    - class: SignalRecord
+      module_path: qlib.workflow.record_temp
+      kwargs: 
+        model: <MODEL>
+        dataset: <DATASET>
+    - class: SigAnaRecord
+      module_path: qlib.workflow.record_temp
+      kwargs: 
+        ana_long_short: False
+        ann_scaler: 252
+    - class: PortAnaRecord
+      module_path: qlib.workflow.record_temp
+      kwargs:
+        config: *port_analysis_config

examples/benchmarks/TabNet/workflow_config_TabNet_Alpha158.yaml

@@ -26,19 +26,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: TabnetModel
@@ -46,6 +51,7 @@ task:
         kwargs:
             d_feat: 158
             pretrain: True
+            seed: 993
     dataset:
         class: DatasetH
         module_path: qlib.data.dataset
@@ -63,7 +69,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/TabNet/workflow_config_TabNet_Alpha360.yaml

@@ -26,19 +26,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: TabnetModel
@@ -46,6 +51,7 @@ task:
         kwargs:
             d_feat: 360
             pretrain: True
+            seed: 993
     dataset:
         class: DatasetH
         module_path: qlib.data.dataset
@@ -63,7 +69,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/Transformer/requirements.txt

@@ -0,0 +1,3 @@
+numpy==1.17.4
+pandas==1.1.2
+torch==1.2.0

examples/benchmarks/Transformer/workflow_config_transformer_Alpha158.yaml

@@ -0,0 +1,89 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market csi300
+benchmark: &benchmark SH000300
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    infer_processors:
+        - class: FilterCol
+          kwargs:
+              fields_group: feature
+              col_list: ["RESI5", "WVMA5", "RSQR5", "KLEN", "RSQR10", "CORR5", "CORD5", "CORR10", 
+                            "ROC60", "RESI10", "VSTD5", "RSQR60", "CORR60", "WVMA60", "STD5", 
+                            "RSQR20", "CORD60", "CORD10", "CORR20", "KLOW"
+                        ]
+        - class: RobustZScoreNorm
+          kwargs:
+              fields_group: feature
+              clip_outlier: true
+        - class: Fillna
+          kwargs:
+              fields_group: feature
+    learn_processors:
+        - class: DropnaLabel
+        - class: CSRankNorm
+          kwargs:
+              fields_group: label
+    label: ["Ref($close, -2) / Ref($close, -1) - 1"] 
+
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: TransformerModel
+        module_path: qlib.contrib.model.pytorch_transformer_ts
+        kwargs:
+            seed: 0
+            n_jobs: 20
+    dataset:
+        class: TSDatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha158
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+            step_len: 20
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

examples/benchmarks/Transformer/workflow_config_transformer_Alpha360.yaml

@@ -0,0 +1,80 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market csi300
+benchmark: &benchmark SH000300
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    infer_processors:
+        - class: RobustZScoreNorm
+          kwargs:
+              fields_group: feature
+              clip_outlier: true
+        - class: Fillna
+          kwargs:
+              fields_group: feature
+    learn_processors:
+        - class: DropnaLabel
+        - class: CSRankNorm
+          kwargs:
+              fields_group: label
+    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: TransformerModel
+        module_path: qlib.contrib.model.pytorch_transformer
+        kwargs:
+            d_feat: 6
+            seed: 0
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha360
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

examples/benchmarks/XGBoost/workflow_config_xgboost_Alpha158.yaml

@@ -12,19 +12,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: XGBModel
@@ -52,7 +57,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/XGBoost/workflow_config_xgboost_Alpha360.yaml

@@ -19,19 +19,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: XGBModel
@@ -59,7 +64,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/highfreq/highfreq_handler.py

@@ -99,8 +99,6 @@ class HighFreqHandler(DataHandlerLP):
         ]
         names += ["$volume_1"]
 
-        fields += ["Cut({0}, 240, None)".format(template_paused.format("Date($close)"))]
-        names += ["date"]
         return fields, names
 
 

examples/highfreq/highfreq_ops.py

@@ -5,30 +5,7 @@ from qlib.data.ops import ElemOperator, PairOperator
 from qlib.config import C
 from qlib.data.cache import H
 from qlib.data.data import Cal
-
-
-def get_calendar_day(freq="day", future=False):
-    """Load High-Freq Calendar Date Using Memcache.
-
-    Parameters
-    ----------
-    freq : str
-        frequency of read calendar file.
-    future : bool
-        whether including future trading day.
-
-    Returns
-    -------
-    _calendar:
-        array of date.
-    """
-    flag = f"{freq}_future_{future}_day"
-    if flag in H["c"]:
-        _calendar = H["c"][flag]
-    else:
-        _calendar = np.array(list(map(lambda x: pd.Timestamp(x.date()), Cal.load_calendar(freq, future))))
-        H["c"][flag] = _calendar
-    return _calendar
+from qlib.contrib.ops.high_freq import get_calendar_day
 
 
 class DayLast(ElemOperator):

examples/highfreq/highfreq_processor.py

@@ -33,6 +33,9 @@ class HighFreqNorm(Processor):
             self.feature_vmin[name] = np.nanmin(part_values)
 
     def __call__(self, df_features):
+        df_features["date"] = pd.to_datetime(
+            df_features.index.get_level_values(level="datetime").to_series().dt.date.values
+        )
         df_features.set_index("date", append=True, drop=True, inplace=True)
         df_values = df_features.values
         names = {

examples/highfreq/workflow_config_High_Freq_Tree_Alpha158.yaml

@@ -59,7 +59,7 @@ task:
     record: 
         - class: "SignalRecord"
           module_path: "qlib.workflow.record_temp"
-          kwargs: {}
+          kwargs: 
         - class: "HFSignalRecord"
           module_path: "qlib.workflow.record_temp"
           kwargs: {}

examples/model_rolling/requirements.txt

@@ -0,0 +1 @@
+xgboost

examples/model_rolling/task_manager_rolling.py

@@ -17,7 +17,7 @@ from qlib.workflow.task.gen import RollingGen, task_generator
 from qlib.workflow.task.manage import TaskManager, run_task
 from qlib.workflow.task.collect import RecorderCollector
 from qlib.model.ens.group import RollingGroup
-from qlib.model.trainer import TrainerRM
+from qlib.model.trainer import TrainerRM, task_train
 from qlib.tests.config import CSI100_RECORD_LGB_TASK_CONFIG, CSI100_RECORD_XGBOOST_TASK_CONFIG
 
 

examples/nested_decision_execution/README.md

@@ -0,0 +1,30 @@
+# Nested Decision Execution
+
+This workflow is an example for nested decision execution in backtesting. Qlib supports nested decision execution in backtesting. It means that users can use different strategies to make trade decision in different frequencies.
+
+## Weekly Portfolio Generation and Daily Order Execution
+
+This workflow provides an example that uses a DropoutTopkStrategy (a strategy based on the daily frequency Lightgbm model) in weekly frequency for portfolio generation and uses SBBStrategyEMA (a rule-based strategy that uses EMA for decision-making) to execute orders in daily frequency. 
+
+### Usage
+
+Start backtesting by running the following command:
+```bash
+    python workflow.py backtest
+```
+
+Start collecting data by running the following command:
+```bash
+    python workflow.py collect_data
+```
+
+## Daily Portfolio Generation and Minutely Order Execution
+
+This workflow also provides a high-frequency example that uses a DropoutTopkStrategy for portfolio generation in daily frequency and uses SBBStrategyEMA to execute orders in minutely frequency. 
+
+### Usage
+
+Start backtesting by running the following command:
+```bash
+    python workflow.py backtest_highfreq
+```

examples/nested_decision_execution/workflow.py

@@ -0,0 +1,204 @@
+#  Copyright (c) Microsoft Corporation.
+#  Licensed under the MIT License.
+
+
+import qlib
+import fire
+from qlib.config import REG_CN, HIGH_FREQ_CONFIG
+from qlib.data import D
+from qlib.utils import exists_qlib_data, init_instance_by_config, flatten_dict
+from qlib.workflow import R
+from qlib.workflow.record_temp import SignalRecord, PortAnaRecord
+from qlib.tests.data import GetData
+from qlib.backtest import collect_data
+
+
+class NestedDecisionExecutionWorkflow:
+
+    market = "csi300"
+    benchmark = "SH000300"
+    data_handler_config = {
+        "start_time": "2008-01-01",
+        "end_time": "2021-05-31",
+        "fit_start_time": "2008-01-01",
+        "fit_end_time": "2014-12-31",
+        "instruments": market,
+    }
+
+    task = {
+        "model": {
+            "class": "LGBModel",
+            "module_path": "qlib.contrib.model.gbdt",
+            "kwargs": {
+                "loss": "mse",
+                "colsample_bytree": 0.8879,
+                "learning_rate": 0.0421,
+                "subsample": 0.8789,
+                "lambda_l1": 205.6999,
+                "lambda_l2": 580.9768,
+                "max_depth": 8,
+                "num_leaves": 210,
+                "num_threads": 20,
+            },
+        },
+        "dataset": {
+            "class": "DatasetH",
+            "module_path": "qlib.data.dataset",
+            "kwargs": {
+                "handler": {
+                    "class": "Alpha158",
+                    "module_path": "qlib.contrib.data.handler",
+                    "kwargs": data_handler_config,
+                },
+                "segments": {
+                    "train": ("2007-01-01", "2014-12-31"),
+                    "valid": ("2015-01-01", "2016-12-31"),
+                    "test": ("2020-01-01", "2021-05-31"),
+                },
+            },
+        },
+    }
+
+    port_analysis_config = {
+        "executor": {
+            "class": "NestedExecutor",
+            "module_path": "qlib.backtest.executor",
+            "kwargs": {
+                "time_per_step": "day",
+                "inner_executor": {
+                    "class": "NestedExecutor",
+                    "module_path": "qlib.backtest.executor",
+                    "kwargs": {
+                        "time_per_step": "30min",
+                        "inner_executor": {
+                            "class": "SimulatorExecutor",
+                            "module_path": "qlib.backtest.executor",
+                            "kwargs": {
+                                "time_per_step": "5min",
+                                "generate_portfolio_metrics": True,
+                                "verbose": True,
+                                "indicator_config": {
+                                    "show_indicator": True,
+                                },
+                            },
+                        },
+                        "inner_strategy": {
+                            "class": "TWAPStrategy",
+                            "module_path": "qlib.contrib.strategy.rule_strategy",
+                        },
+                        "generate_portfolio_metrics": True,
+                        "indicator_config": {
+                            "show_indicator": True,
+                        },
+                    },
+                },
+                "inner_strategy": {
+                    "class": "SBBStrategyEMA",
+                    "module_path": "qlib.contrib.strategy.rule_strategy",
+                    "kwargs": {
+                        "instruments": market,
+                        "freq": "1min",
+                    },
+                },
+                "track_data": True,
+                "generate_portfolio_metrics": True,
+                "indicator_config": {
+                    "show_indicator": True,
+                },
+            },
+        },
+        "backtest": {
+            "start_time": "2020-09-20",
+            "end_time": "2021-05-20",
+            "account": 100000000,
+            "exchange_kwargs": {
+                "freq": "1min",
+                "limit_threshold": 0.095,
+                "deal_price": "close",
+                "open_cost": 0.0005,
+                "close_cost": 0.0015,
+                "min_cost": 5,
+            },
+        },
+    }
+
+    def _init_qlib(self):
+        """initialize qlib"""
+        provider_uri_day = "~/.qlib/qlib_data/cn_data"  # target_dir
+        GetData().qlib_data(target_dir=provider_uri_day, region=REG_CN, version="v2", exists_skip=True)
+        provider_uri_1min = HIGH_FREQ_CONFIG.get("provider_uri")
+        GetData().qlib_data(
+            target_dir=provider_uri_1min, interval="1min", region=REG_CN, version="v2", exists_skip=True
+        )
+        provider_uri_map = {"1min": provider_uri_1min, "day": provider_uri_day}
+        qlib.init(provider_uri=provider_uri_map, dataset_cache=None, expression_cache=None)
+
+    def _train_model(self, model, dataset):
+        with R.start(experiment_name="train"):
+            R.log_params(**flatten_dict(self.task))
+            model.fit(dataset)
+            R.save_objects(**{"params.pkl": model})
+
+            # prediction
+            recorder = R.get_recorder()
+            sr = SignalRecord(model, dataset, recorder)
+            sr.generate()
+
+    def backtest(self):
+        self._init_qlib()
+        model = init_instance_by_config(self.task["model"])
+        dataset = init_instance_by_config(self.task["dataset"])
+        self._train_model(model, dataset)
+        strategy_config = {
+            "class": "TopkDropoutStrategy",
+            "module_path": "qlib.contrib.strategy.signal_strategy",
+            "kwargs": {
+                "signal": (model, dataset),
+                "topk": 50,
+                "n_drop": 5,
+            },
+        }
+        self.port_analysis_config["strategy"] = strategy_config
+        self.port_analysis_config["backtest"]["benchmark"] = self.benchmark
+
+        with R.start(experiment_name="backtest"):
+
+            recorder = R.get_recorder()
+            par = PortAnaRecord(
+                recorder,
+                self.port_analysis_config,
+                risk_analysis_freq=["day", "30min", "5min"],
+                indicator_analysis_freq=["day", "30min", "5min"],
+                indicator_analysis_method="value_weighted",
+            )
+            par.generate()
+
+        # user could use following methods to analysis the position
+        # report_normal_df = recorder.load_object("portfolio_analysis/report_normal_1day.pkl")
+        # from qlib.contrib.report import analysis_position
+        # analysis_position.report_graph(report_normal_df)
+
+    def collect_data(self):
+        self._init_qlib()
+        model = init_instance_by_config(self.task["model"])
+        dataset = init_instance_by_config(self.task["dataset"])
+        self._train_model(model, dataset)
+        executor_config = self.port_analysis_config["executor"]
+        backtest_config = self.port_analysis_config["backtest"]
+        backtest_config["benchmark"] = self.benchmark
+        strategy_config = {
+            "class": "TopkDropoutStrategy",
+            "module_path": "qlib.contrib.strategy.signal_strategy",
+            "kwargs": {
+                "signal": (model, dataset),
+                "topk": 50,
+                "n_drop": 5,
+            },
+        }
+        data_generator = collect_data(executor=executor_config, strategy=strategy_config, **backtest_config)
+        for trade_decision in data_generator:
+            print(trade_decision)
+
+
+if __name__ == "__main__":
+    fire.Fire(NestedDecisionExecutionWorkflow)

examples/rolling_process_data/workflow.py

@@ -21,7 +21,6 @@ class RollingDataWorkflow:
 
     def _init_qlib(self):
         """initialize qlib"""
-        # use yahoo_cn_1min data
         provider_uri = "~/.qlib/qlib_data/cn_data"  # target_dir
         GetData().qlib_data(target_dir=provider_uri, region=REG_CN, exists_skip=True)
         qlib.init(provider_uri=provider_uri, region=REG_CN)

examples/run_all_model.py

@@ -6,6 +6,7 @@ import sys
 import fire
 import time
 import glob
+import yaml
 import shutil
 import signal
 import inspect
@@ -24,22 +25,6 @@ from qlib.workflow import R
 from qlib.tests.data import GetData
 
 
-# init qlib
-provider_uri = "~/.qlib/qlib_data/cn_data"
-exp_folder_name = "run_all_model_records"
-exp_path = str(Path(os.getcwd()).resolve() / exp_folder_name)
-exp_manager = {
-    "class": "MLflowExpManager",
-    "module_path": "qlib.workflow.expm",
-    "kwargs": {
-        "uri": "file:" + exp_path,
-        "default_exp_name": "Experiment",
-    },
-}
-
-GetData().qlib_data(target_dir=provider_uri, region=REG_CN, exists_skip=True)
-qlib.init(provider_uri=provider_uri, region=REG_CN, exp_manager=exp_manager)
-
 # decorator to check the arguments
 def only_allow_defined_args(function_to_decorate):
     @functools.wraps(function_to_decorate)
@@ -88,11 +73,12 @@ def create_env():
     sys.stderr.write("\n")
     # get anaconda activate path
     conda_activate = Path(os.environ["CONDA_PREFIX"]) / "bin" / "activate"  # TODO: FIX ME!
-    return env_path, python_path, conda_activate
+    return temp_dir, env_path, python_path, conda_activate
 
 
 # function to execute the cmd
-def execute(cmd):
+def execute(cmd, wait_when_err=False, raise_err=True):
+    print("Running CMD:", cmd)
     with subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=1, universal_newlines=True, shell=True) as p:
         for line in p.stdout:
             sys.stdout.write(line.split("\b")[0])
@@ -102,6 +88,10 @@ def execute(cmd):
                 sys.stdout.write("\b" * 10 + "\b".join(line.split("\b")[1:-1]))
 
     if p.returncode != 0:
+        if wait_when_err:
+            input("Press Enter to Continue")
+        if raise_err:
+            raise RuntimeError(f"Error when executing command: {cmd}")
         return p.stderr
     else:
         return None
@@ -131,14 +121,23 @@ def get_all_folders(models, exclude) -> dict:
 def get_all_files(folder_path, dataset) -> (str, str):
     yaml_path = str(Path(f"{folder_path}") / f"*{dataset}*.yaml")
     req_path = str(Path(f"{folder_path}") / f"*.txt")
-    return glob.glob(yaml_path)[0], glob.glob(req_path)[0]
+    yaml_file = glob.glob(yaml_path)
+    req_file = glob.glob(req_path)
+    if len(yaml_file) == 0:
+        return None, None
+    else:
+        return yaml_file[0], req_file[0]
 
 
 # function to retrieve all the results
 def get_all_results(folders) -> dict:
     results = dict()
     for fn in folders:
-        exp = R.get_exp(experiment_name=fn, create=False)
+        try:
+            exp = R.get_exp(experiment_name=fn, create=False)
+        except ValueError:
+            # No experiment results
+            continue
         recorders = exp.list_recorders()
         result = dict()
         result["annualized_return_with_cost"] = list()
@@ -152,9 +151,12 @@ def get_all_results(folders) -> dict:
             if recorders[recorder_id].status == "FINISHED":
                 recorder = R.get_recorder(recorder_id=recorder_id, experiment_name=fn)
                 metrics = recorder.list_metrics()
-                result["annualized_return_with_cost"].append(metrics["excess_return_with_cost.annualized_return"])
-                result["information_ratio_with_cost"].append(metrics["excess_return_with_cost.information_ratio"])
-                result["max_drawdown_with_cost"].append(metrics["excess_return_with_cost.max_drawdown"])
+                if "1day.excess_return_with_cost.annualized_return" not in metrics:
+                    print(f"{recorder_id} is skipped due to incomplete result")
+                    continue
+                result["annualized_return_with_cost"].append(metrics["1day.excess_return_with_cost.annualized_return"])
+                result["information_ratio_with_cost"].append(metrics["1day.excess_return_with_cost.information_ratio"])
+                result["max_drawdown_with_cost"].append(metrics["1day.excess_return_with_cost.max_drawdown"])
                 result["ic"].append(metrics["IC"])
                 result["icir"].append(metrics["ICIR"])
                 result["rank_ic"].append(metrics["Rank IC"])
@@ -182,117 +184,202 @@ def gen_and_save_md_table(metrics, dataset):
     return table
 
 
-# function to run the all the models
-@only_allow_defined_args
-def run(times=1, models=None, dataset="Alpha360", exclude=False):
-    """
-    Please be aware that this function can only work under Linux. MacOS and Windows will be supported in the future.
-    Any PR to enhance this method is highly welcomed. Besides, this script doesn't support parrallel running the same model
-    for multiple times, and this will be fixed in the future development.
+# read yaml, remove seed kwargs of model, and then save file in the temp_dir
+def gen_yaml_file_without_seed_kwargs(yaml_path, temp_dir):
+    with open(yaml_path, "r") as fp:
+        config = yaml.load(fp)
+    try:
+        del config["task"]["model"]["kwargs"]["seed"]
+    except KeyError:
+        # If the key does not exists, use original yaml
+        # NOTE: it is very important if the model most run in original path(when sys.rel_path is used)
+        return yaml_path
+    else:
+        # otherwise, generating a new yaml without random seed
+        file_name = yaml_path.split("/")[-1]
+        temp_path = os.path.join(temp_dir, file_name)
+        with open(temp_path, "w") as fp:
+            yaml.dump(config, fp)
+        return temp_path
 
-    Parameters:
-    -----------
-    times : int
-        determines how many times the model should be running.
-    models : str or list
-        determines the specific model or list of models to run or exclude.
-    exclude : boolean
-        determines whether the model being used is excluded or included.
-    dataset : str
-        determines the dataset to be used for each model.
 
-    Usage:
-    -------
-    Here are some use cases of the function in the bash:
+class ModelRunner:
+    def _init_qlib(self, exp_folder_name):
+        # init qlib
+        GetData().qlib_data(exists_skip=True)
+        qlib.init(
+            exp_manager={
+                "class": "MLflowExpManager",
+                "module_path": "qlib.workflow.expm",
+                "kwargs": {
+                    "uri": "file:" + str(Path(os.getcwd()).resolve() / exp_folder_name),
+                    "default_exp_name": "Experiment",
+                },
+            }
+        )
 
-    .. code-block:: bash
+    # function to run the all the models
+    @only_allow_defined_args
+    def run(
+        self,
+        times=1,
+        models=None,
+        dataset="Alpha360",
+        exclude=False,
+        qlib_uri: str = "git+https://github.com/microsoft/qlib#egg=pyqlib",
+        exp_folder_name: str = "run_all_model_records",
+        wait_before_rm_env: bool = False,
+        wait_when_err: bool = False,
+    ):
+        """
+        Please be aware that this function can only work under Linux. MacOS and Windows will be supported in the future.
+        Any PR to enhance this method is highly welcomed. Besides, this script doesn't support parallel running the same model
+        for multiple times, and this will be fixed in the future development.
 
-        # Case 1 - run all models multiple times
-        python run_all_model.py 3
+        Parameters:
+        -----------
+        times : int
+            determines how many times the model should be running.
+        models : str or list
+            determines the specific model or list of models to run or exclude.
+        exclude : boolean
+            determines whether the model being used is excluded or included.
+        dataset : str
+            determines the dataset to be used for each model.
+        qlib_uri : str
+            the uri to install qlib with pip
+            it could be url on the we or local path
+        exp_folder_name: str
+            the name of the experiment folder
+        wait_before_rm_env : bool
+            wait before remove environment.
+        wait_when_err : bool
+            wait when errors raised when executing commands
 
-        # Case 2 - run specific models multiple times
-        python run_all_model.py 3 mlp
+        Usage:
+        -------
+        Here are some use cases of the function in the bash:
 
-        # Case 3 - run specific models multiple times with specific dataset
-        python run_all_model.py 3 mlp Alpha158
+        .. code-block:: bash
 
-        # Case 4 - run other models except those are given as arguments for multiple times
-        python run_all_model.py 3 [mlp,tft,lstm] --exclude=True
+            # Case 1 - run all models multiple times
+            python run_all_model.py run 3
 
-        # Case 5 - run specific models for one time
-        python run_all_model.py --models=[mlp,lightgbm]
+            # Case 2 - run specific models multiple times
+            python run_all_model.py run 3 mlp
 
-        # Case 6 - run other models except those are given as aruments for one time
-        python run_all_model.py --models=[mlp,tft,sfm] --exclude=True
+            # Case 3 - run specific models multiple times with specific dataset
+            python run_all_model.py run 3 mlp Alpha158
 
-    """
-    # get all folders
-    folders = get_all_folders(models, exclude)
-    # init error messages:
-    errors = dict()
-    # run all the model for iterations
-    for fn in folders:
-        # create env by anaconda
-        env_path, python_path, conda_activate = create_env()
-        # get all files
-        sys.stderr.write("Retrieving files...\n")
-        yaml_path, req_path = get_all_files(folders[fn], dataset)
-        sys.stderr.write("\n")
-        # install requirements.txt
-        sys.stderr.write("Installing requirements.txt...\n")
-        execute(f"{python_path} -m pip install -r {req_path}")
-        sys.stderr.write("\n")
-        # setup gpu for tft
-        if fn == "TFT":
-            execute(
-                f"conda install -y --prefix {env_path} anaconda cudatoolkit=10.0 && conda install -y --prefix {env_path} cudnn"
-            )
+            # Case 4 - run other models except those are given as arguments for multiple times
+            python run_all_model.py run 3 [mlp,tft,lstm] --exclude=True
+
+            # Case 5 - run specific models for one time
+            python run_all_model.py run --models=[mlp,lightgbm]
+
+            # Case 6 - run other models except those are given as arguments for one time
+            python run_all_model.py run --models=[mlp,tft,sfm] --exclude=True
+
+        """
+        self._init_qlib(exp_folder_name)
+
+        # get all folders
+        folders = get_all_folders(models, exclude)
+        # init error messages:
+        errors = dict()
+        # run all the model for iterations
+        for fn in folders:
+            # get all files
+            sys.stderr.write("Retrieving files...\n")
+            yaml_path, req_path = get_all_files(folders[fn], dataset)
+            if yaml_path is None:
+                sys.stderr.write(f"There is no {dataset}.yaml file in {folders[fn]}")
+                continue
             sys.stderr.write("\n")
-        # install qlib
-        sys.stderr.write("Installing qlib...\n")
-        execute(f"{python_path} -m pip install --upgrade pip")  # TODO: FIX ME!
-        execute(f"{python_path} -m pip install --upgrade cython")  # TODO: FIX ME!
-        if fn == "TFT":
-            execute(
-                f"cd {env_path} && {python_path} -m pip install --upgrade --force-reinstall --ignore-installed PyYAML -e git+https://github.com/microsoft/qlib#egg=pyqlib"
-            )  # TODO: FIX ME!
-        else:
-            execute(
-                f"cd {env_path} && {python_path} -m pip install --upgrade --force-reinstall -e git+https://github.com/microsoft/qlib#egg=pyqlib"
-            )  # TODO: FIX ME!
-        sys.stderr.write("\n")
-        # run workflow_by_config for multiple times
-        for i in range(times):
-            sys.stderr.write(f"Running the model: {fn} for iteration {i+1}...\n")
-            errs = execute(
-                f"{python_path} {env_path / 'src/pyqlib/qlib/workflow/cli.py'} {yaml_path} {fn} {exp_folder_name}"
-            )
-            if errs is not None:
-                _errs = errors.get(fn, {})
-                _errs.update({i: errs})
-                errors[fn] = _errs
+            # create env by anaconda
+            temp_dir, env_path, python_path, conda_activate = create_env()
+
+            # install requirements.txt
+            sys.stderr.write("Installing requirements.txt...\n")
+            with open(req_path) as f:
+                content = f.read()
+            if "torch" in content:
+                # automatically install pytorch according to nvidia's version
+                execute(
+                    f"{python_path} -m pip install light-the-torch", wait_when_err=wait_when_err
+                )  # for automatically installing torch according to the nvidia driver
+                execute(
+                    f"{env_path / 'bin' / 'ltt'} install --install-cmd '{python_path} -m pip install {{packages}}' -- -r {req_path}",
+                    wait_when_err=wait_when_err,
+                )
+            else:
+                execute(f"{python_path} -m pip install -r {req_path}", wait_when_err=wait_when_err)
             sys.stderr.write("\n")
-        # remove env
-        sys.stderr.write(f"Deleting the environment: {env_path}...\n")
-        shutil.rmtree(env_path)
-    # getting all results
-    sys.stderr.write(f"Retrieving results...\n")
-    results = get_all_results(folders)
-    # calculating the mean and std
-    sys.stderr.write(f"Calculating the mean and std of results...\n")
-    results = cal_mean_std(results)
-    # generating md table
-    sys.stderr.write(f"Generating markdown table...\n")
-    gen_and_save_md_table(results, dataset)
-    sys.stderr.write("\n")
-    # print erros
-    sys.stderr.write(f"Here are some of the errors of the models...\n")
-    pprint(errors)
-    sys.stderr.write("\n")
-    # move results folder
-    shutil.move(exp_path, exp_path + f"_{dataset}_{datetime.now().strftime('%Y-%m-%d_%H:%M:%S')}")
-    shutil.move("table.md", f"table_{dataset}_{datetime.now().strftime('%Y-%m-%d_%H:%M:%S')}.md")
+
+            # read yaml, remove seed kwargs of model, and then save file in the temp_dir
+            yaml_path = gen_yaml_file_without_seed_kwargs(yaml_path, temp_dir)
+            # setup gpu for tft
+            if fn == "TFT":
+                execute(
+                    f"conda install -y --prefix {env_path} anaconda cudatoolkit=10.0 && conda install -y --prefix {env_path} cudnn",
+                    wait_when_err=wait_when_err,
+                )
+                sys.stderr.write("\n")
+            # install qlib
+            sys.stderr.write("Installing qlib...\n")
+            execute(f"{python_path} -m pip install --upgrade pip", wait_when_err=wait_when_err)  # TODO: FIX ME!
+            execute(f"{python_path} -m pip install --upgrade cython", wait_when_err=wait_when_err)  # TODO: FIX ME!
+            if fn == "TFT":
+                execute(
+                    f"cd {env_path} && {python_path} -m pip install --upgrade --force-reinstall --ignore-installed PyYAML -e {qlib_uri}",
+                    wait_when_err=wait_when_err,
+                )  # TODO: FIX ME!
+            else:
+                execute(
+                    f"cd {env_path} && {python_path} -m pip install --upgrade --force-reinstall -e {qlib_uri}",
+                    wait_when_err=wait_when_err,
+                )  # TODO: FIX ME!
+            sys.stderr.write("\n")
+            # run workflow_by_config for multiple times
+            for i in range(times):
+                sys.stderr.write(f"Running the model: {fn} for iteration {i+1}...\n")
+                errs = execute(
+                    f"{python_path} {env_path / 'bin' / 'qrun'} {yaml_path} {fn} {exp_folder_name}",
+                    wait_when_err=wait_when_err,
+                )
+                if errs is not None:
+                    _errs = errors.get(fn, {})
+                    _errs.update({i: errs})
+                    errors[fn] = _errs
+                sys.stderr.write("\n")
+            # remove env
+            sys.stderr.write(f"Deleting the environment: {env_path}...\n")
+            if wait_before_rm_env:
+                input("Press Enter to Continue")
+            shutil.rmtree(env_path)
+        # print errors
+        sys.stderr.write(f"Here are some of the errors of the models...\n")
+        pprint(errors)
+        self._collect_results(exp_folder_name, dataset)
+
+    def _collect_results(self, exp_folder_name, dataset):
+        folders = get_all_folders(exp_folder_name, dataset)
+        # getting all results
+        sys.stderr.write(f"Retrieving results...\n")
+        results = get_all_results(folders)
+        if len(results) > 0:
+            # calculating the mean and std
+            sys.stderr.write(f"Calculating the mean and std of results...\n")
+            results = cal_mean_std(results)
+            # generating md table
+            sys.stderr.write(f"Generating markdown table...\n")
+            gen_and_save_md_table(results, dataset)
+            sys.stderr.write("\n")
+        sys.stderr.write("\n")
+        # move results folder
+        shutil.move(exp_folder_name, exp_folder_name + f"_{dataset}_{datetime.now().strftime('%Y-%m-%d_%H:%M:%S')}")
+        shutil.move("table.md", f"table_{dataset}_{datetime.now().strftime('%Y-%m-%d_%H:%M:%S')}.md")
 
 
 if __name__ == "__main__":
-    fire.Fire(run)  # run all the model
+    fire.Fire(ModelRunner)  # run all the model

examples/workflow_by_code.ipynb

@@ -20,9 +20,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "scrolled": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "import sys, site\n",
@@ -66,7 +64,6 @@
     "from qlib.config import REG_CN\n",
     "from qlib.contrib.model.gbdt import LGBModel\n",
     "from qlib.contrib.data.handler import Alpha158\n",
-    "from qlib.contrib.strategy.strategy import TopkDropoutStrategy\n",
     "from qlib.contrib.evaluate import (\n",
     "    backtest as normal_backtest,\n",
     "    risk_analysis,\n",
@@ -197,27 +194,40 @@
     "# prediction, backtest & analysis\n",
     "###################################\n",
     "port_analysis_config = {\n",
+    "    \"executor\": {\n",
+    "        \"class\": \"SimulatorExecutor\",\n",
+    "        \"module_path\": \"qlib.backtest.executor\",\n",
+    "        \"kwargs\": {\n",
+    "            \"time_per_step\": \"day\",\n",
+    "            \"generate_portfolio_metrics\": True,\n",
+    "        },\n",
+    "    },\n",
     "    \"strategy\": {\n",
     "        \"class\": \"TopkDropoutStrategy\",\n",
-    "        \"module_path\": \"qlib.contrib.strategy.strategy\",\n",
+    "        \"module_path\": \"qlib.contrib.strategy.model_strategy\",\n",
     "        \"kwargs\": {\n",
+    "            \"model\": model,\n",
+    "            \"dataset\": dataset,\n",
     "            \"topk\": 50,\n",
     "            \"n_drop\": 5,\n",
     "        },\n",
     "    },\n",
     "    \"backtest\": {\n",
-    "        \"verbose\": False,\n",
-    "        \"limit_threshold\": 0.095,\n",
+    "        \"start_time\": \"2017-01-01\",\n",
+    "        \"end_time\": \"2020-08-01\",\n",
     "        \"account\": 100000000,\n",
     "        \"benchmark\": benchmark,\n",
-    "        \"deal_price\": \"close\",\n",
-    "        \"open_cost\": 0.0005,\n",
-    "        \"close_cost\": 0.0015,\n",
-    "        \"min_cost\": 5,\n",
+    "        \"exchange_kwargs\": {\n",
+    "            \"freq\": \"day\",\n",
+    "            \"limit_threshold\": 0.095,\n",
+    "            \"deal_price\": \"close\",\n",
+    "            \"open_cost\": 0.0005,\n",
+    "            \"close_cost\": 0.0015,\n",
+    "            \"min_cost\": 5,\n",
+    "        },\n",
     "    },\n",
     "}\n",
     "\n",
-    "\n",
     "# backtest and analysis\n",
     "with R.start(experiment_name=\"backtest_analysis\"):\n",
     "    recorder = R.get_recorder(recorder_id=rid, experiment_name=\"train_model\")\n",
@@ -230,7 +240,7 @@
     "    sr.generate()\n",
     "\n",
     "    # backtest & analysis\n",
-    "    par = PortAnaRecord(recorder, port_analysis_config)\n",
+    "    par = PortAnaRecord(recorder, port_analysis_config, \"day\")\n",
     "    par.generate()\n"
    ]
   },
@@ -250,11 +260,12 @@
     "from qlib.contrib.report import analysis_model, analysis_position\n",
     "from qlib.data import D\n",
     "recorder = R.get_recorder(recorder_id=ba_rid, experiment_name=\"backtest_analysis\")\n",
+    "print(recorder)\n",
     "pred_df = recorder.load_object(\"pred.pkl\")\n",
     "pred_df_dates = pred_df.index.get_level_values(level='datetime')\n",
-    "report_normal_df = recorder.load_object(\"portfolio_analysis/report_normal.pkl\")\n",
-    "positions = recorder.load_object(\"portfolio_analysis/positions_normal.pkl\")\n",
-    "analysis_df = recorder.load_object(\"portfolio_analysis/port_analysis.pkl\")"
+    "report_normal_df = recorder.load_object(\"portfolio_analysis/report_normal_1day.pkl\")\n",
+    "positions = recorder.load_object(\"portfolio_analysis/positions_normal_1day.pkl\")\n",
+    "analysis_df = recorder.load_object(\"portfolio_analysis/port_analysis_1day.pkl\")"
    ]
   },
   {
@@ -349,7 +360,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -362,8 +373,7 @@
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.3"
+   "pygments_lexer": "ipython3"
   },
   "toc": {
    "base_numbering": 1,
@@ -381,4 +391,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

examples/workflow_by_code.py

@@ -17,32 +17,43 @@ if __name__ == "__main__":
     GetData().qlib_data(target_dir=provider_uri, region=REG_CN, exists_skip=True)
     qlib.init(provider_uri=provider_uri, region=REG_CN)
 
+    model = init_instance_by_config(CSI300_GBDT_TASK["model"])
+    dataset = init_instance_by_config(CSI300_GBDT_TASK["dataset"])
+
     port_analysis_config = {
+        "executor": {
+            "class": "SimulatorExecutor",
+            "module_path": "qlib.backtest.executor",
+            "kwargs": {
+                "time_per_step": "day",
+                "generate_portfolio_metrics": True,
+            },
+        },
         "strategy": {
             "class": "TopkDropoutStrategy",
-            "module_path": "qlib.contrib.strategy.strategy",
+            "module_path": "qlib.contrib.strategy.signal_strategy",
             "kwargs": {
+                "signal": (model, dataset),
                 "topk": 50,
                 "n_drop": 5,
             },
         },
         "backtest": {
-            "verbose": False,
-            "limit_threshold": 0.095,
+            "start_time": "2017-01-01",
+            "end_time": "2020-08-01",
             "account": 100000000,
             "benchmark": CSI300_BENCH,
-            "deal_price": "close",
-            "open_cost": 0.0005,
-            "close_cost": 0.0015,
-            "min_cost": 5,
-            "return_order": True,
+            "exchange_kwargs": {
+                "freq": "day",
+                "limit_threshold": 0.095,
+                "deal_price": "close",
+                "open_cost": 0.0005,
+                "close_cost": 0.0015,
+                "min_cost": 5,
+            },
         },
     }
 
-    # model initialization
-    model = init_instance_by_config(CSI300_GBDT_TASK["model"])
-    dataset = init_instance_by_config(CSI300_GBDT_TASK["dataset"])
-
     # NOTE: This line is optional
     # It demonstrates that the dataset can be used standalone.
     example_df = dataset.prepare("train")
@@ -61,5 +72,5 @@ if __name__ == "__main__":
 
         # backtest. If users want to use backtest based on their own prediction,
         # please refer to https://qlib.readthedocs.io/en/latest/component/recorder.html#record-template.
-        par = PortAnaRecord(recorder, port_analysis_config)
+        par = PortAnaRecord(recorder, port_analysis_config, "day")
         par.generate()

qlib/__init__.py

@@ -1,17 +1,16 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
+from pathlib import Path
 
-
-__version__ = "0.7.0"
+_version_path = Path(__file__).absolute().parent / "VERSION.txt"  # This file is copyed from setup.py
+__version__ = _version_path.read_text(encoding="utf-8").strip()
 __version__bak = __version__  # This version is backup for QlibConfig.reset_qlib_version
-
-
 import os
+from typing import Union
 import yaml
 import logging
 import platform
 import subprocess
-from pathlib import Path
 from .log import get_module_logger
 
 
@@ -33,69 +32,71 @@ def init(default_conf="client", **kwargs):
     H.clear()
     C.set(default_conf, **kwargs)
 
-    # check path if server/local
-    if C.get_uri_type() == C.LOCAL_URI:
-        if not os.path.exists(C["provider_uri"]):
-            if C["auto_mount"]:
-                logger.error(
-                    f"Invalid provider uri: {C['provider_uri']}, please check if a valid provider uri has been set. This path does not exist."
-                )
-            else:
-                logger.warning(f"auto_path is False, please make sure {C['mount_path']} is mounted")
-    elif C.get_uri_type() == C.NFS_URI:
-        _mount_nfs_uri(C)
-    else:
-        raise NotImplementedError(f"This type of URI is not supported")
+    # mount nfs
+    for _freq, provider_uri in C.provider_uri.items():
+        mount_path = C["mount_path"][_freq]
+        # check path if server/local
+        uri_type = C.dpm.get_uri_type(provider_uri)
+        if uri_type == C.LOCAL_URI:
+            if not Path(provider_uri).exists():
+                if C["auto_mount"]:
+                    logger.error(
+                        f"Invalid provider uri: {provider_uri}, please check if a valid provider uri has been set. This path does not exist."
+                    )
+                else:
+                    logger.warning(f"auto_path is False, please make sure {mount_path} is mounted")
+        elif uri_type == C.NFS_URI:
+            _mount_nfs_uri(provider_uri, mount_path, C["auto_mount"])
+        else:
+            raise NotImplementedError(f"This type of URI is not supported")
 
     C.register()
 
     if "flask_server" in C:
         logger.info(f"flask_server={C['flask_server']}, flask_port={C['flask_port']}")
     logger.info("qlib successfully initialized based on %s settings." % default_conf)
-    logger.info(f"data_path={C.get_data_path()}")
+    data_path = {_freq: C.dpm.get_data_uri(_freq) for _freq in C.dpm.provider_uri.keys()}
+    logger.info(f"data_path={data_path}")
 
 
-def _mount_nfs_uri(C):
+def _mount_nfs_uri(provider_uri, mount_path, auto_mount: bool = False):
 
     LOG = get_module_logger("mount nfs", level=logging.INFO)
-
+    if mount_path is None:
+        raise ValueError(f"Invalid mount path: {mount_path}!")
     # FIXME: the C["provider_uri"] is modified in this function
     # If it is not modified, we can pass only  provider_uri or mount_path instead of C
-    mount_command = "sudo mount.nfs %s %s" % (C["provider_uri"], C["mount_path"])
+    mount_command = "sudo mount.nfs %s %s" % (provider_uri, mount_path)
     # If the provider uri looks like this 172.23.233.89//data/csdesign'
     # It will be a nfs path. The client provider will be used
-    if not C["auto_mount"]:
-        if not os.path.exists(C["mount_path"]):
+    if not auto_mount:
+        if not Path(mount_path).exists():
             raise FileNotFoundError(
-                f"Invalid mount path: {C['mount_path']}! Please mount manually: {mount_command} or Set init parameter `auto_mount=True`"
+                f"Invalid mount path: {mount_path}! Please mount manually: {mount_command} or Set init parameter `auto_mount=True`"
             )
     else:
         # Judging system type
         sys_type = platform.system()
         if "win" in sys_type.lower():
             # system: window
-            exec_result = os.popen("mount -o anon %s %s" % (C["provider_uri"], C["mount_path"] + ":"))
+            exec_result = os.popen("mount -o anon %s %s" % (provider_uri, mount_path + ":"))
             result = exec_result.read()
             if "85" in result:
-                LOG.warning("already mounted or window mount path already exists")
+                LOG.warning(f"{provider_uri} on Windows:{mount_path} is already mounted")
             elif "53" in result:
                 raise OSError("not find network path")
             elif "error" in result or "错误" in result:
                 raise OSError("Invalid mount path")
-            elif C["provider_uri"] in result:
+            elif provider_uri in result:
                 LOG.info("window success mount..")
             else:
                 raise OSError(f"unknown error: {result}")
 
-            # config mount path
-            C["mount_path"] = C["mount_path"] + ":\\"
         else:
             # system: linux/Unix/Mac
             # check mount
-            _remote_uri = C["provider_uri"]
-            _remote_uri = _remote_uri[:-1] if _remote_uri.endswith("/") else _remote_uri
-            _mount_path = C["mount_path"]
-            _mount_path = _mount_path[:-1] if _mount_path.endswith("/") else _mount_path
+            _remote_uri = provider_uri[:-1] if provider_uri.endswith("/") else provider_uri
+            _mount_path = mount_path[:-1] if mount_path.endswith("/") else mount_path
             _check_level_num = 2
             _is_mount = False
             while _check_level_num:
@@ -121,11 +122,9 @@ def _mount_nfs_uri(C):
 
             if not _is_mount:
                 try:
-                    os.makedirs(C["mount_path"], exist_ok=True)
+                    Path(mount_path).mkdir(parents=True, exist_ok=True)
                 except Exception:
-                    raise OSError(
-                        f"Failed to create directory {C['mount_path']}, please create {C['mount_path']} manually!"
-                    )
+                    raise OSError(f"Failed to create directory {mount_path}, please create {mount_path} manually!")
 
                 # check nfs-common
                 command_res = os.popen("dpkg -l | grep nfs-common")
@@ -136,11 +135,11 @@ def _mount_nfs_uri(C):
                 command_status = os.system(mount_command)
                 if command_status == 256:
                     raise OSError(
-                        f"mount {C['provider_uri']} on {C['mount_path']} error! Needs SUDO! Please mount manually: {mount_command}"
+                        f"mount {provider_uri} on {mount_path} error! Needs SUDO! Please mount manually: {mount_command}"
                     )
                 elif command_status == 32512:
                     # LOG.error("Command error")
-                    raise OSError(f"mount {C['provider_uri']} on {C['mount_path']} error! Command error")
+                    raise OSError(f"mount {provider_uri} on {mount_path} error! Command error")
                 elif command_status == 0:
                     LOG.info("Mount finished")
             else:
@@ -153,14 +152,17 @@ def init_from_yaml_conf(conf_path, **kwargs):
     :param conf_path: A path to the qlib config in yml format
     """
 
-    with open(conf_path) as f:
-        config = yaml.safe_load(f)
+    if conf_path is None:
+        config = {}
+    else:
+        with open(conf_path) as f:
+            config = yaml.safe_load(f)
     config.update(kwargs)
     default_conf = config.pop("default_conf", "client")
     init(default_conf, **config)
 
 
-def get_project_path(config_name="config.yaml", cur_path=None) -> Path:
+def get_project_path(config_name="config.yaml", cur_path: Union[Path, str, None] = None) -> Path:
     """
     If users are building a project follow the following pattern.
     - Qlib is a sub folder in project path
@@ -189,6 +191,7 @@ def get_project_path(config_name="config.yaml", cur_path=None) -> Path:
     """
     if cur_path is None:
         cur_path = Path(__file__).absolute().resolve()
+    cur_path = Path(cur_path)
     while True:
         if (cur_path / config_name).exists():
             return cur_path
@@ -204,6 +207,40 @@ def auto_init(**kwargs):
         - The parsing process will be affected by the `conf_type` of the configuration file
     - Init qlib with default config
     - Skip initialization if already initialized
+
+    :**kwargs: it may contain following parameters
+                cur_path: the start path to find the project path
+
+    Here are two examples of the configuration
+
+    Example 1)
+    If you want create a new project-specific config based on a shared configure, you can use  `conf_type: ref`
+
+    .. code-block:: yaml
+
+        conf_type: ref
+        qlib_cfg: '<shared_yaml_config_path>'    # this could be null reference no config from other files
+        # following configs in `qlib_cfg_update` is project=specific
+        qlib_cfg_update:
+            exp_manager:
+                class: "MLflowExpManager"
+                module_path: "qlib.workflow.expm"
+                kwargs:
+                    uri: "file://<your mlflow experiment path>"
+                    default_exp_name: "Experiment"
+
+    Example 2)
+    If you wan to create simple a stand alone config, you can use following config(a.k.a `conf_type: origin`)
+
+    .. code-block:: python
+
+        exp_manager:
+            class: "MLflowExpManager"
+            module_path: "qlib.workflow.expm"
+            kwargs:
+                uri: "file://<your mlflow experiment path>"
+                default_exp_name: "Experiment"
+
     """
     kwargs["skip_if_reg"] = kwargs.get("skip_if_reg", True)
 
@@ -212,6 +249,7 @@ def auto_init(**kwargs):
     except FileNotFoundError:
         init(**kwargs)
     else:
+        logger = get_module_logger("Initialization")
         conf_pp = pp / "config.yaml"
         with conf_pp.open() as f:
             conf = yaml.safe_load(f)
@@ -225,8 +263,14 @@ def auto_init(**kwargs):
             # - There is a shared configure file and you don't want to edit it inplace.
             # - The shared configure may be updated later and you don't want to copy it.
             # - You have some customized config.
-            qlib_conf_path = conf["qlib_cfg"]
-            qlib_conf_update = conf.get("qlib_cfg_update")
-            init_from_yaml_conf(qlib_conf_path, **qlib_conf_update, **kwargs)
-        logger = get_module_logger("Initialization")
+            qlib_conf_path = conf.get("qlib_cfg", None)
+
+            # merge the arguments
+            qlib_conf_update = conf.get("qlib_cfg_update", {})
+            for k, v in kwargs.items():
+                if k in qlib_conf_update:
+                    logger.warning(f"`qlib_conf_update` from conf_pp is override by `kwargs` on key '{k}'")
+            qlib_conf_update.update(kwargs)
+
+            init_from_yaml_conf(qlib_conf_path, **qlib_conf_update)
         logger.info(f"Auto load project config: {conf_pp}")