log environment automatically

* Simplify the download command(microsoft#1232)

* Update simplified download instruction

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,6 +8,7 @@
 <!--- Why is this change required? What problem does it solve? -->
 
 ## How Has This Been Tested?
+<!---  Put an `x` in all the boxes that apply: --->
 - [ ] Pass the test by running: `pytest qlib/tests/test_all_pipeline.py` under upper directory of `qlib`.
 - [ ] If you are adding a new feature, test on your own test scripts.
 

.github/workflows/python-publish.yml

@@ -12,8 +12,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, macos-latest]
-        python-version: [3.6, 3.7, 3.8]
+        os: [windows-latest, macos-11]
+        # FIXME:  macos-latest will raise error now.
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-version: [3.7, 3.8]
 
     steps:
     - uses: actions/checkout@v2
@@ -44,7 +46,8 @@ jobs:
     - name: Build wheel on Linux
       uses: RalfG/python-wheels-manylinux-build@v0.3.1-manylinux2010_x86_64
       with:
-        python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38'
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-versions: 'cp37-cp37m cp38-cp38'
         build-requirements: 'numpy cython'
     - name: Set up Python
       uses: actions/setup-python@v2

.github/workflows/test.yml

@@ -1,120 +0,0 @@
-name: Test 
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [windows-latest, ubuntu-16.04, ubuntu-18.04, ubuntu-20.04, macos-latest]
-        python-version: [3.6, 3.7, 3.8, 3.9]
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Lint with Black
-      run: |
-        cd ..
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe -m pip install black
-          $CONDA\\python.exe -m black qlib -l 120 --check --diff
-        else
-          sudo $CONDA/bin/python -m pip install black
-          $CONDA/bin/python -m black qlib -l 120 --check --diff
-        fi
-      shell: bash
-
-    # Test Qlib installed with pip
-    - name: Install Qlib with pip
-      run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe -m pip install numpy==1.19.5
-          $CONDA\\python.exe -m pip install pyqlib --ignore-installed ruamel.yaml numpy --user
-        else
-          sudo $CONDA/bin/python -m pip install numpy==1.19.5
-          sudo $CONDA/bin/python -m pip install pyqlib --ignore-installed ruamel.yaml numpy
-        fi
-      shell: bash
-    
-    - name: Install Lightgbm for MacOS
-      if: runner.os == 'macOS'
-      run: |
-        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
-        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
-
-    - name: Test data downloads
-      run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
-        else
-          $CONDA/bin/python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
-        fi
-      shell: bash
-
-    - name: Test workflow by config (install from pip)
-      run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe qlib\\workflow\\cli.py examples\\benchmarks\\LightGBM\\workflow_config_lightgbm_Alpha158.yaml 
-          $CONDA\\python.exe -m pip uninstall -y pyqlib
-        else
-          $CONDA/bin/python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml 
-          sudo $CONDA/bin/python -m pip uninstall -y pyqlib
-        fi
-      shell: bash
-      
-    # Test Qlib installed from source
-    - name: Install Qlib from source
-      run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe -m pip install --upgrade cython
-          $CONDA\\python.exe -m pip install numpy jupyter jupyter_contrib_nbextensions
-          $CONDA\\python.exe -m pip install -U scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't
-          $CONDA\\python.exe setup.py install
-        else
-          sudo $CONDA/bin/python -m pip install --upgrade cython
-          sudo $CONDA/bin/python -m pip install numpy jupyter jupyter_contrib_nbextensions
-          sudo $CONDA/bin/python -m pip install -U scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't
-          sudo $CONDA/bin/python setup.py install
-        fi
-      shell: bash
-
-    - name: Install test dependencies
-      run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe -m pip install --upgrade pip
-          $CONDA\\python.exe -m pip install black pytest
-        else
-          sudo $CONDA/bin/python -m pip install --upgrade pip
-          sudo $CONDA/bin/python -m pip install black pytest
-        fi
-      shell: bash 
-
-    - name: Unit tests with Pytest
-      run: |
-        cd tests
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe -m pytest . --durations=0
-        else
-          $CONDA/bin/python -m pytest . --durations=0
-        fi
-      shell: bash
-
-    - name: Test workflow by config (install from source)
-      run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          $CONDA\\python.exe qlib\\workflow\\cli.py examples\\benchmarks\\LightGBM\\workflow_config_lightgbm_Alpha158.yaml 
-        else
-          $CONDA/bin/python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml 
-        fi
-      shell: bash

.github/workflows/test_qlib_from_pip.yml

@@ -0,0 +1,57 @@
+name: Test qlib from pip
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    timeout-minutes: 120
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest]
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-version: [3.7, 3.8]
+
+    steps:
+    - name: Test qlib from pip
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Update pip to the latest version
+      run: |
+        python -m pip install --upgrade pip
+
+    - name: Qlib installation test
+      run: |
+        python -m pip install pyqlib
+        # Specify the numpy version because the numpy upgrade caused the CI test to fail, 
+        # and this line of code will be removed when the next version of qlib is released.
+        python -m pip install "numpy<1.23"
+
+    - name: Install Lightgbm for MacOS
+      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
+      run: |
+        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
+        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
+        # FIX MacOS error: Segmentation fault
+        # reference: https://github.com/microsoft/LightGBM/issues/4229
+        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
+        brew unlink libomp
+        brew install libomp.rb
+
+    - name: Downloads dependencies data
+      run: |
+        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
+
+    - name: Test workflow by config
+      run: |
+        qrun examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml

.github/workflows/test_qlib_from_source.yml

@@ -0,0 +1,155 @@
+name: Test qlib from source
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    timeout-minutes: 180
+    # we may retry for 3 times for `Unit tests with Pytest`
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest]
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-version: [3.7, 3.8]
+
+    steps:
+    - name: Test qlib from source
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Update pip to the latest version
+      run: |
+        python -m pip install --upgrade pip
+
+    - name: Installing pytorch for macos
+      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
+      run: |
+        python -m pip install torch torchvision torchaudio
+
+    - name: Installing pytorch for ubuntu
+      if: ${{ matrix.os == 'ubuntu-18.04' || matrix.os == 'ubuntu-20.04' }}
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
+
+    - name: Installing pytorch for windows
+      if: ${{ matrix.os == 'windows-latest' }}
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install torch torchvision torchaudio
+
+    - name: Set up Python tools
+      run: |
+        python -m pip install --upgrade cython
+        python -m pip install -e .[dev]
+
+    - name: Lint with Black
+      run: |
+        black . -l 120 --check --diff
+
+    - name: Make html with sphinx
+      run: |
+        cd docs 
+        sphinx-build -b html . build
+        cd ..
+
+    # Check Qlib with pylint
+    # TODO: These problems we will solve in the future. Important among them are: W0221, W0223, W0237, E1102
+      # C0103: invalid-name
+      # C0209: consider-using-f-string
+      # R0402: consider-using-from-import
+      # R1705: no-else-return
+      # R1710: inconsistent-return-statements
+      # R1725: super-with-arguments
+      # R1735: use-dict-literal
+      # W0102: dangerous-default-value
+      # W0212: protected-access
+      # W0221: arguments-differ
+      # W0223: abstract-method
+      # W0231: super-init-not-called
+      # W0237: arguments-renamed
+      # W0612: unused-variable
+      # W0621: redefined-outer-name
+      # W0622: redefined-builtin
+      # FIXME: specify exception type
+      # W0703: broad-except
+      # W1309: f-string-without-interpolation
+      # E1102: not-callable
+      # E1136: unsubscriptable-object
+    # References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962
+    - name: Check Qlib with pylint
+      run: |
+        pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' qlib --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500"
+
+    # The following flake8 error codes were ignored:
+      # E501 line too long
+        # Description: We have used black to limit the length of each line to 120.
+      # F541 f-string is missing placeholders
+        # Description: The same thing is done when using pylint for detection.
+      # E266 too many leading '#' for block comment
+        # Description: To make the code more readable, a lot of "#" is used.
+        # This error code appears centrally in:
+          # qlib/backtest/executor.py
+          # qlib/data/ops.py
+          # qlib/utils/__init__.py
+      # E402 module level import not at top of file
+        # Description: There are times when module level import is not available at the top of the file.
+      # W503 line break before binary operator
+        # Description: Since black formats the length of each line of code, it has to perform a line break when a line of arithmetic is too long.
+      # E731 do not assign a lambda expression, use a def
+        # Description: Restricts the use of lambda expressions, but at some point lambda expressions are required.
+      # E203 whitespace before ':'
+        # Description: If there is whitespace before ":", it cannot pass the black check.
+    - name: Check Qlib with flake8
+      run: |
+        flake8 --ignore=E501,F541,E266,E402,W503,E731,E203 --per-file-ignores="__init__.py:F401,F403" qlib
+
+    # https://github.com/python/mypy/issues/10600
+    - name: Check Qlib with mypy
+      run: |
+        mypy qlib --install-types --non-interactive || true
+        mypy qlib --verbose
+
+    - name: Test data downloads
+      run: |
+        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
+        azcopy copy https://qlibpublic.blob.core.windows.net/data/rl /tmp/qlibpublic/data --recursive
+        mv /tmp/qlibpublic/data tests/.data
+
+    - name: Install Lightgbm for MacOS
+      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
+      run: |
+        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
+        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
+        # FIX MacOS error: Segmentation fault
+        # reference: https://github.com/microsoft/LightGBM/issues/4229
+        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
+        brew unlink libomp
+        brew install libomp.rb
+
+    - name: Test workflow by config (install from source)
+      run: |
+        # Version 0.52.0 of numba must be installed manually in CI, otherwise it will cause incompatibility with the latest version of numpy.
+        python -m pip install numba==0.52.0
+        # You must update numpy manually, because when installing python tools, it will try to uninstall numpy and cause CI to fail.
+        python -m pip install --upgrade numpy
+        python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+
+    - name: Unit tests with Pytest
+      uses: nick-fields/retry@v2
+      with:
+        timeout_minutes: 60
+        max_attempts: 3
+        command: |
+          cd tests
+          python -m pytest . -m "not slow" --durations=0

.github/workflows/test_qlib_from_source_slow.yml

@@ -0,0 +1,59 @@
+name: Test qlib from source slow
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    timeout-minutes: 720
+    # we may retry for 3 times for `Unit tests with Pytest`
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest]
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-version: [3.7, 3.8]
+
+    steps:
+    - name: Test qlib from source slow
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Set up Python tools
+      run: |
+        python -m pip install --upgrade pip
+        # python -m pip is necessary to upgrade pip.
+        pip install --upgrade cython numpy
+        pip install -e .[dev]
+
+    - name: Downloads dependencies data
+      run: |
+        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
+
+    - name: Install Lightgbm for MacOS
+      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
+      run: |
+        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
+        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
+        # FIX MacOS error: Segmentation fault
+        # reference: https://github.com/microsoft/LightGBM/issues/4229
+        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
+        brew unlink libomp
+        brew install libomp.rb
+
+    - name: Unit tests with Pytest
+      uses: nick-fields/retry@v2
+      with:
+        timeout_minutes: 240
+        max_attempts: 3
+        command: |
+          cd tests
+          python -m pytest . -m "slow" --durations=0

.gitignore

@@ -20,12 +20,17 @@ dist/
 .nvimrc
 .vscode
 
+qlib/VERSION.txt
 qlib/data/_libs/expanding.cpp
 qlib/data/_libs/rolling.cpp
 examples/estimator/estimator_example/
 
 *.egg-info/
 
+# test related
+test-output.xml
+.output
+.data
 
 # special software
 mlruns/
@@ -33,8 +38,10 @@ mlruns/
 tags
 
 .pytest_cache/
+.mypy_cache/
 .vscode/
 
 *.swp
 
 ./pretrain
+.idea/

.mypy.ini

@@ -0,0 +1,17 @@
+[mypy]
+exclude = (?x)(
+    ^qlib/backtest/high_performance_ds\.py$
+    | ^qlib/contrib
+    | ^qlib/data
+    | ^qlib/model
+    | ^qlib/strategy
+    | ^qlib/tests
+    | ^qlib/utils
+    | ^qlib/workflow
+    | ^qlib/config\.py$
+    | ^qlib/log\.py$
+    | ^qlib/__init__\.py$
+  )
+ignore_missing_imports = true
+disallow_incomplete_defs = true
+follow_imports = skip

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+repos:
+-   repo: https://github.com/psf/black
+    rev: 22.6.0
+    hooks:
+    -   id: black
+        args: ["qlib", "-l 120"]
+
+-   repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+        - id: flake8
+          args: ["--ignore=E501,F541,E266,E402,W503,E731,E203"]

.pylintrc

@@ -0,0 +1,5 @@
+[TYPECHECK]
+# https://stackoverflow.com/a/53572939 
+# List of members which are set dynamically and missed by Pylint inference
+# system, and so shouldn't trigger E1101 when accessed.
+generated-members=numpy.*, torch.*

.readthedocs.yml

@@ -17,5 +17,5 @@ python:
   version: 3.7
   install:
     - requirements: docs/requirements.txt
-    - method: setuptools
-      path: .
+    - method: pip
+      path: .

CHANGES.rst

@@ -1,63 +1,63 @@
 Changelog
-====================
+=========
 Here you can see the full list of changes between each QLib release.
 
 Version 0.1.0
---------------------
+-------------
 This is the initial release of QLib library.
 
 Version 0.1.1
---------------------
+-------------
 Performance optimize. Add more features and operators.
 
 Version 0.1.2
---------------------
-- Support operator syntax. Now ``High() - Low()`` is equivalent to ``Sub(High(), Low())``.   
+-------------
+- Support operator syntax. Now ``High() - Low()`` is equivalent to ``Sub(High(), Low())``.
 - Add more technical indicators.
 
 Version 0.1.3
---------------------
+-------------
 Bug fix and add instruments filtering mechanism.
 
 Version 0.2.0
---------------------
+-------------
 - Redesign ``LocalProvider`` database format for performance improvement.
 - Support load features as string fields.
 - Add scripts for database construction.
 - More operators and technical indicators.
 
 Version 0.2.1
---------------------
+-------------
 - Support registering user-defined ``Provider``.
 - Support use operators in string format, e.g. ``['Ref($close, 1)']`` is valid field format.
-- Support dynamic fields in ``$some_field`` format. And exising fields like ``Close()`` may be deprecated in the future.
+- Support dynamic fields in ``$some_field`` format. And existing fields like ``Close()`` may be deprecated in the future.
 
 Version 0.2.2
---------------------
+-------------
 - Add ``disk_cache`` for reusing features (enabled by default).
 - Add ``qlib.contrib`` for experimental model construction and evaluation.
 
 
 Version 0.2.3
---------------------
+-------------
 - Add ``backtest`` module
 - Decoupling the Strategy, Account, Position, Exchange from the backtest module
 
 Version 0.2.4
---------------------
+-------------
 - Add ``profit attribution`` module
 - Add ``rick_control`` and ``cost_control`` strategies
-  
+
 Version 0.3.0
---------------------
+-------------
 - Add ``estimator`` module
 
 Version 0.3.1
---------------------
+-------------
 - Add ``filter`` module
 
 Version 0.3.2
---------------------
+-------------
 - Add real price trading, if the ``factor`` field in the data set is incomplete, use ``adj_price`` trading
 - Refactor ``handler`` ``launcher`` ``trainer`` code
 - Support ``backtest`` configuration parameters in the configuration file
@@ -65,24 +65,24 @@ Version 0.3.2
 - Fix bug of ``filter`` module
 
 Version 0.3.3
--------------------
+-------------
 - Fix bug of ``filter`` module
 
 Version 0.3.4
---------------------
+-------------
 - Support for ``finetune model``
 - Refactor ``fetcher`` code
 
 Version 0.3.5
---------------------
+-------------
 - Support multi-label training, you can provide multiple label in ``handler``. (But LightGBM doesn't support due to the algorithm itself)
 - Refactor ``handler`` code, dataset.py is no longer used, and you can deploy your own labels and features in ``feature_label_config``
 - Handler only offer DataFrame. Also, ``trainer`` and model.py only receive DataFrame
-- Change ``split_rolling_data``, we roll the data on market calender now, not on normal date
+- Change ``split_rolling_data``, we roll the data on market calendar now, not on normal date
 - Move some date config from ``handler`` to ``trainer``
 
 Version 0.4.0
---------------------
+-------------
 - Add `data` package that holds all data-related codes
 - Reform the data provider structure
 - Create a server for data centralized management `qlib-server<https://amc-msra.visualstudio.com/trading-algo/_git/qlib-server>`_
@@ -100,7 +100,7 @@ Version 0.4.0
 
 
 Version 0.4.1
---------------------
+-------------
 - Add support Windows
 - Fix ``instruments`` type bug
 - Fix ``features`` is empty bug(It will cause failure in updating)
@@ -112,19 +112,19 @@ Version 0.4.1
 
 
 Version 0.4.2
---------------------
+-------------
 - Refactor DataHandler
 - Add ``Alpha360`` DataHandler
 
 
 Version 0.4.3
---------------------
+-------------
 - Implementing Online Inference and Trading Framework
 - Refactoring The interfaces of backtest and strategy module.
 
 
 Version 0.4.4
---------------------
+-------------
 - Optimize cache generation performance
 - Add report module
 - Fix bug when using ``ServerDatasetCache`` offline.
@@ -138,7 +138,7 @@ Version 0.4.4
 
 
 Version 0.4.5
---------------------
+-------------
 - Add multi-kernel implementation for both client and server.
     - Support a new way to load data from client which skips dataset cache.
     - Change the default dataset method from single kernel implementation to multi kernel implementation.
@@ -146,19 +146,34 @@ Version 0.4.5
 - Support a new method to write config file by using dict.
 
 Version 0.4.6
---------------------
+-------------
 - Some bugs are fixed
     - The default config in `Version 0.4.5` is not friendly to daily frequency data.
     - Backtest error in TopkWeightStrategy when `WithInteract=True`.
 
 
 Version 0.5.0
---------------------
+-------------
 - First opensource version
     - Refine the docs, code
     - Add baselines
     - public data crawler
 
-Version greater than Version 0.5.0
-----------------------------------
+
+Version 0.8.0
+-------------
+- The backtest is greatly refactored.
+    - Nested decision execution framework is supported
+    - There are lots of changes for daily trading, it is hard to list all of them. But a few important changes could be noticed
+        - The trading limitation is more accurate;
+            - In `previous version <https://github.com/microsoft/qlib/blob/v0.7.2/qlib/contrib/backtest/exchange.py#L160>`_, longing and shorting actions share the same action.
+            - In `current version <https://github.com/microsoft/qlib/blob/7c31012b507a3823117bddcc693fc64899460b2a/qlib/backtest/exchange.py#L304>`_, the trading limitation is different between logging and shorting action.
+        - The constant is different when calculating annualized metrics.
+            - `Current version <https://github.com/microsoft/qlib/blob/7c31012b507a3823117bddcc693fc64899460b2a/qlib/contrib/evaluate.py#L42>`_ uses more accurate constant than `previous version <https://github.com/microsoft/qlib/blob/v0.7.2/qlib/contrib/evaluate.py#L22>`_
+        - `A new version <https://github.com/microsoft/qlib/blob/7c31012b507a3823117bddcc693fc64899460b2a/qlib/tests/data.py#L17>`_ of data is released. Due to the unstability of Yahoo data source, the data may be different after downloading data again.
+        - Users could check out the backtesting results between  `Current version <https://github.com/microsoft/qlib/tree/7c31012b507a3823117bddcc693fc64899460b2a/examples/benchmarks>`_ and `previous version <https://github.com/microsoft/qlib/tree/v0.7.2/examples/benchmarks>`_
+
+
+Other Versions
+--------------
 Please refer to `Github release Notes <https://github.com/microsoft/qlib/releases>`_

MANIFEST.in

@@ -0,0 +1 @@
+include qlib/VERSION.txt

README.md

@@ -11,20 +11,33 @@
 Recent released features
 | Feature | Status |
 | --                      | ------    |
-| TCTS Model | [Released](https://github.com/microsoft/qlib/pull/491) on July 1, 2021 |
-| Online serving and automatic model rolling | :star: [Released](https://github.com/microsoft/qlib/pull/290) on May 17, 2021 | 
-| DoubleEnsemble Model | [Released](https://github.com/microsoft/qlib/pull/286) on Mar 2, 2021 | 
-| High-frequency data processing example | [Released](https://github.com/microsoft/qlib/pull/257) on Feb 5, 2021  |
-| High-frequency trading example | [Part of code released](https://github.com/microsoft/qlib/pull/227) on Jan 28, 2021  | 
-| High-frequency data(1min) | [Released](https://github.com/microsoft/qlib/pull/221) on Jan 27, 2021 |
-| Tabnet Model | [Released](https://github.com/microsoft/qlib/pull/205) on Jan 22, 2021 | 
+| HIST and IGMTF models | :chart_with_upwards_trend: [Released](https://github.com/microsoft/qlib/pull/1040) on Apr 10, 2022 |
+| Qlib [notebook tutorial](https://github.com/microsoft/qlib/tree/main/examples/tutorial) | 📖 [Released](https://github.com/microsoft/qlib/pull/1037) on Apr 7, 2022 | 
+| Ibovespa index data | :rice: [Released](https://github.com/microsoft/qlib/pull/990) on Apr 6, 2022 |
+| Point-in-Time database | :hammer: [Released](https://github.com/microsoft/qlib/pull/343) on Mar 10, 2022 |
+| Arctic Provider Backend & Orderbook data example | :hammer: [Released](https://github.com/microsoft/qlib/pull/744) on Jan 17, 2022 |
+| Meta-Learning-based framework & DDG-DA  | :chart_with_upwards_trend:  :hammer: [Released](https://github.com/microsoft/qlib/pull/743) on Jan 10, 2022 | 
+| Planning-based portfolio optimization | :hammer: [Released](https://github.com/microsoft/qlib/pull/754) on Dec 28, 2021 | 
+| Release Qlib v0.8.0 | :octocat: [Released](https://github.com/microsoft/qlib/releases/tag/v0.8.0) on Dec 8, 2021 |
+| ADD model | :chart_with_upwards_trend: [Released](https://github.com/microsoft/qlib/pull/704) on Nov 22, 2021 |
+| ADARNN  model | :chart_with_upwards_trend: [Released](https://github.com/microsoft/qlib/pull/689) on Nov 14, 2021 |
+| TCN  model | :chart_with_upwards_trend: [Released](https://github.com/microsoft/qlib/pull/668) on Nov 4, 2021 |
+| Nested Decision Framework | :hammer: [Released](https://github.com/microsoft/qlib/pull/438) on Oct 1, 2021. [Example](https://github.com/microsoft/qlib/blob/main/examples/nested_decision_execution/workflow.py) and [Doc](https://qlib.readthedocs.io/en/latest/component/highfreq.html) |
+| Temporal Routing Adaptor (TRA) | :chart_with_upwards_trend: [Released](https://github.com/microsoft/qlib/pull/531) on July 30, 2021 |
+| Transformer & Localformer | :chart_with_upwards_trend: [Released](https://github.com/microsoft/qlib/pull/508) on July 22, 2021 |
+| Release Qlib v0.7.0 | :octocat: [Released](https://github.com/microsoft/qlib/releases/tag/v0.7.0) on July 12, 2021 |
+| TCTS Model | :chart_with_upwards_trend: [Released](https://github.com/microsoft/qlib/pull/491) on July 1, 2021 |
+| Online serving and automatic model rolling | :hammer:  [Released](https://github.com/microsoft/qlib/pull/290) on May 17, 2021 | 
+| DoubleEnsemble Model | :chart_with_upwards_trend: [Released](https://github.com/microsoft/qlib/pull/286) on Mar 2, 2021 | 
+| High-frequency data processing example | :hammer: [Released](https://github.com/microsoft/qlib/pull/257) on Feb 5, 2021  |
+| High-frequency trading example | :chart_with_upwards_trend: [Part of code released](https://github.com/microsoft/qlib/pull/227) on Jan 28, 2021  | 
+| High-frequency data(1min) | :rice: [Released](https://github.com/microsoft/qlib/pull/221) on Jan 27, 2021 |
+| Tabnet Model | :chart_with_upwards_trend: [Released](https://github.com/microsoft/qlib/pull/205) on Jan 22, 2021 |
 
 Features released before 2021 are not listed here.
 
-
-
 <p align="center">
-  <img src="http://fintech.msra.cn/images_v060/logo/1.png" />
+  <img src="http://fintech.msra.cn/images_v070/logo/1.png" />
 </p>
 
 
@@ -36,54 +49,78 @@ With Qlib, users can easily try ideas to create better Quant investment strategi
 
 For more details, please refer to our paper ["Qlib: An AI-oriented Quantitative Investment Platform"](https://arxiv.org/abs/2009.11189).
 
-- [**Plans**](#plans)
-- [Framework of Qlib](#framework-of-qlib)
-- [Quick Start](#quick-start)
-  - [Installation](#installation)
-  - [Data Preparation](#data-preparation)
-  - [Auto Quant Research Workflow](#auto-quant-research-workflow)
-  - [Building Customized Quant Research Workflow by Code](#building-customized-quant-research-workflow-by-code)
-- [**Quant Model Zoo**](#quant-model-zoo)
-  - [Run a single model](#run-a-single-model)
-  - [Run multiple models](#run-multiple-models)
-- [**Quant Dataset Zoo**](#quant-dataset-zoo)
-- [More About Qlib](#more-about-qlib)
-- [Offline Mode and Online Mode](#offline-mode-and-online-mode)
-  - [Performance of Qlib Data Server](#performance-of-qlib-data-server)
-- [Related Reports](#related-reports)
-- [Contact Us](#contact-us)
-- [Contributing](#contributing)
 
+<table>
+  <tbody>
+    <tr>
+      <th>Frameworks, Tutorial, Data & DevOps</th>
+      <th>Main Challenges & Solutions in Quant Research</th>
+    </tr>
+    <tr>
+      <td>
+        <li><a href="#plans"><strong>Plans</strong></a></li>
+        <li><a href="#framework-of-qlib">Framework of Qlib</a></li>
+        <li><a href="#quick-start">Quick Start</a></li>
+          <ul dir="auto">
+            <li type="circle"><a href="#installation">Installation</a> </li>
+            <li type="circle"><a href="#data-preparation">Data Preparation</a></li>
+            <li type="circle"><a href="#auto-quant-research-workflow">Auto Quant Research Workflow</a></li>
+            <li type="circle"><a href="#building-customized-quant-research-workflow-by-code">Building Customized Quant Research Workflow by Code</a></li></ul>
+        <li><a href="#quant-dataset-zoo"><strong>Quant Dataset Zoo</strong></a></li>
+        <li><a href="#more-about-qlib">More About Qlib</a></li>
+        <li><a href="#offline-mode-and-online-mode">Offline Mode and Online Mode</a>
+        <ul>
+          <li type="circle"><a href="#performance-of-qlib-data-server">Performance of Qlib Data Server</a></li></ul>
+        <li><a href="#related-reports">Related Reports</a></li>
+        <li><a href="#contact-us">Contact Us</a></li>
+        <li><a href="#contributing">Contributing</a></li>
+      </td>
+      <td valign="baseline">
+        <li><a href="#main-challenges--solutions-in-quant-research">Main Challenges &amp; Solutions in Quant Research</a>
+          <ul>
+            <li type="circle"><a href="#forecasting-finding-valuable-signalspatterns">Forecasting: Finding Valuable Signals/Patterns</a>
+              <ul>
+                <li type="disc"><a href="#quant-model-paper-zoo"><strong>Quant Model (Paper) Zoo</strong></a>
+                  <ul>
+                    <li type="circle"><a href="#run-a-single-model">Run a Single Model</a></li>
+                    <li type="circle"><a href="#run-multiple-models">Run Multiple Models</a></li>
+                  </ul>
+                </li>
+              </ul>
+            </li>
+          <li type="circle"><a href="#adapting-to-market-dynamics">Adapting to Market Dynamics</a></li>
+          </ul>
+        </li>
+      </td>
+    </tr>
+  </tbody>
+</table>
 
 # Plans
 New features under development(order by estimated release time).
 Your feedbacks about the features are very important.
-| Feature                        | Status      |
-| --                      | ------    |
-| Planning-based portfolio optimization | Under review:  https://github.com/microsoft/qlib/pull/280 | 
-| Fund data supporting and analysis  |  Under review: https://github.com/microsoft/qlib/pull/292 |
-| Point-in-Time database | Under review: https://github.com/microsoft/qlib/pull/343 |
-| High-frequency trading | Under review: https://github.com/microsoft/qlib/pull/408 | 
-| Meta-Learning-based data selection | Initial opensource version under development |
+<!-- | Feature                        | Status      | -->
+<!-- | --                      | ------    | -->
 
 # Framework of Qlib
 
 <div style="align: center">
-<img src="http://fintech.msra.cn/images_v060/framework.png?v=0.2" />
+<img src="docs/_static/img/framework.svg" />
 </div>
 
-
 At the module level, Qlib is a platform that consists of the above components. The components are designed as loose-coupled modules, and each component could be used stand-alone.
 
 | Name                   | Description                                                                                                                                                                                                                                                                                                                                                             |
 | ------                 | -----                                                                                                                                                                                                                                                                                                                                                                   |
 | `Infrastructure` layer | `Infrastructure` layer provides underlying support for Quant research. `DataServer` provides a high-performance infrastructure for users to manage and retrieve raw data. `Trainer` provides a flexible interface to control the training process of models, which enable algorithms to control the training process.                                                       |
-| `Workflow` layer       | `Workflow` layer covers the whole workflow of quantitative investment. `Information Extractor` extracts data for models. `Forecast Model` focuses on producing all kinds of forecast signals (e.g. _alpha_, risk) for other modules. With these signals `Portfolio Generator` will generate the target portfolio and produce orders to be executed by `Order Executor`. |
+| `Workflow` layer       | `Workflow` layer covers the whole workflow of quantitative investment. `Information Extractor` extracts data for models. `Forecast Model` focuses on producing all kinds of forecast signals (e.g. _alpha_, risk) for other modules. With these signals `Decision Generator` will generate the target trading decisions(i.e. portfolio, orders)  to be executed by `Execution Env` (i.e. the trading market).  There may be multiple levels of `Trading Agent` and `Execution Env` (e.g. an _order executor trading agent and intraday order execution environment_ could behave like an interday trading environment and nested in  _daily portfolio management trading agent and interday trading environment_  ) |
 | `Interface` layer      | `Interface` layer tries to present a user-friendly interface for the underlying system. `Analyser` module will provide users detailed analysis reports of forecasting signals, portfolios and execution results                                                                                                                                                                 |
 
 * The modules with hand-drawn style are under development and will be released in the future.
 * The modules with dashed borders are highly user-customizable and extendible.
 
+(p.s. framework image is created with https://draw.io/)
+
 
 # Quick Start
 
@@ -99,14 +136,15 @@ Here is a quick **[demo](https://terminalizer.com/view/3f24561a4470)** shows how
 This table demonstrates the supported Python version of `Qlib`:
 |               | install with pip           | install from source  | plot |
 | ------------- |:---------------------:|:--------------------:|:----:|
-| Python 3.6    | :heavy_check_mark:    | :heavy_check_mark: (only with `Anaconda`)                  | :heavy_check_mark: |
 | Python 3.7    | :heavy_check_mark:    | :heavy_check_mark:   | :heavy_check_mark: |
 | Python 3.8    | :heavy_check_mark:    | :heavy_check_mark:   | :heavy_check_mark: |
 | Python 3.9    | :x:                   | :heavy_check_mark:   | :x: |
 
 **Note**: 
+1. **Conda** is suggested for managing your Python environment.
 1. Please pay attention that installing cython in Python 3.6 will raise some error when installing ``Qlib`` from source. If users use Python 3.6 on their machines, it is recommended to *upgrade* Python to version 3.7 or use `conda`'s Python to install ``Qlib`` from source.
-2. For Python 3.9, `Qlib` supports running workflows such as training models, doing backtest and plot most of the related figures (those included in [notebook](examples/workflow_by_code.ipynb)). However, plotting for the *model performance* is not supported for now and we will fix this when the dependent packages are upgraded in the future.
+1. For Python 3.9, `Qlib` supports running workflows such as training models, doing backtest and plot most of the related figures (those included in [notebook](examples/workflow_by_code.ipynb)). However, plotting for the *model performance* is not supported for now and we will fix this when the dependent packages are upgraded in the future.
+1. `Qlib`Requires `tables` package, `hdf5` in tables does not support python3.9. 
 
 ### Install with pip
 Users can easily install ``Qlib`` by pip according to the following command.
@@ -128,22 +166,29 @@ Also, users can install the latest dev version ``Qlib`` by the source code accor
   ```
 
 * Clone the repository and install ``Qlib`` as follows.
-  * If you haven't installed qlib by the command ``pip install pyqlib`` before:
-    ```bash
-    git clone https://github.com/microsoft/qlib.git && cd qlib
-    python setup.py install
-    ```
-  * If you have already installed the stable version by the command ``pip install pyqlib``:
     ```bash
     git clone https://github.com/microsoft/qlib.git && cd qlib
     pip install .
     ```
-  **Note**: **Only** the command ``pip install .`` **can** overwrite the stable version installed by ``pip install pyqlib``, while the command ``python setup.py install`` **can't**.
+  **Note**:  You can install Qlib with `python setup.py install` as well. But it is not the recommanded approach. It will skip `pip` and cause obscure problems. For example, **only** the command ``pip install .`` **can** overwrite the stable version installed by ``pip install pyqlib``, while the command ``python setup.py install`` **can't**.
 
-**Tips**: If you fail to install `Qlib` or run the examples in your environment,  comparing your steps and the [CI workflow](.github/workflows/test.yml) may help you find the problem.
+**Tips**: If you fail to install `Qlib` or run the examples in your environment,  comparing your steps and the [CI workflow](.github/workflows/test_qlib_from_source.yml) may help you find the problem.
 
 ## Data Preparation
 Load and prepare data by running the following code:
+
+### Get with module
+  ```bash
+  # get 1d data
+  python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn
+
+  # get 1min data
+  python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data_1min --region cn --interval 1min
+
+  ```
+
+### Get from source
+
   ```bash
   # get 1d data
   python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn
@@ -155,15 +200,19 @@ Load and prepare data by running the following code:
 
 This dataset is created by public data collected by [crawler scripts](scripts/data_collector/), which have been released in
 the same repository.
-Users could create the same dataset with it. 
+Users could create the same dataset with it. [Description of dataset](https://github.com/microsoft/qlib/tree/main/scripts/data_collector#description-of-dataset)
 
 *Please pay **ATTENTION** that the data is collected from [Yahoo Finance](https://finance.yahoo.com/lookup), and the data might not be perfect.
 We recommend users to prepare their own data if they have a high-quality dataset. For more information, users can refer to the [related document](https://qlib.readthedocs.io/en/latest/component/data.html#converting-csv-format-into-qlib-format)*.
 
-### Automatic update of daily frequency data(from yahoo finance)
+### Automatic update of daily frequency data (from yahoo finance)
+  > This step is *Optional* if users only want to try their models and strategies on history data.
+  > 
   > It is recommended that users update the data manually once (--trading_date 2021-05-25) and then set it to update automatically.
-
-  > For more information refer to: [yahoo collector](https://github.com/microsoft/qlib/tree/main/scripts/data_collector/yahoo#automatic-update-of-daily-frequency-datafrom-yahoo-finance)
+  >
+  > **NOTE**: Users can't incrementally  update data based on the offline data provided by Qlib(some fields are removed to reduce the data size). Users should use [yahoo collector](https://github.com/microsoft/qlib/tree/main/scripts/data_collector/yahoo#automatic-update-of-daily-frequency-datafrom-yahoo-finance) to download Yahoo data from scratch and then incrementally update it.
+  > 
+  > For more information, please refer to: [yahoo collector](https://github.com/microsoft/qlib/tree/main/scripts/data_collector/yahoo#automatic-update-of-daily-frequency-datafrom-yahoo-finance)
 
   * Automatic update of data to the "qlib" directory each trading day(Linux)
       * use *crontab*: `crontab -e`
@@ -188,7 +237,7 @@ We recommend users to prepare their own data if they have a high-quality dataset
   ```python
   import qlib
   from qlib.data import D
-  from qlib.config import REG_CN
+  from qlib.constant import REG_CN
 
   # Initialization
   mount_path = "~/.qlib/qlib_data/cn_data"  # target_dir
@@ -245,19 +294,19 @@ Qlib provides a tool named `qrun` to run the whole workflow automatically (inclu
 2. Graphical Reports Analysis: Run `examples/workflow_by_code.ipynb` with `jupyter notebook` to get graphical reports
     - Forecasting signal (model prediction) analysis
       - Cumulative Return of groups
-      ![Cumulative Return](http://fintech.msra.cn/images_v060/analysis/analysis_model_cumulative_return.png?v=0.1)
+      ![Cumulative Return](http://fintech.msra.cn/images_v070/analysis/analysis_model_cumulative_return.png?v=0.1)
       - Return distribution
-      ![long_short](http://fintech.msra.cn/images_v060/analysis/analysis_model_long_short.png?v=0.1)
+      ![long_short](http://fintech.msra.cn/images_v070/analysis/analysis_model_long_short.png?v=0.1)
       - Information Coefficient (IC)
-      ![Information Coefficient](http://fintech.msra.cn/images_v060/analysis/analysis_model_IC.png?v=0.1)        
-      ![Monthly IC](http://fintech.msra.cn/images_v060/analysis/analysis_model_monthly_IC.png?v=0.1)
-      ![IC](http://fintech.msra.cn/images_v060/analysis/analysis_model_NDQ.png?v=0.1)
+      ![Information Coefficient](http://fintech.msra.cn/images_v070/analysis/analysis_model_IC.png?v=0.1)
+      ![Monthly IC](http://fintech.msra.cn/images_v070/analysis/analysis_model_monthly_IC.png?v=0.1)
+      ![IC](http://fintech.msra.cn/images_v070/analysis/analysis_model_NDQ.png?v=0.1)
       - Auto Correlation of forecasting signal (model prediction)
-      ![Auto Correlation](http://fintech.msra.cn/images_v060/analysis/analysis_model_auto_correlation.png?v=0.1)
+      ![Auto Correlation](http://fintech.msra.cn/images_v070/analysis/analysis_model_auto_correlation.png?v=0.1)
 
     - Portfolio analysis
       - Backtest return
-      ![Report](http://fintech.msra.cn/images_v060/analysis/report.png?v=0.1)
+      ![Report](http://fintech.msra.cn/images_v070/analysis/report.png?v=0.1)
       <!-- 
       - Score IC
       ![Score IC](docs/_static/img/score_ic.png)
@@ -273,49 +322,76 @@ Qlib provides a tool named `qrun` to run the whole workflow automatically (inclu
 ## Building Customized Quant Research Workflow by Code
 The automatic workflow may not suit the research workflow of all Quant researchers. To support a flexible Quant research workflow, Qlib also provides a modularized interface to allow researchers to build their own workflow by code. [Here](examples/workflow_by_code.ipynb) is a demo for customized Quant research workflow by code.
 
+# Main Challenges & Solutions in Quant Research
+Quant investment is an very unique scenario with lots of key challenges to be solved.
+Currently, Qlib provides some solutions for several of them.
 
-# [Quant Model Zoo](examples/benchmarks)
+## Forecasting: Finding Valuable Signals/Patterns
+Accurate forecasting of the stock price trend is a very important part to construct profitable portfolios.
+However, huge amount of data with various formats in the financial market which make it challenging to build forecasting models.
+
+An increasing number of SOTA Quant research works/papers, which focus on building forecasting models to mine valuable signals/patterns in complex financial data, are released in `Qlib`
+
+
+### [Quant Model (Paper) Zoo](examples/benchmarks)
 
 Here is a list of models built on `Qlib`.
-- [GBDT based on XGBoost (Tianqi Chen, et al. KDD 2016)](qlib/contrib/model/xgboost.py)
-- [GBDT based on LightGBM (Guolin Ke, et al. NIPS 2017)](qlib/contrib/model/gbdt.py)
-- [GBDT based on Catboost (Liudmila Prokhorenkova, et al. NIPS 2018)](qlib/contrib/model/catboost_model.py)
-- [MLP based on pytorch](qlib/contrib/model/pytorch_nn.py)
-- [LSTM based on pytorch (Sepp Hochreiter, et al. Neural omputation 1997)](qlib/contrib/model/pytorch_lstm.py)
-- [GRU based on pytorch (Kyunghyun Cho, et al. 2014)](qlib/contrib/model/pytorch_gru.py)
-- [ALSTM based on pytorch (Yao Qin, et al. IJCAI 2017)](qlib/contrib/model/pytorch_alstm.py)
-- [GATs based on pytorch (Petar Velickovic, et al. 2017)](qlib/contrib/model/pytorch_gats.py)
-- [SFM based on pytorch (Liheng Zhang, et al. KDD 2017)](qlib/contrib/model/pytorch_sfm.py)
-- [TFT based on tensorflow (Bryan Lim, et al. International Journal of Forecasting 2019)](examples/benchmarks/TFT/tft.py)
-- [TabNet based on pytorch (Sercan O. Arik, et al. AAAI 2019)](qlib/contrib/model/pytorch_tabnet.py)
-- [DoubleEnsemble based on LightGBM (Chuheng Zhang, et al. ICDM 2020)](qlib/contrib/model/double_ensemble.py)
-- [TCTS based on pytorch (Xueqing Wu, et al. ICML 2021)](qlib/contrib/model/pytorch_tcts.py)
+- [GBDT based on XGBoost (Tianqi Chen, et al. KDD 2016)](examples/benchmarks/XGBoost/)
+- [GBDT based on LightGBM (Guolin Ke, et al. NIPS 2017)](examples/benchmarks/LightGBM/)
+- [GBDT based on Catboost (Liudmila Prokhorenkova, et al. NIPS 2018)](examples/benchmarks/CatBoost/)
+- [MLP based on pytorch](examples/benchmarks/MLP/)
+- [LSTM based on pytorch (Sepp Hochreiter, et al. Neural computation 1997)](examples/benchmarks/LSTM/)
+- [GRU based on pytorch (Kyunghyun Cho, et al. 2014)](examples/benchmarks/GRU/)
+- [ALSTM based on pytorch (Yao Qin, et al. IJCAI 2017)](examples/benchmarks/ALSTM)
+- [GATs based on pytorch (Petar Velickovic, et al. 2017)](examples/benchmarks/GATs/)
+- [SFM based on pytorch (Liheng Zhang, et al. KDD 2017)](examples/benchmarks/SFM/)
+- [TFT based on tensorflow (Bryan Lim, et al. International Journal of Forecasting 2019)](examples/benchmarks/TFT/)
+- [TabNet based on pytorch (Sercan O. Arik, et al. AAAI 2019)](examples/benchmarks/TabNet/)
+- [DoubleEnsemble based on LightGBM (Chuheng Zhang, et al. ICDM 2020)](examples/benchmarks/DoubleEnsemble/)
+- [TCTS based on pytorch (Xueqing Wu, et al. ICML 2021)](examples/benchmarks/TCTS/)
+- [Transformer based on pytorch (Ashish Vaswani, et al. NeurIPS 2017)](examples/benchmarks/Transformer/)
+- [Localformer based on pytorch (Juyong Jiang, et al.)](examples/benchmarks/Localformer/)
+- [TRA based on pytorch (Hengxu, Dong, et al. KDD 2021)](examples/benchmarks/TRA/)
+- [TCN based on pytorch (Shaojie Bai, et al. 2018)](examples/benchmarks/TCN/)
+- [ADARNN based on pytorch (YunTao Du, et al. 2021)](examples/benchmarks/ADARNN/)
+- [ADD based on pytorch (Hongshun Tang, et al.2020)](examples/benchmarks/ADD/)
+- [IGMTF based on pytorch (Wentao Xu, et al.2021)](examples/benchmarks/IGMTF/)
+- [HIST based on pytorch (Wentao Xu, et al.2021)](examples/benchmarks/HIST/)
 
 Your PR of new Quant models is highly welcomed.
 
 The performance of each model on the `Alpha158` and `Alpha360` dataset can be found [here](examples/benchmarks/README.md).
 
-## Run a single model
+### Run a single model
 All the models listed above are runnable with ``Qlib``. Users can find the config files we provide and some details about the model through the [benchmarks](examples/benchmarks) folder. More information can be retrieved at the model files listed above.
 
 `Qlib` provides three different ways to run a single model, users can pick the one that fits their cases best:
 - Users can use the tool `qrun` mentioned above to run a model's workflow based from a config file.
 - Users can create a `workflow_by_code` python script based on the [one](examples/workflow_by_code.py) listed in the `examples` folder.
 
-- Users can use the script [`run_all_model.py`](examples/run_all_model.py) listed in the `examples` folder to run a model. Here is an example of the specific shell command to be used: `python run_all_model.py --models=lightgbm`, where the `--models` arguments can take any number of models listed above(the available models can be found  in [benchmarks](examples/benchmarks/)). For more use cases, please refer to the file's [docstrings](examples/run_all_model.py).
+- Users can use the script [`run_all_model.py`](examples/run_all_model.py) listed in the `examples` folder to run a model. Here is an example of the specific shell command to be used: `python run_all_model.py run --models=lightgbm`, where the `--models` arguments can take any number of models listed above(the available models can be found  in [benchmarks](examples/benchmarks/)). For more use cases, please refer to the file's [docstrings](examples/run_all_model.py).
+    - **NOTE**: Each baseline has different environment dependencies, please make sure that your python version aligns with the requirements(e.g. TFT only supports Python 3.6~3.7 due to the limitation of `tensorflow==1.15.0`)
 
-## Run multiple models
-`Qlib` also provides a script [`run_all_model.py`](examples/run_all_model.py) which can run multiple models for several iterations. (**Note**: the script only support *Linux* for now. Other OS will be supported in the future. Besides, it doesn't support parrallel running the same model for multiple times as well, and this will be fixed in the future development too.)
+### Run multiple models
+`Qlib` also provides a script [`run_all_model.py`](examples/run_all_model.py) which can run multiple models for several iterations. (**Note**: the script only support *Linux* for now. Other OS will be supported in the future. Besides, it doesn't support parallel running the same model for multiple times as well, and this will be fixed in the future development too.)
 
 The script will create a unique virtual environment for each model, and delete the environments after training. Thus, only experiment results such as `IC` and `backtest` results will be generated and stored.
 
 Here is an example of running all the models for 10 iterations:
 ```python
-python run_all_model.py 10
+python run_all_model.py run 10
 ```
 
 It also provides the API to run specific models at once. For more use cases, please refer to the file's [docstrings](examples/run_all_model.py). 
 
+## [Adapting to Market Dynamics](examples/benchmarks_dynamic)
+
+Due to the non-stationary nature of the environment of the financial market, the data distribution may change in different periods, which makes the performance of models build on training data decays in the future test data.
+So adapting the forecasting models/strategies to market dynamics is very important to the model/strategies' performance.
+
+Here is a list of solutions built on `Qlib`.
+- [Rolling Retraining](examples/benchmarks_dynamic/baseline/)
+- [DDG-DA on pytorch (Wendi, et al. AAAI 2022)](examples/benchmarks_dynamic/DDG-DA/)
 
 # Quant Dataset Zoo
 Dataset plays a very important role in Quant. Here is a list of the datasets built on `Qlib`:
@@ -329,6 +405,8 @@ Dataset plays a very important role in Quant. Here is a list of the datasets bui
 Your PR to build new Quant dataset is highly welcomed.
 
 # More About Qlib
+If you want to have a quick glance at the most frequently used components of qlib, you can try notebooks [here](examples/tutorial/).
+
 The detailed documents are organized in [docs](docs/).
 [Sphinx](http://www.sphinx-doc.org) and the readthedocs theme is required to build the documentation in html formats. 
 ```bash
@@ -370,9 +448,7 @@ Such overheads greatly slow down the data loading process.
 Qlib data are stored in a compact format, which is efficient to be combined into arrays for scientific computation.
 
 # Related Reports
-- [【华泰金工林晓明团队】图神经网络选股与Qlib实践——华泰人工智能系列之四十二](https://mp.weixin.qq.com/s/w5fDB6oAv9dO6vlhf1kmhA)
 - [Guide To Qlib: Microsoft’s AI Investment Platform](https://analyticsindiamag.com/qlib/)
-- [【华泰金工林晓明团队】微软AI量化投资平台Qlib体验——华泰人工智能系列之四十](https://mp.weixin.qq.com/s/Brcd7im4NibJOJzZfMn6tQ)
 - [微软也搞AI量化平台？还是开源的！](https://mp.weixin.qq.com/s/47bP5YwxfTp2uTHjUBzJQQ)
 - [微矿Qlib：业内首个AI量化投资开源平台](https://mp.weixin.qq.com/s/vsJv7lsgjEi-ALYUz4CvtQ)
 
@@ -385,11 +461,44 @@ Qlib data are stored in a compact format, which is efficient to be combined into
 Join IM discussion groups:
 |[Gitter](https://gitter.im/Microsoft/qlib)|
 |----|
-|![image](http://fintech.msra.cn/images_v060/qrcode/gitter_qr.png)|
+|![image](http://fintech.msra.cn/images_v070/qrcode/gitter_qr.png)|
 
 # Contributing
+We appreciate all contributions and thank all the contributors!
+<a href="https://github.com/microsoft/qlib/graphs/contributors"><img src="https://contrib.rocks/image?repo=microsoft/qlib" /></a>
 
-This project welcomes contributions and suggestions.  Most contributions require you to agree to a
+Before we released Qlib as an open-source project on Github in Sep 2020, Qlib is an internal project in our group. Unfortunately, the internal commit history is not kept. A lot of members in our group have also contributed a lot to Qlib, which includes Ruihua Wang, Yinda Zhang, Haisu Yu, Shuyu Wang, Bochen Pang, and [Dong Zhou](https://github.com/evanzd/evanzd). Especially thanks to [Dong Zhou](https://github.com/evanzd/evanzd) due to his initial version of Qlib.
+
+## Guidance
+
+This project welcomes contributions and suggestions.  
+**Here are some 
+[code standards and development guidance](docs/developer/code_standard_and_dev_guide.rst) for submiting a pull request.**
+
+Making contributions is not a hard thing. Solving an issue(maybe just answering a question raised in [issues list](https://github.com/microsoft/qlib/issues) or [gitter](https://gitter.im/Microsoft/qlib)), fixing/issuing a bug, improving the documents and even fixing a typo are important contributions to Qlib.
+
+For example, if you want to contribute to Qlib's document/code, you can follow the steps in the figure below.
+<p align="center">
+  <img src="https://github.com/demon143/qlib/blob/main/docs/_static/img/change%20doc.gif" />
+</p>
+
+If you don't know how to start to contribute, you can refer to the following examples.
+| Type | Examples |
+| -- | -- |
+| Solving issues | [Answer a question](https://github.com/microsoft/qlib/issues/749);  [issuing](https://github.com/microsoft/qlib/issues/765) or [fixing](https://github.com/microsoft/qlib/pull/792) a bug |
+| Docs | [Improve docs quality](https://github.com/microsoft/qlib/pull/797/files) ;  [Fix a typo](https://github.com/microsoft/qlib/pull/774) | 
+| Feature |  Implement a [requested feature](https://github.com/microsoft/qlib/projects) like [this](https://github.com/microsoft/qlib/pull/754); [Refactor interfaces](https://github.com/microsoft/qlib/pull/539/files) |
+| Dataset | [Add a dataset](https://github.com/microsoft/qlib/pull/733) | 
+| Models |  [Implement a new model](https://github.com/microsoft/qlib/pull/689), [some instructions to contribute models](https://github.com/microsoft/qlib/tree/main/examples/benchmarks#contributing) |
+
+[Good first issues](https://github.com/microsoft/qlib/labels/good%20first%20issue) are labelled to indicate that they are easy to start your contributions.
+
+You can find some impefect implementation in Qlib by  `rg 'TODO|FIXME' qlib`
+ 
+If you would like to become one of Qlib's maintainers to contribute more (e.g. help merge PR, triage issues), please contact us by email([qlib@microsoft.com](mailto:qlib@microsoft.com)).  We are glad to help to upgrade your permission.
+
+## Licence
+Most contributions require you to agree to a
 Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
 the right to use your contribution. For details, visit https://cla.opensource.microsoft.com.
 

docs/FAQ/FAQ.rst

@@ -3,7 +3,7 @@ Qlib FAQ
 ############
 
 Qlib Frequently Asked Questions
-================================
+===============================
 .. contents::
     :depth: 1
     :local:
@@ -13,7 +13,7 @@ Qlib Frequently Asked Questions
 
 
 1. RuntimeError: An attempt has been made to start a new process before the current process has finished its bootstrapping phase...
-------------------------------------------------------------------------------------------------------------------------------------
+-----------------------------------------------------------------------------------------------------------------------------------
 
 .. code-block:: console
 
@@ -52,7 +52,7 @@ This is caused by the limitation of multiprocessing under windows OS. Please ref
 
 
 2. qlib.data.cache.QlibCacheException: It sees the key(...) of the redis lock has existed in your redis db now.
------------------------------------------------------------------------------------------------------------------
+---------------------------------------------------------------------------------------------------------------
 
 It sees the key of the redis lock has existed in your redis db now. You can use the following command to clear your redis keys and rerun your commands
 
@@ -72,7 +72,7 @@ If the issue is not resolved, use ``keys *`` to find if multiple keys exist. If
 Also, feel free to post a new issue in our GitHub repository. We always check each issue carefully and try our best to solve them.
 
 3. ModuleNotFoundError: No module named 'qlib.data._libs.rolling'
-------------------------------------------------------------------------------------------------------------------------------------
+-----------------------------------------------------------------
 
 .. code-block:: python
 
@@ -97,4 +97,57 @@ Also, feel free to post a new issue in our GitHub repository. We always check ea
 
         python setup.py build_ext --inplace
 
-- If the error occurs when importing ``qlib`` package with command ``python`` , users need to change the running directory to ensure that the script does not run in the project directory.
+- If the error occurs when importing ``qlib`` package with command ``python`` , users need to change the running directory to ensure that the script does not run in the project directory.
+
+
+4. BadNamespaceError: / is not a connected namespace
+----------------------------------------------------
+
+.. code-block:: python
+
+      File "qlib_online.py", line 35, in <module>
+        cal = D.calendar()
+      File "e:\code\python\microsoft\qlib_latest\qlib\qlib\data\data.py", line 973, in calendar
+        return Cal.calendar(start_time, end_time, freq, future=future)
+      File "e:\code\python\microsoft\qlib_latest\qlib\qlib\data\data.py", line 798, in calendar
+        self.conn.send_request(
+      File "e:\code\python\microsoft\qlib_latest\qlib\qlib\data\client.py", line 101, in send_request
+        self.sio.emit(request_type + "_request", request_content)
+      File "G:\apps\miniconda\envs\qlib\lib\site-packages\python_socketio-5.3.0-py3.8.egg\socketio\client.py", line 369, in emit
+        raise exceptions.BadNamespaceError(
+      BadNamespaceError: / is not a connected namespace.
+
+- The version of ``python-socketio`` in qlib needs to be the same as the version of ``python-socketio`` in qlib-server:
+
+    .. code-block:: bash
+
+        pip install -U python-socketio==<qlib-server python-socketio version>
+
+
+5. TypeError: send() got an unexpected keyword argument 'binary'
+----------------------------------------------------------------
+
+.. code-block:: python
+
+      File "qlib_online.py", line 35, in <module>
+        cal = D.calendar()
+      File "e:\code\python\microsoft\qlib_latest\qlib\qlib\data\data.py", line 973, in calendar
+        return Cal.calendar(start_time, end_time, freq, future=future)
+      File "e:\code\python\microsoft\qlib_latest\qlib\qlib\data\data.py", line 798, in calendar
+        self.conn.send_request(
+      File "e:\code\python\microsoft\qlib_latest\qlib\qlib\data\client.py", line 101, in send_request
+        self.sio.emit(request_type + "_request", request_content)
+      File "G:\apps\miniconda\envs\qlib\lib\site-packages\socketio\client.py", line 263, in emit
+        self._send_packet(packet.Packet(packet.EVENT, namespace=namespace,
+      File "G:\apps\miniconda\envs\qlib\lib\site-packages\socketio\client.py", line 339, in _send_packet
+        self.eio.send(ep, binary=binary)
+      TypeError: send() got an unexpected keyword argument 'binary'
+
+
+- The ``python-engineio`` version needs to be compatible with the ``python-socketio`` version, reference: https://github.com/miguelgrinberg/python-socketio#version-compatibility
+
+    .. code-block:: bash
+
+        pip install -U python-engineio==<compatible python-socketio version>
+        # or
+        pip install -U python-socketio==3.1.2 python-engineio==3.13.2

docs/advanced/PIT.rst

@@ -0,0 +1,136 @@
+.. _pit:
+
+============================
+(P)oint-(I)n-(T)ime Database
+============================
+.. currentmodule:: qlib
+
+
+Introduction
+------------
+Point-in-time data is a very important consideration when performing any sort of historical market analysis.
+
+For example, let’s say we are backtesting a trading strategy and we are using the past five years of historical data as our input.
+Our model is assumed to trade once a day, at the market close, and we’ll say we are calculating the trading signal for 1 January 2020 in our backtest. At that point, we should only have data for 1 January 2020, 31 December 2019, 30 December 2019 etc.
+
+In financial data (especially financial reports), the same piece of data may be amended for multiple times overtime.  If we only use the latest version for historical backtesting, data leakage will happen.
+Point-in-time database is designed for solving this problem to make sure user get the right version of data at any historical timestamp. It will keep the performance of online trading and historical backtesting the same.
+
+
+
+Data Preparation
+----------------
+
+Qlib provides a crawler to help users to download financial data and then a converter to dump the data in Qlib format.
+Please follow `scripts/data_collector/pit/README.md <https://github.com/microsoft/qlib/tree/main/scripts/data_collector/pit/>`_ to download and convert data.
+Besides, you can find some additional usage examples there.
+
+
+File-based design for PIT data
+------------------------------
+
+Qlib provides a file-based storage for PIT data.
+
+For each feature, it contains 4 columns, i.e. date, period, value, _next.
+Each row corresponds to a statement.
+
+The meaning of each feature with filename like `XXX_a.data`:
+
+- `date`: the statement's date of publication.
+- `period`: the period of the statement. (e.g. it will be quarterly frequency in most of the markets)
+    - If it is an annual period, it will be an integer corresponding to the year
+    - If it is an quarterly  periods, it will be an integer like `<year><index of quarter>`.  The last two decimal digits represents the index of quarter. Others represent the year.
+- `value`: the described value
+- `_next`: the byte index of the next occurance of the field.
+
+Besides the feature data, an index `XXX_a.index` is included to speed up the querying performance
+
+The statements are soted by the `date` in ascending order from the beginning of the file.
+
+.. code-block:: python
+
+    # the data format from XXXX.data
+    array([(20070428, 200701, 0.090219  , 4294967295),
+           (20070817, 200702, 0.13933   , 4294967295),
+           (20071023, 200703, 0.24586301, 4294967295),
+           (20080301, 200704, 0.3479    ,         80),
+           (20080313, 200704, 0.395989  , 4294967295),
+           (20080422, 200801, 0.100724  , 4294967295),
+           (20080828, 200802, 0.24996801, 4294967295),
+           (20081027, 200803, 0.33412001, 4294967295),
+           (20090325, 200804, 0.39011699, 4294967295),
+           (20090421, 200901, 0.102675  , 4294967295),
+           (20090807, 200902, 0.230712  , 4294967295),
+           (20091024, 200903, 0.30072999, 4294967295),
+           (20100402, 200904, 0.33546099, 4294967295),
+           (20100426, 201001, 0.083825  , 4294967295),
+           (20100812, 201002, 0.200545  , 4294967295),
+           (20101029, 201003, 0.260986  , 4294967295),
+           (20110321, 201004, 0.30739301, 4294967295),
+           (20110423, 201101, 0.097411  , 4294967295),
+           (20110831, 201102, 0.24825101, 4294967295),
+           (20111018, 201103, 0.318919  , 4294967295),
+           (20120323, 201104, 0.4039    ,        420),
+           (20120411, 201104, 0.403925  , 4294967295),
+           (20120426, 201201, 0.112148  , 4294967295),
+           (20120810, 201202, 0.26484701, 4294967295),
+           (20121026, 201203, 0.370487  , 4294967295),
+           (20130329, 201204, 0.45004699, 4294967295),
+           (20130418, 201301, 0.099958  , 4294967295),
+           (20130831, 201302, 0.21044201, 4294967295),
+           (20131016, 201303, 0.30454299, 4294967295),
+           (20140325, 201304, 0.394328  , 4294967295),
+           (20140425, 201401, 0.083217  , 4294967295),
+           (20140829, 201402, 0.16450299, 4294967295),
+           (20141030, 201403, 0.23408499, 4294967295),
+           (20150421, 201404, 0.319612  , 4294967295),
+           (20150421, 201501, 0.078494  , 4294967295),
+           (20150828, 201502, 0.137504  , 4294967295),
+           (20151023, 201503, 0.201709  , 4294967295),
+           (20160324, 201504, 0.26420501, 4294967295),
+           (20160421, 201601, 0.073664  , 4294967295),
+           (20160827, 201602, 0.136576  , 4294967295),
+           (20161029, 201603, 0.188062  , 4294967295),
+           (20170415, 201604, 0.244385  , 4294967295),
+           (20170425, 201701, 0.080614  , 4294967295),
+           (20170728, 201702, 0.15151   , 4294967295),
+           (20171026, 201703, 0.25416601, 4294967295),
+           (20180328, 201704, 0.32954201, 4294967295),
+           (20180428, 201801, 0.088887  , 4294967295),
+           (20180802, 201802, 0.170563  , 4294967295),
+           (20181029, 201803, 0.25522   , 4294967295),
+           (20190329, 201804, 0.34464401, 4294967295),
+           (20190425, 201901, 0.094737  , 4294967295),
+           (20190713, 201902, 0.        ,       1040),
+           (20190718, 201902, 0.175322  , 4294967295),
+           (20191016, 201903, 0.25581899, 4294967295)],
+          dtype=[('date', '<u4'), ('period', '<u4'), ('value', '<f8'), ('_next', '<u4')])
+    # - each row contains 20 byte
+
+
+    # The data format from XXXX.index.  It consists of two parts
+    # 1) the start index of the data. So the first part of the info will be like
+    2007
+    # 2) the remain index data will be like information below
+    #    - The data indicate the **byte index** of first data update of a period.
+    #    - e.g. Because the info at both byte 80 and 100 corresponds to 200704. The byte index of first occurance (i.e. 100) is recorded in the data.
+    array([         0,         20,         40,         60,        100,
+                  120,        140,        160,        180,        200,
+                  220,        240,        260,        280,        300,
+                  320,        340,        360,        380,        400,
+                  440,        460,        480,        500,        520,
+                  540,        560,        580,        600,        620,
+                  640,        660,        680,        700,        720,
+                  740,        760,        780,        800,        820,
+                  840,        860,        880,        900,        920,
+                  940,        960,        980,       1000,       1020,
+                 1060, 4294967295], dtype=uint32)
+
+
+
+
+Known limitations:
+
+- Currently, the PIT database is designed for quarterly or annually factors, which can handle fundamental data of financial reports in most markets.
+- Qlib leverage the file name to identify the type of the data. File with name like `XXX_q.data` corresponds to quarterly data. File with name like `XXX_a.data` corresponds to annual data.
+- The caclulation of PIT is not performed in the optimal way. There is great potential to boost the performance of PIT data calcuation.

docs/advanced/alpha.rst

@@ -1,12 +1,12 @@
 .. _alpha:
 
-===========================
-Building Formulaic Alphas 
-===========================
+=========================
+Building Formulaic Alphas
+=========================
 .. currentmodule:: qlib
 
 Introduction
-===================
+============
 
 In quantitative trading practice, designing novel factors that can explain and predict future asset returns are of vital importance to the profitability of a strategy. Such factors are usually called alpha factors, or alphas in short.
 
@@ -15,28 +15,28 @@ A formulaic alpha, as the name suggests, is a kind of alpha that can be presente
 
 
 Building Formulaic Alphas in ``Qlib``
-======================================
+=====================================
 
 In ``Qlib``, users can easily build formulaic alphas.
 
 Example
------------------
+-------
 
 `MACD`, short for moving average convergence/divergence, is a formulaic alpha used in technical analysis of stock prices. It is designed to reveal changes in the strength, direction, momentum, and duration of a trend in a stock's price.
 
 `MACD` can be presented as the following formula:
 
-.. math:: 
+.. math::
 
     MACD = 2\times (DIF-DEA)
 
 .. note::
 
     `DIF` means Differential value, which is 12-period EMA minus 26-period EMA.
-    
+
     .. math::
 
-        DIF = \frac{EMA(CLOSE, 12) - EMA(CLOSE, 26)}{CLOSE} 
+        DIF = \frac{EMA(CLOSE, 12) - EMA(CLOSE, 26)}{CLOSE}
 
     `DEA`means a 9-period EMA of the DIF.
 
@@ -65,7 +65,7 @@ Users can use ``Data Handler`` to build formulaic alphas `MACD` in qlib:
     >> print(df)
                             feature     label
                                MACD     LABEL
-    datetime   instrument                    
+    datetime   instrument
     2010-01-04 SH600000   -0.011547 -0.019672
                SH600004    0.002745 -0.014721
                SH600006    0.010133  0.002911
@@ -79,7 +79,7 @@ Users can use ``Data Handler`` to build formulaic alphas `MACD` in qlib:
                SZ300315   -0.030557  0.012455
 
 Reference
-===========
+=========
 
 To learn more about ``Data Loader``, please refer to `Data Loader <../component/data.html#data-loader>`_
 

docs/advanced/serial.rst

@@ -1,26 +1,26 @@
 .. _serial:
 
-=================================
+=============
 Serialization
-=================================
+=============
 .. currentmodule:: qlib
 
 Introduction
-===================
-``Qlib`` supports dumping the state of ``DataHandler``, ``DataSet``, ``Processor`` and ``Model``, etc. into a disk and reloading them. 
+============
+``Qlib`` supports dumping the state of ``DataHandler``, ``DataSet``, ``Processor`` and ``Model``, etc. into a disk and reloading them.
 
 Serializable Class
-========================
+==================
 
-``Qlib`` provides a base class ``qlib.utils.serial.Serializable``, whose state can be dumped into or loaded from disk in `pickle` format. 
+``Qlib`` provides a base class ``qlib.utils.serial.Serializable``, whose state can be dumped into or loaded from disk in `pickle` format.
 When users dump the state of a ``Serializable`` instance, the attributes of the instance whose name **does not** start with `_` will be saved on the disk.
 However, users can use ``config`` method or override ``default_dump_all`` attribute to prevent this feature.
 
 Users can also override ``pickle_backend`` attribute to choose a pickle backend. The supported value is "pickle" (default and common) and "dill" (dump more things such as function, more information in `here <https://pypi.org/project/dill/>`_).
 
 Example
-==========================
-``Qlib``'s serializable class includes  ``DataHandler``, ``DataSet``, ``Processor`` and ``Model``, etc., which are subclass of  ``qlib.utils.serial.Serializable``. 
+=======
+``Qlib``'s serializable class includes  ``DataHandler``, ``DataSet``, ``Processor`` and ``Model``, etc., which are subclass of  ``qlib.utils.serial.Serializable``.
 Specifically, ``qlib.data.dataset.DatasetH`` is one of them. Users can serialize ``DatasetH`` as follows.
 
 .. code-block:: Python
@@ -33,7 +33,7 @@ Specifically, ``qlib.data.dataset.DatasetH`` is one of them. Users can serialize
         dataset = pickle.load(file_dataset)
 
 .. note::
-    Only state of ``DatasetH`` should be saved on the disk, such as some `mean` and `variance` used for data normalization, etc. 
+    Only state of ``DatasetH`` should be saved on the disk, such as some `mean` and `variance` used for data normalization, etc.
 
     After reloading the ``DatasetH``, users need to reinitialize it. It means that users can reset some states of ``DatasetH`` or ``QlibDataHandler`` such as `instruments`, `start_time`, `end_time` and `segments`, etc.,  and generate new data according to the states (data is not state and should not be saved on the disk).
 
@@ -41,5 +41,5 @@ A more detailed example is in this `link <https://github.com/microsoft/qlib/tree
 
 
 API
-===================
+===
 Please refer to `Serializable API <../reference/api.html#module-qlib.utils.serial.Serializable>`_.

docs/advanced/server.rst

@@ -1,15 +1,15 @@
 .. _server:
 
-=================================
+=============================
 ``Online`` & ``Offline`` mode
-=================================
+=============================
 .. currentmodule:: qlib
 
 
 Introduction
-=============
+============
 
-``Qlib`` supports ``Online`` mode and ``Offline`` mode. Only the ``Offline`` mode is introduced in this document. 
+``Qlib`` supports ``Online`` mode and ``Offline`` mode. Only the ``Offline`` mode is introduced in this document.
 
 The ``Online`` mode is designed to solve the following problems:
 
@@ -18,12 +18,12 @@ The ``Online`` mode is designed to solve the following problems:
 - Make the data can be accessed in a remote way.
 
 Qlib-Server
-===============
+===========
 
-``Qlib-Server`` is the assorted server system for ``Qlib``, which utilizes ``Qlib`` for basic calculations and provides extensive server system and cache mechanism. With QLibServer, the data provided for ``Qlib`` can be managed in a centralized manner. With ``Qlib-Server``, users can use ``Qlib`` in ``Online`` mode. 
+``Qlib-Server`` is the assorted server system for ``Qlib``, which utilizes ``Qlib`` for basic calculations and provides extensive server system and cache mechanism. With QLibServer, the data provided for ``Qlib`` can be managed in a centralized manner. With ``Qlib-Server``, users can use ``Qlib`` in ``Online`` mode.
 
 
 
 Reference
-=================
-If users are interested in ``Qlib-Server`` and ``Online`` mode, please refer to `Qlib-Server Project <https://github.com/microsoft/qlib-server>`_ and `Qlib-Server Document <https://qlib-server.readthedocs.io/en/latest/>`_.
+=========
+If users are interested in ``Qlib-Server`` and ``Online`` mode, please refer to `Qlib-Server Project <https://github.com/microsoft/qlib-server>`_ and `Qlib-Server Document <https://qlib-server.readthedocs.io/en/latest/>`_.

docs/advanced/task_management.rst

@@ -1,17 +1,20 @@
 .. _task_management:
 
-=================================
+===============
 Task Management
-=================================
+===============
 .. currentmodule:: qlib
 
 
 Introduction
-=============
+============
 
 The `Workflow <../component/introduction.html>`_ part introduces how to run research workflow in a loosely-coupled way. But it can only execute one ``task`` when you use ``qrun``.
 To automatically generate and execute different tasks, ``Task Management`` provides a whole process including `Task Generating`_, `Task Storing`_, `Task Training`_ and `Task Collecting`_. 
-With this module, users can run their ``task`` automatically at different periods, in different losses, or even by different models.
+With this module, users can run their ``task`` automatically at different periods, in different losses, or even by different models.The processes of task generation, model training and combine and collect data are shown in the following figure.
+
+.. image:: ../_static/img/Task-Gen-Recorder-Collector.svg
+    :align: center
 
 This whole process can be used in `Online Serving <../component/online.html>`_.
 
@@ -33,7 +36,7 @@ Here is the base class of ``TaskGen``:
 This class allows users to verify the effect of data from different periods on the model in one experiment. More information is `here <../reference/api.html#TaskGen>`_.
 
 Task Storing
-===============
+============
 To achieve higher efficiency and the possibility of cluster operation, ``Task Manager`` will store all tasks in `MongoDB <https://www.mongodb.com/>`_.
 ``TaskManager`` can fetch undone tasks automatically and manage the lifecycle of a set of tasks with error handling.
 Users **MUST** finish the configuration of `MongoDB <https://www.mongodb.com/>`_ when using this module.
@@ -54,7 +57,7 @@ Users need to provide the MongoDB URL and database name for using ``TaskManager`
 More information of ``Task Manager`` can be found in `here <../reference/api.html#TaskManager>`_.
 
 Task Training
-===============
+=============
 After generating and storing those ``task``, it's time to run the ``task`` which is in the *WAITING* status.
 ``Qlib`` provides a method called ``run_task`` to run those ``task`` in task pool, however, users can also customize how tasks are executed.
 An easy way to get the ``task_func`` is using ``qlib.model.trainer.task_train`` directly.
@@ -74,6 +77,8 @@ If you do not want to use ``Task Manager`` to manage tasks, then use TrainerR to
 
 Task Collecting
 ===============
+Before collecting model training results, you need to use the ``qlib.init`` to specify the path of mlruns.
+
 To collect the results of ``task`` after training, ``Qlib`` provides `Collector <../reference/api.html#Collector>`_, `Group <../reference/api.html#Group>`_ and `Ensemble <../reference/api.html#Ensemble>`_ to collect the results in a readable, expandable and loosely-coupled way.
 
 `Collector <../reference/api.html#Collector>`_ can collect objects from everywhere and process them such as merging, grouping, averaging and so on. It has 2 step action including ``collect`` (collect anything in a dict) and ``process_collect`` (process collected dict).
@@ -82,8 +87,10 @@ To collect the results of ``task`` after training, ``Qlib`` provides `Collector
 For example: {(A,B,C1): object, (A,B,C2): object} ---``group``---> {(A,B): {C1: object, C2: object}} ---``reduce``---> {(A,B): object}
 
 `Ensemble <../reference/api.html#Ensemble>`_ can merge the objects in an ensemble. 
-For example: {C1: object, C2: object} ---``Ensemble``---> object
+For example: {C1: object, C2: object} ---``Ensemble``---> object.
+You can set the ensembles you want in the ``Collector``'s process_list.
+Common ensembles include ``AverageEnsemble`` and ``RollingEnsemble``. Average ensemble is used to ensemble the results of different models in the same time period. Rollingensemble is used to ensemble the results of different models in the same time period
 
 So the hierarchy is ``Collector``'s second step corresponds to ``Group``. And ``Group``'s second step correspond to ``Ensemble``.
 
-For more information, please see `Collector <../reference/api.html#Collector>`_, `Group <../reference/api.html#Group>`_ and `Ensemble <../reference/api.html#Ensemble>`_, or the `example <https://github.com/microsoft/qlib/tree/main/examples/model_rolling/task_manager_rolling.py>`_.
+For more information, please see `Collector <../reference/api.html#Collector>`_, `Group <../reference/api.html#Group>`_ and `Ensemble <../reference/api.html#Ensemble>`_, or the `example <https://github.com/microsoft/qlib/tree/main/examples/model_rolling/task_manager_rolling.py>`_.

docs/changelog/changelog.rst

@@ -1,2 +1 @@
 .. include:: ../../CHANGES.rst
-	     

docs/component/backtest.rst

@@ -1,114 +0,0 @@
-.. _backtest:
-
-============================================
-Intraday Trading: Model&Strategy Testing
-============================================
-.. currentmodule:: qlib
-
-Introduction
-===================
-
-``Intraday Trading`` is designed to test models and strategies, which help users to check the performance of a custom model/strategy.
-
-
-.. note::
-
-    ``Intraday Trading`` uses ``Order Executor`` to trade and execute orders output by ``Portfolio Strategy``. ``Order Executor`` is a component in `Qlib Framework <../introduction/introduction.html#framework>`_, which can execute orders. ``VWAP Executor`` and ``Close Executor`` is supported by ``Qlib`` now. In the future, ``Qlib`` will support ``HighFreq Executor`` also. 
-
-
-
-Example
-===========================
-
-Users need to generate a `prediction score`(a pandas DataFrame) with MultiIndex<instrument, datetime> and a `score` column. And users need to assign a strategy used in backtest, if strategy is not assigned,
-a `TopkDropoutStrategy` strategy with `(topk=50, n_drop=5, risk_degree=0.95, limit_threshold=0.0095)` will be used.
-If ``Strategy`` module is not users' interested part, `TopkDropoutStrategy` is enough. 
-
-The simple example of the default strategy is as follows.
-
-.. code-block:: python
-
-    from qlib.contrib.evaluate import backtest
-    # pred_score is the prediction score
-    report, positions = backtest(pred_score, topk=50, n_drop=0.5, verbose=False, limit_threshold=0.0095)
-
-To know more about backtesting with a specific ``Strategy``, please refer to `Portfolio Strategy <strategy.html>`_.
-
-To know more about the prediction score `pred_score` output by ``Forecast Model``, please refer to `Forecast Model: Model Training & Prediction <model.html>`_.
-
-Prediction Score
------------------
-
-The `prediction score` is a pandas DataFrame. Its index is <datetime(pd.Timestamp), instrument(str)> and it must
-contains a `score` column.
-
-A prediction sample is shown as follows.
-
-.. code-block:: python
-
-      datetime instrument     score
-    2019-01-04   SH600000 -0.505488
-    2019-01-04   SZ002531 -0.320391
-    2019-01-04   SZ000999  0.583808
-    2019-01-04   SZ300569  0.819628
-    2019-01-04   SZ001696 -0.137140
-                 ...            ...
-    2019-04-30   SZ000996 -1.027618
-    2019-04-30   SH603127  0.225677
-    2019-04-30   SH603126  0.462443
-    2019-04-30   SH603133 -0.302460
-    2019-04-30   SZ300760 -0.126383
-
-``Forecast Model`` module can make predictions, please refer to `Forecast Model: Model Training & Prediction <model.html>`_.
-
-Backtest Result
-------------------
-
-The backtest results are in the following form:
-
-.. code-block:: python
-
-                                                      risk
-    excess_return_without_cost mean               0.000605
-                               std                0.005481
-                               annualized_return  0.152373
-                               information_ratio  1.751319
-                               max_drawdown      -0.059055
-    excess_return_with_cost    mean               0.000410
-                               std                0.005478
-                               annualized_return  0.103265
-                               information_ratio  1.187411
-                               max_drawdown      -0.075024
-
-
-
-- `excess_return_without_cost`
-    - `mean`
-        Mean value of the `CAR` (cumulative abnormal return) without cost
-    - `std`
-        The `Standard Deviation` of `CAR` (cumulative abnormal return) without cost.
-    - `annualized_return`
-        The `Annualized Rate` of `CAR` (cumulative abnormal return) without cost.
-    - `information_ratio`
-        The `Information Ratio` without cost. please refer to `Information Ratio – IR <https://www.investopedia.com/terms/i/informationratio.asp>`_.
-    - `max_drawdown`
-        The `Maximum Drawdown` of `CAR` (cumulative abnormal return) without cost, please refer to `Maximum Drawdown (MDD) <https://www.investopedia.com/terms/m/maximum-drawdown-mdd.asp>`_.
-
-- `excess_return_with_cost`
-    - `mean`
-        Mean value of the `CAR` (cumulative abnormal return) series with cost
-    - `std`
-        The `Standard Deviation` of `CAR` (cumulative abnormal return) series with cost.
-    - `annualized_return`
-        The `Annualized Rate` of `CAR` (cumulative abnormal return) with cost.
-    - `information_ratio`
-        The `Information Ratio` with cost. please refer to `Information Ratio – IR <https://www.investopedia.com/terms/i/informationratio.asp>`_.
-    - `max_drawdown`
-        The `Maximum Drawdown` of `CAR` (cumulative abnormal return) with cost, please refer to `Maximum Drawdown (MDD) <https://www.investopedia.com/terms/m/maximum-drawdown-mdd.asp>`_.
-
-
-
-Reference
-==============
-
-To know more about ``Intraday Trading``, please refer to `Intraday Trading <../reference/api.html#module-qlib.contrib.evaluate>`_.

docs/component/data.rst

@@ -1,13 +1,13 @@
 .. _data:
 
-================================
+==================================
 Data Layer: Data Framework & Usage
-================================
+==================================
 
 Introduction
-============================
+============
 
-``Data Layer`` provides user-friendly APIs to manage and retrieve data. It provides high-performance data infrastructure. 
+``Data Layer`` provides user-friendly APIs to manage and retrieve data. It provides high-performance data infrastructure.
 
 It is designed for quantitative investment. For example, users could build formulaic alphas with ``Data Layer`` easily. Please refer to `Building Formulaic Alphas <../advanced/alpha.html>`_ for more details.
 
@@ -21,12 +21,18 @@ The introduction of ``Data Layer`` includes the following parts.
 - Cache
 - Data and Cache File Structure
 
+Here is a typical example of Qlib data workflow
+
+- Users download data and converting data into Qlib format(with filename suffix `.bin`).  In this step, typically only some basic data are stored on disk(such as OHLCV).
+- Creating some basic features based on Qlib's expression Engine(e.g. "Ref($close, 60) / $close", the return of last 60 trading days). Supported operators in the expression engine can be found `here <https://github.com/microsoft/qlib/blob/main/qlib/data/ops.py>`_. This step is typically implemented in Qlib's `Data Loader <https://qlib.readthedocs.io/en/latest/component/data.html#data-loader>`_ which is a component of `Data Handler <https://qlib.readthedocs.io/en/latest/component/data.html#data-handler>`_ .
+- If users require more complicated data processing (e.g. data normalization),  `Data Handler <https://qlib.readthedocs.io/en/latest/component/data.html#data-handler>`_ support user-customized processors to process data(some predefined processors can be found `here <https://github.com/microsoft/qlib/blob/main/qlib/data/dataset/processor.py>`_).  The processors are different from operators in expression engine. It is designed for some complicated data processing methods which is hard to supported in operators in expression engine.
+- At last, `Dataset <https://qlib.readthedocs.io/en/latest/component/data.html#dataset>`_ is responsible to prepare model-specific dataset from the processed data of Data Handler
 
 Data Preparation
-============================
+================
 
 Qlib Format Data
-------------------
+----------------
 
 We've specially designed a data structure to manage financial data, please refer to the `File storage design section in Qlib paper <https://arxiv.org/abs/2009.11189>`_ for detailed information.
 Such data will be stored with filename suffix `.bin` (We'll call them `.bin` file, `.bin` format, or qlib format). `.bin` file is designed for scientific computing on finance data.
@@ -44,8 +50,15 @@ Alpha158                  √                  √
 Also, ``Qlib`` provides a high-frequency dataset. Users can run a high-frequency dataset example through this `link <https://github.com/microsoft/qlib/tree/main/examples/highfreq>`_.
 
 Qlib Format Dataset
---------------------
-``Qlib`` has provided an off-the-shelf dataset in `.bin` format, users could use the script ``scripts/get_data.py`` to download the China-Stock dataset as follows.
+-------------------
+``Qlib`` has provided an off-the-shelf dataset in `.bin` format, users could use the script ``scripts/get_data.py`` to download the China-Stock dataset as follows. User can also use numpy to load `.bin` file to validate data.
+The price volume data look different from the actual dealling price because of they are **adjusted** (`adjusted price <https://www.investopedia.com/terms/a/adjusted_closing_price.asp>`_).  And then you may find that the adjusted price may be different from different data sources. This is because different data sources may vary in the way of adjusting prices. Qlib normalize the price on first trading day of each stock to 1 when adjusting them.
+Users can leverage `$factor` to get the original trading price (e.g. `$close / $factor` to get the original close price).
+
+Here are some discussions about the price adjusting of Qlib. 
+
+- https://github.com/microsoft/qlib/issues/991#issuecomment-1075252402
+
 
 .. code-block:: bash
 
@@ -96,7 +109,7 @@ Automatic update of daily frequency data
 
 
 Converting CSV Format into Qlib Format
--------------------------------------------
+--------------------------------------
 
 ``Qlib`` has provided the script ``scripts/dump_bin.py`` to convert **any** data in CSV format into `.bin` files (``Qlib`` format) as long as they are in the correct format.
 
@@ -118,16 +131,16 @@ Users can also provide their own data in CSV format. However, the CSV data **mus
 - CSV file is named after a specific stock *or* the CSV file includes a column of the stock name
 
     - Name the CSV file after a stock: `SH600000.csv`, `AAPL.csv` (not case sensitive).
-    
+
     - CSV file includes a column of the stock name. User **must** specify the column name when dumping the data. Here is an example:
 
         .. code-block:: bash
 
             python scripts/dump_bin.py dump_all ... --symbol_field_name symbol
-        
+
         where the data are in the following format:
 
-        .. code-block:: 
+        .. code-block::
 
             symbol,close
             SH600000,120
@@ -137,10 +150,10 @@ Users can also provide their own data in CSV format. However, the CSV data **mus
     .. code-block:: bash
 
         python scripts/dump_bin.py dump_all ... --date_field_name date
-    
+
     where the data are in the following format:
 
-    .. code-block:: 
+    .. code-block::
 
         symbol,date,close,open,volume
         SH600000,2020-11-01,120,121,12300000
@@ -164,7 +177,7 @@ After conversion, users can find their Qlib format data in the directory `~/.qli
 .. note::
 
     The arguments of `--include_fields` should correspond with the column names of CSV files. The columns names of dataset provided by ``Qlib`` should include open, close, high, low, volume and factor at least.
-    
+
     - `open`
         The adjusted opening price
     - `close`
@@ -178,10 +191,11 @@ After conversion, users can find their Qlib format data in the directory `~/.qli
     - `factor`
         The Restoration factor. Normally, ``factor = adjusted_price / original_price``, `adjusted price` reference: `split adjusted <https://www.investopedia.com/terms/s/splitadjusted.asp>`_
 
-    In the convention of `Qlib` data processing, `open, close, high, low, volume, money and factor` will be set to NaN if the stock is suspended. 
+    In the convention of `Qlib` data processing, `open, close, high, low, volume, money and factor` will be set to NaN if the stock is suspended.
+    If you want to use your own alpha-factor which can't be calculate by OCHLV, like PE, EPS and so on, you could add it to the CSV files with OHCLV together and then dump it to the Qlib format data.
 
 Stock Pool (Market)
---------------------------------
+-------------------
 
 ``Qlib`` defines `stock pool <https://github.com/microsoft/qlib/blob/main/examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml#L4>`_ as stock list and their date ranges. Predefined stock pools (e.g. csi300) may be imported as follows.
 
@@ -191,7 +205,7 @@ Stock Pool (Market)
 
 
 Multiple Stock Modes
---------------------------------
+--------------------
 
 ``Qlib`` now provides two different stock modes for users: China-Stock Mode & US-Stock Mode. Here are some different settings of these two modes:
 
@@ -209,23 +223,23 @@ The `trade unit` defines the unit number of stocks can be used in a trade, and t
     - Download china-stock in qlib format, please refer to section `Qlib Format Dataset <#qlib-format-dataset>`_.
     - Initialize ``Qlib`` in china-stock mode
         Supposed that users download their Qlib format data in the directory ``~/.qlib/qlib_data/cn_data``. Users only need to initialize ``Qlib`` as follows.
-        
+
         .. code-block:: python
 
-            from qlib.config import REG_CN
+            from qlib.constant import REG_CN
             qlib.init(provider_uri='~/.qlib/qlib_data/cn_data', region=REG_CN)
-        
+
 
 - If users use ``Qlib`` in US-stock mode, US-stock data is required. ``Qlib`` also provides a script to download US-stock data. Users can use ``Qlib`` in US-stock mode according to the following steps:
     - Download us-stock in qlib format, please refer to section `Qlib Format Dataset <#qlib-format-dataset>`_.
     - Initialize ``Qlib`` in US-stock mode
         Supposed that users prepare their Qlib format data in the directory ``~/.qlib/qlib_data/us_data``. Users only need to initialize ``Qlib`` as follows.
-        
+
         .. code-block:: python
 
             from qlib.config import REG_US
             qlib.init(provider_uri='~/.qlib/qlib_data/us_data', region=REG_US)
-        
+
 
 .. note::
 
@@ -233,14 +247,14 @@ The `trade unit` defines the unit number of stocks can be used in a trade, and t
 
 
 Data API
-========================
+========
 
 Data Retrieval
----------------
+--------------
 Users can use APIs in ``qlib.data`` to retrieve data, please refer to `Data Retrieval <../start/getdata.html>`_.
 
 Feature
-------------------
+-------
 
 ``Qlib`` provides `Feature` and `ExpressionOps` to fetch the features according to users' needs.
 
@@ -255,7 +269,7 @@ Feature
 To know more about  ``Feature``, please refer to `Feature API <../reference/api.html#module-qlib.data.base>`_.
 
 Filter
--------------------
+------
 ``Qlib`` provides `NameDFilter` and `ExpressionDFilter` to filter the instruments according to users' needs.
 
 - `NameDFilter`
@@ -263,7 +277,7 @@ Filter
 
 - `ExpressionDFilter`
     Expression dynamic instrument filter. Filter the instruments based on a certain expression. An expression rule indicating a certain feature field is required.
-    
+
     - `basic features filter`: rule_expression = '$close/$open>5'
     - `cross-sectional features filter` \: rule_expression = '$rank($close)<10'
     - `time-sequence features filter`: rule_expression = '$Ref($close, 3)>100'
@@ -290,29 +304,29 @@ Here is a simple example showing how to use filter in a basic ``Qlib`` workflow
 To know more about ``Filter``, please refer to `Filter API <../reference/api.html#module-qlib.data.filter>`_.
 
 Reference
--------------
+---------
 
 To know more about ``Data API``, please refer to `Data API <../reference/api.html#data>`_.
 
 
 Data Loader
-=================
+===========
 
 ``Data Loader`` in ``Qlib`` is designed to load raw data from the original data source. It will be loaded and used in the ``Data Handler`` module.
 
 QlibDataLoader
----------------
+--------------
 
 The ``QlibDataLoader`` class in ``Qlib`` is such an interface that allows users to load raw data from the ``Qlib`` data source.
 
 StaticDataLoader
----------------
+----------------
 
 The ``StaticDataLoader`` class in ``Qlib`` is such an interface that allows users to load raw data from file or as provided.
 
 
 Interface
-------------
+---------
 
 Here are some interfaces of the ``QlibDataLoader`` class:
 
@@ -320,28 +334,28 @@ Here are some interfaces of the ``QlibDataLoader`` class:
     :members:
 
 API
------------
+---
 
 To know more about ``Data Loader``, please refer to `Data Loader API <../reference/api.html#module-qlib.data.dataset.loader>`_.
 
 
 Data Handler
-=================
+============
 
 The ``Data Handler`` module in ``Qlib`` is designed to handler those common data processing methods which will be used by most of the models.
 
-Users can use ``Data Handler`` in an automatic workflow by ``qrun``, refer to `Workflow: Workflow Management <workflow.html>`_ for more details. 
+Users can use ``Data Handler`` in an automatic workflow by ``qrun``, refer to `Workflow: Workflow Management <workflow.html>`_ for more details.
 
 DataHandlerLP
---------------
+-------------
 
-In addition to use ``Data Handler`` in an automatic workflow with ``qrun``, ``Data Handler`` can be used as an independent module, by which users can easily preprocess data (standardization, remove NaN, etc.) and build datasets. 
+In addition to use ``Data Handler`` in an automatic workflow with ``qrun``, ``Data Handler`` can be used as an independent module, by which users can easily preprocess data (standardization, remove NaN, etc.) and build datasets.
 
-In order to achieve so, ``Qlib`` provides a base class `qlib.data.dataset.DataHandlerLP <../reference/api.html#qlib.data.dataset.handler.DataHandlerLP>`_. The core idea of this class is that: we will have some leanable ``Processors`` which can learn the parameters of data processing(e.g., parameters for zscore normalization). When new data comes in, these `trained` ``Processors`` can then process the new data and thus processing real-time data in an efficient way becomes possible. More information about ``Processors`` will be listed in the next subsection.
+In order to achieve so, ``Qlib`` provides a base class `qlib.data.dataset.DataHandlerLP <../reference/api.html#qlib.data.dataset.handler.DataHandlerLP>`_. The core idea of this class is that: we will have some learnable ``Processors`` which can learn the parameters of data processing(e.g., parameters for zscore normalization). When new data comes in, these `trained` ``Processors`` can then process the new data and thus processing real-time data in an efficient way becomes possible. More information about ``Processors`` will be listed in the next subsection.
 
 
 Interface
-----------------------
+---------
 
 Here are some important interfaces that ``DataHandlerLP`` provides:
 
@@ -355,7 +369,7 @@ Also, users can pass ``qlib.contrib.data.processor.ConfigSectionProcessor`` that
 
 
 Processor
-----------
+---------
 
 The ``Processor`` module in ``Qlib`` is designed to be learnable and it is responsible for handling data processing such as `normalization` and `drop none/nan features/labels`.
 
@@ -373,14 +387,14 @@ The ``Processor`` module in ``Qlib`` is designed to be learnable and it is respo
 - ``CSRankNorm``: `processor` that applies cross sectional rank normalization.
 - ``CSZFillna``: `processor` that fills N/A values in a cross sectional way by the mean of the column.
 
-Users can also create their own `processor` by inheriting the base class of ``Processor``. Please refer to the implementation of all the processors for more information (`Processor Link <https://github.com/microsoft/qlib/blob/main/qlib/data/dataset/processor.py>`_). 
+Users can also create their own `processor` by inheriting the base class of ``Processor``. Please refer to the implementation of all the processors for more information (`Processor Link <https://github.com/microsoft/qlib/blob/main/qlib/data/dataset/processor.py>`_).
 
 To know more about ``Processor``, please refer to `Processor API <../reference/api.html#module-qlib.data.dataset.processor>`_.
 
 Example
---------------
+-------
 
-``Data Handler`` can be run with ``qrun`` by modifying the configuration file, and can also be used as a single module. 
+``Data Handler`` can be run with ``qrun`` by modifying the configuration file, and can also be used as a single module.
 
 Know more about how to run ``Data Handler`` with ``qrun``, please refer to `Workflow: Workflow Management <workflow.html>`_
 
@@ -418,17 +432,17 @@ Qlib provides implemented data handler `Alpha158`. The following example shows h
 .. note:: In the ``Alpha158``, ``Qlib`` uses the label `Ref($close, -2)/Ref($close, -1) - 1` that means the change from T+1 to T+2, rather than `Ref($close, -1)/$close - 1`, of which the reason is that when getting the T day close price of a china stock, the stock can be bought on T+1 day and sold on T+2 day.
 
 API
----------
+---
 
 To know more about ``Data Handler``, please refer to `Data Handler API <../reference/api.html#module-qlib.data.dataset.handler>`_.
 
 
 Dataset
-=================
+=======
 
 The ``Dataset`` module in ``Qlib`` aims to prepare data for model training and inferencing.
 
-The motivation of this module is that we want to maximize the flexibility of of different models to handle data that are suitable for themselves. This module gives the model the flexibility to process their data in an unique way. For instance, models such as ``GBDT`` may work well on data that contains `nan` or `None` value, while neural networks such as ``MLP`` will break down on such data. 
+The motivation of this module is that we want to maximize the flexibility of different models to handle data that are suitable for themselves. This module gives the model the flexibility to process their data in an unique way. For instance, models such as ``GBDT`` may work well on data that contains `nan` or `None` value, while neural networks such as ``MLP`` will break down on such data.
 
 If user's model need process its data in a different way, user could implement his own ``Dataset`` class. If the model's
 data processing is not special, ``DatasetH`` can be used directly.
@@ -439,18 +453,18 @@ The ``DatasetH`` class is the `dataset` with `Data Handler`. Here is the most im
     :members:
 
 API
----------
+---
 
 To know more about ``Dataset``, please refer to `Dataset API <../reference/api.html#dataset>`_.
 
 
 Cache
-==========
+=====
 
 ``Cache`` is an optional module that helps accelerate providing data by saving some frequently-used data as cache file. ``Qlib`` provides a `Memcache` class to cache the most-frequently-used data in memory, an inheritable `ExpressionCache` class, and an inheritable `DatasetCache` class.
 
 Global Memory Cache
----------------------
+-------------------
 
 `Memcache` is a global memory cache mechanism that composes of three `MemCacheUnit` instances to cache **Calendar**, **Instruments**, and **Features**. The `MemCache` is defined globally in `cache.py` as `H`. Users can use `H['c'], H['i'], H['f']` to get/set `memcache`.
 
@@ -462,7 +476,7 @@ Global Memory Cache
 
 
 ExpressionCache
------------------
+---------------
 
 `ExpressionCache` is a cache mechanism that saves expressions such as **Mean($close, 5)**. Users can inherit this base class to define their own cache mechanism that saves expressions according to the following steps.
 
@@ -477,7 +491,7 @@ The following shows the details about the interfaces:
 ``Qlib`` has currently provided implemented disk cache `DiskExpressionCache` which inherits from `ExpressionCache` . The expressions data will be stored in the disk.
 
 DatasetCache
------------------
+------------
 
 `DatasetCache` is a cache mechanism that saves datasets. A certain dataset is regulated by a stock pool configuration (or a series of instruments, though not recommended), a list of expressions or static feature fields, the start time, and end time for the collected features and the frequency. Users can inherit this base class to define their own cache mechanism that saves datasets according to the following steps.
 
@@ -494,7 +508,7 @@ The following shows the details about the interfaces:
 
 
 Data and Cache File Structure
-==================================
+=============================
 
 We've specially designed a file structure to manage data and cache, please refer to the `File storage design section in Qlib paper <https://arxiv.org/abs/2009.11189>`_ for detailed information. The file structure of data and cache is listed as follows.
 
@@ -527,4 +541,3 @@ We've specially designed a file structure to manage data and cache, please refer
                 - .meta : an assorted meta file recording the stockpool config, field names and visit times
                 - .index : an assorted index file recording the line index of all calendars
             - ...
-

docs/component/highfreq.rst

@@ -0,0 +1,38 @@
+.. _highfreq:
+
+========================================================================
+Design of Nested Decision Execution Framework for High-Frequency Trading
+========================================================================
+.. currentmodule:: qlib
+
+Introduction
+============
+
+Daily trading (e.g. portfolio management) and intraday trading (e.g. orders execution) are two hot topics in Quant investment and usually studied separately.
+
+To get the join trading performance of daily and intraday trading, they must interact with each other and run backtest jointly.
+In order to support the joint backtest strategies in multiple levels, a corresponding framework is required. None of the publicly available high-frequency trading frameworks considers multi-level joint trading, which make the backtesting aforementioned inaccurate.
+
+Besides backtesting, the optimization of strategies from different levels is not standalone and can be affected by each other.
+For example, the best portfolio management strategy may change with the performance of order executions(e.g. a portfolio with higher turnover may becomes a better choice when we improve the order execution strategies).
+To achieve the overall good performance , it is necessary to consider the interaction of strategies in different level.
+
+Therefore, building a new framework for trading in multiple levels becomes necessary to solve the various problems mentioned above, for which we designed a nested decision execution framework that consider the interaction of strategies.
+
+.. image:: ../_static/img/framework.svg
+
+The design of the framework is shown in the yellow part in the middle of the figure above. Each level consists of ``Trading Agent`` and ``Execution Env``. ``Trading Agent`` has its own data processing module (``Information Extractor``), forecasting module (``Forecast Model``) and decision generator (``Decision Generator``). The trading algorithm generates the decisions by the ``Decision Generator`` based on the forecast signals output by the ``Forecast Module``, and the decisions generated by the trading algorithm are passed to the ``Execution Env``, which returns the execution results.
+
+The frequency of trading algorithm, decision content and execution environment can be customized by users (e.g. intraday trading, daily-frequency trading, weekly-frequency trading), and the execution environment can be nested with finer-grained trading algorithm and execution environment inside (i.e. sub-workflow in the figure, e.g. daily-frequency orders can be turned into finer-grained decisions by splitting orders within the day). The flexibility of nested decision execution framework makes it easy for users to explore the effects of combining different levels of trading strategies and break down the optimization barriers between different levels of trading algorithm.
+
+Example
+=======
+
+An example of nested decision execution framework for high-frequency can be found `here <https://github.com/microsoft/qlib/blob/main/examples/nested_decision_execution/workflow.py>`_.
+
+
+Besides, the above examples, here are some other related work about high-frequency trading in Qlib.
+
+- `Prediction with high-frequency data <https://github.com/microsoft/qlib/tree/main/examples/highfreq#benchmarks-performance-predicting-the-price-trend-in-high-frequency-data>`_
+- `Examples <https://github.com/microsoft/qlib/blob/main/examples/orderbook_data/>`_ to extract features form high-frequency data without fixed frequency.
+- `A paper <https://github.com/microsoft/qlib/tree/high-freq-execution#high-frequency-execution>`_ for high-frequency trading.

docs/component/meta.rst

@@ -0,0 +1,68 @@
+.. _meta:
+
+======================================================
+Meta Controller: Meta-Task & Meta-Dataset & Meta-Model
+======================================================
+.. currentmodule:: qlib
+
+
+Introduction
+============
+``Meta Controller`` provides guidance to ``Forecast Model``, which aims to learn regular patterns among a series of forecasting tasks and use learned patterns to guide forthcoming forecasting tasks. Users can implement their own meta-model instance based on ``Meta Controller`` module.
+
+Meta Task
+=========
+
+A `Meta Task` instance is the basic element in the meta-learning framework. It saves the data that can be used for the `Meta Model`. Multiple `Meta Task` instances may share the same `Data Handler`, controlled by `Meta Dataset`. Users should use `prepare_task_data()` to obtain the data that can be directly fed into the `Meta Model`.
+
+.. autoclass:: qlib.model.meta.task.MetaTask
+    :members:
+
+Meta Dataset
+============
+
+`Meta Dataset` controls the meta-information generating process. It is on the duty of providing data for training the `Meta Model`. Users should use `prepare_tasks` to retrieve a list of `Meta Task` instances.
+
+.. autoclass:: qlib.model.meta.dataset.MetaTaskDataset
+    :members:
+
+Meta Model
+==========
+
+General Meta Model
+------------------
+`Meta Model` instance is the part that controls the workflow. The usage of the `Meta Model` includes:
+1. Users train their `Meta Model` with the `fit` function.
+2. The `Meta Model` instance guides the workflow by giving useful information via the `inference` function.
+
+.. autoclass:: qlib.model.meta.model.MetaModel
+    :members:
+
+Meta Task Model
+---------------
+This type of meta-model may interact with task definitions directly. Then, the `Meta Task Model` is the class for them to inherit from. They guide the base tasks by modifying the base task definitions. The function `prepare_tasks` can be used to obtain the modified base task definitions.
+
+.. autoclass:: qlib.model.meta.model.MetaTaskModel
+    :members:
+
+Meta Guide Model
+----------------
+This type of meta-model participates in the training process of the base forecasting model. The meta-model may guide the base forecasting models during their training to improve their performances.
+
+.. autoclass:: qlib.model.meta.model.MetaGuideModel
+    :members:
+
+
+Example
+=======
+``Qlib`` provides an implementation of ``Meta Model`` module, ``DDG-DA``,
+which adapts to the market dynamics.
+
+``DDG-DA`` includes four steps:
+
+1. Calculate meta-information and encapsulate it into ``Meta Task`` instances. All the meta-tasks form a ``Meta Dataset`` instance.
+2. Train ``DDG-DA`` based on the training data of the meta-dataset.
+3. Do the inference of the ``DDG-DA`` to get guide information.
+4. Apply guide information to the forecasting models to improve their performances.
+
+The `above example <https://github.com/microsoft/qlib/tree/main/examples/benchmarks_dynamic/DDG-DA>`_ can be found in ``examples/benchmarks_dynamic/DDG-DA/workflow.py``.

docs/component/model.rst

@@ -1,13 +1,13 @@
 .. _model:
 
-============================================
+===========================================
 Forecast Model: Model Training & Prediction
-============================================
+===========================================
 
 Introduction
-===================
+============
 
-``Forecast Model`` is designed to make the `prediction score` about stocks. Users can use the ``Forecast Model`` in an automatic workflow by ``qrun``, please refer to `Workflow: Workflow Management <workflow.html>`_.  
+``Forecast Model`` is designed to make the `prediction score` about stocks. Users can use the ``Forecast Model`` in an automatic workflow by ``qrun``, please refer to `Workflow: Workflow Management <workflow.html>`_.
 
 Because the components in ``Qlib`` are designed in a loosely-coupled way, ``Forecast Model`` can be used as an independent module also.
 
@@ -22,11 +22,11 @@ The base class provides the following interfaces:
     :members:
 
 ``Qlib`` also provides a base class `qlib.model.base.ModelFT <../reference/api.html#qlib.model.base.ModelFT>`_, which includes the method for finetuning the model.
-    
+
 For other interfaces such as `finetune`, please refer to `Model API <../reference/api.html#module-qlib.model.base>`_.
 
 Example
-==================
+=======
 
 ``Qlib``'s `Model Zoo` includes models such as ``LightGBM``, ``MLP``, ``LSTM``, etc.. These models are treated as the baselines of ``Forecast Model``. The following steps show how to run`` LightGBM`` as an independent module.
 
@@ -84,7 +84,7 @@ Example
                 },
             },
         }
-        
+
         # model initiaiton
         model = init_instance_by_config(task["model"])
         dataset = init_instance_by_config(task["dataset"])
@@ -100,19 +100,22 @@ Example
             sr = SignalRecord(model, dataset, recorder)
             sr.generate()
 
-    .. note:: 
-        
+    .. note::
+
         `Alpha158` is the data handler provided by ``Qlib``, please refer to `Data Handler <data.html#data-handler>`_.
         `SignalRecord` is the `Record Template` in ``Qlib``, please refer to `Workflow <recorder.html#record-template>`_.
 
 Also, the above example has been given in ``examples/train_backtest_analyze.ipynb``.
+Technically, the meaning of the model prediction depends on the label setting designed by user.
+By default, the meaning of the score is normally the rating of the instruments by the forecasting model. The higher the score, the more profit the instruments.
+
 
 Custom Model
-===================
+============
 
 Qlib supports custom models. If users are interested in customizing their own models and integrating the models into ``Qlib``, please refer to `Custom Model Integration <../start/integration.html>`_.
 
 
 API
-===================
+===
 Please refer to `Model API <../reference/api.html#module-qlib.model.base>`_.

docs/component/online.rst

@@ -1,13 +1,13 @@
 .. _online:
 
-=================================
+==============
 Online Serving
-=================================
+==============
 .. currentmodule:: qlib
 
 
 Introduction
-=============
+============
 
 .. image:: ../_static/img/online_serving.png
     :align: center
@@ -15,32 +15,38 @@ Introduction
 
 In addition to backtesting, one way to test a model is effective is to make predictions in real market conditions or even do real trading based on those predictions.
 ``Online Serving`` is a set of modules for online models using the latest data,
-which including `Online Manager <#Online Manager>`_, `Online Strategy <#Online Strategy>`_, `Online Tool <#Online Tool>`_, `Updater <#Updater>`_. 
+which including `Online Manager <#Online Manager>`_, `Online Strategy <#Online Strategy>`_, `Online Tool <#Online Tool>`_, `Updater <#Updater>`_.
 
 `Here <https://github.com/microsoft/qlib/tree/main/examples/online_srv>`_ are several examples for reference, which demonstrate different features of ``Online Serving``.
 If you have many models or `task` needs to be managed, please consider `Task Management <../advanced/task_management.html>`_.
 The `examples <https://github.com/microsoft/qlib/tree/main/examples/online_srv>`_ are based on some components in `Task Management <../advanced/task_management.html>`_ such as ``TrainerRM`` or ``Collector``.
 
+**NOTE**: User should keep his data source updated to support online serving. For example, Qlib provides `a batch of scripts <https://github.com/microsoft/qlib/blob/main/scripts/data_collector/yahoo/README.md#automatic-update-of-daily-frequency-datafrom-yahoo-finance>`_ to help users update Yahoo daily data.
+
+Known limitations currently
+- Currently, the daily updating prediction for the next trading day is supported. But generating orders for the next trading day is not supported due to the `limitations of public data <https://github.com/microsoft/qlib/issues/215#issuecomment-766293563>_`
+
+
 Online Manager
-=============
+==============
 
 .. automodule:: qlib.workflow.online.manager
     :members:
 
 Online Strategy
-=============
+===============
 
 .. automodule:: qlib.workflow.online.strategy
     :members:
 
 Online Tool
-=============
+===========
 
 .. automodule:: qlib.workflow.online.utils
     :members:
 
 Updater
-=============
+=======
 
 .. automodule:: qlib.workflow.online.update
-    :members:
+    :members:

docs/component/recorder.rst

@@ -6,8 +6,8 @@ Qlib Recorder: Experiment Management
 .. currentmodule:: qlib
 
 Introduction
-===================
-``Qlib`` contains an experiment management system named ``QlibRecorder``, which is designed to help users handle experiment and analyse results in an efficient way. 
+============
+``Qlib`` contains an experiment management system named ``QlibRecorder``, which is designed to help users handle experiment and analyse results in an efficient way.
 
 There are three components of the system:
 
@@ -34,13 +34,13 @@ Here is a general view of the structure of the system:
             - Recorder 2
             - ...
         - ...
-        
-This experiment management system defines a set of interface and provided a concrete implementation ``MLflowExpManager``, which is based on the machine learning platform: ``MLFlow`` (`link <https://mlflow.org/>`_). 
 
-If users set the implementation of ``ExpManager`` to be ``MLflowExpManager``, they can use the command `mlflow ui` to visualize and check the experiment results. For more information, pleaes refer to the related documents `here <https://www.mlflow.org/docs/latest/cli.html#mlflow-ui>`_.
+This experiment management system defines a set of interface and provided a concrete implementation ``MLflowExpManager``, which is based on the machine learning platform: ``MLFlow`` (`link <https://mlflow.org/>`_).
+
+If users set the implementation of ``ExpManager`` to be ``MLflowExpManager``, they can use the command `mlflow ui` to visualize and check the experiment results. For more information, please refer to the related documents `here <https://www.mlflow.org/docs/latest/cli.html#mlflow-ui>`_.
 
 Qlib Recorder
-===================
+=============
 ``QlibRecorder`` provides a high level API for users to use the experiment management system. The interfaces are wrapped in the variable ``R`` in ``Qlib``, and users can directly use ``R`` to interact with the system. The following command shows how to import ``R`` in Python:
 
 .. code-block:: Python
@@ -55,7 +55,7 @@ Here are the available interfaces of ``QlibRecorder``:
     :members:
 
 Experiment Manager
-===================
+==================
 
 The ``ExpManager`` module in ``Qlib`` is responsible for managing different experiments. Most of the APIs of ``ExpManager`` are similar to ``QlibRecorder``, and the most important API will be the ``get_exp`` method. User can directly refer to the documents above for some detailed information about how to use the ``get_exp`` method.
 
@@ -65,7 +65,7 @@ The ``ExpManager`` module in ``Qlib`` is responsible for managing different expe
 For other interfaces such as `create_exp`, `delete_exp`, please refer to `Experiment Manager API <../reference/api.html#experiment-manager>`_.
 
 Experiment
-===================
+==========
 
 The ``Experiment`` class is solely responsible for a single experiment, and it will handle any operations that are related to an experiment. Basic methods such as `start`, `end` an experiment are included. Besides, methods related to `recorders` are also available: such methods include `get_recorder` and `list_recorders`.
 
@@ -77,7 +77,7 @@ For other interfaces such as `search_records`, `delete_recorder`, please refer t
 ``Qlib`` also provides a default ``Experiment``, which will be created and used under certain situations when users use the APIs such as `log_metrics` or `get_exp`. If the default ``Experiment`` is used, there will be related logged information when running ``Qlib``. Users are able to change the name of the default ``Experiment`` in the config file of ``Qlib`` or during ``Qlib``'s `initialization <../start/initialization.html#parameters>`_, which is set to be '`Experiment`'.
 
 Recorder
-===================
+========
 
 The ``Recorder`` class is responsible for a single recorder. It will handle some detailed operations such as ``log_metrics``, ``log_params`` of a single run. It is designed to help user to easily track results and things being generated during a run.
 
@@ -89,7 +89,7 @@ Here are some important APIs that are not included in the ``QlibRecorder``:
 For other interfaces such as `save_objects`, `load_object`, please refer to `Recorder API <../reference/api.html#recorder>`_.
 
 Record Template
-===================
+===============
 
 The ``RecordTemp`` class is a class that enables generate experiment results such as IC and backtest in a certain format. We have provided three different `Record Template` class:
 
@@ -123,7 +123,6 @@ Here is a simple exampke of what is done in ``PortAnaRecord``, which users can r
         "n_drop": 5,
     }
     BACKTEST_CONFIG = {
-        "verbose": False,
         "limit_threshold": 0.095,
         "account": 100000000,
         "benchmark": BENCHMARK,
@@ -132,7 +131,7 @@ Here is a simple exampke of what is done in ``PortAnaRecord``, which users can r
         "close_cost": 0.0015,
         "min_cost": 5,
     }
-    
+
     strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)
     report_normal, positions_normal = normal_backtest(pred_score, strategy=strategy, **BACKTEST_CONFIG)
 
@@ -144,3 +143,9 @@ Here is a simple exampke of what is done in ``PortAnaRecord``, which users can r
     print(analysis_df)
 
 For more information about the APIs, please refer to `Record Template API <../reference/api.html#module-qlib.workflow.record_temp>`_.
+
+
+
+Known Limitations
+=================
+- The Python objects are saved based on pickle, which may results in issues when the environment dumping objects and loading objects are different.

docs/component/report.rst

@@ -1,11 +1,11 @@
 .. _report:
 
-==========================================
+=======================================
 Analysis: Evaluation & Results Analysis
-==========================================
+=======================================
 
 Introduction
-===================
+============
 
 ``Analysis`` is designed to show the graphical reports of ``Intraday Trading`` , which helps users to evaluate and analyse investment portfolios visually. The following are some graphics to view:
 
@@ -20,8 +20,11 @@ Introduction
     - model_performance_graph
 
 
+All of the accumulated profit metrics(e.g. return, max drawdown) in Qlib are calculated by summation.
+This avoids the metrics or the plots being skewed exponentially over time.
+
 Graphical Reports
-===================
+=================
 
 Users can run the following code to get all supported reports.
 
@@ -38,13 +41,13 @@ Users can run the following code to get all supported reports.
 
 
 Usage & Example
-===================
+===============
 
 Usage of `analysis_position.report`
 -----------------------------------
 
 API
-~~~~~~~~~~~~~~~~
+~~~
 
 .. automodule:: qlib.contrib.report.analysis_position.report
     :members:
@@ -55,7 +58,7 @@ Graphical Result
 .. note::
 
     - Axis X: Trading day
-    - Axis Y: 
+    - Axis Y:
         - `cum bench`
             Cumulative returns series of benchmark
         - `cum return wo cost`
@@ -79,34 +82,34 @@ Graphical Result
     - The shaded part above: Maximum drawdown corresponding to `cum return wo cost`
     - The shaded part below: Maximum drawdown corresponding to `cum ex return wo cost`
 
-.. image:: ../_static/img/analysis/report.png 
+.. image:: ../_static/img/analysis/report.png
 
 
 Usage of `analysis_position.score_ic`
 -------------------------------------
 
 API
-~~~~~~~~~~~~~~~~
+~~~
 
 .. automodule:: qlib.contrib.report.analysis_position.score_ic
     :members:
 
 
 Graphical Result
-~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~
 
-.. note:: 
+.. note::
 
     - Axis X: Trading day
-    - Axis Y: 
+    - Axis Y:
         - `ic`
             The `Pearson correlation coefficient` series between `label` and `prediction score`.
-            In the above example, the `label` is formulated as `Ref($close, -1)/$close - 1`. Please refer to `Data Feature <data.html#feature>`_ for more details.
-                
+            In the above example, the `label` is formulated as `Ref($close, -2)/Ref($close, -1)-1`. Please refer to `Data Feature <data.html#feature>`_ for more details.
+
         - `rank_ic`
             The `Spearman's rank correlation coefficient` series between `label` and `prediction score`.
 
-.. image:: ../_static/img/analysis/score_ic.png 
+.. image:: ../_static/img/analysis/score_ic.png
 
 
 .. Usage of `analysis_position.cumulative_return`
@@ -121,7 +124,7 @@ Graphical Result
 .. Graphical Result
 .. ~~~~~~~~~~~~~~~~~
 ..
-.. .. note:: 
+.. .. note::
 ..
 ..     - Axis X: Trading day
 ..     - Axis Y:
@@ -131,27 +134,27 @@ Graphical Result
 ..     - In the **buy_minus_sell** graph, the **y** value of the **weight** graph at the bottom is `buy_weight + sell_weight`.
 ..     - In each graph, the **red line** in the histogram on the right represents the average.
 ..
-.. .. image:: ../_static/img/analysis/cumulative_return_buy.png 
+.. .. image:: ../_static/img/analysis/cumulative_return_buy.png
 ..
-.. .. image:: ../_static/img/analysis/cumulative_return_sell.png 
+.. .. image:: ../_static/img/analysis/cumulative_return_sell.png
 ..
-.. .. image:: ../_static/img/analysis/cumulative_return_buy_minus_sell.png 
+.. .. image:: ../_static/img/analysis/cumulative_return_buy_minus_sell.png
 ..
-.. .. image:: ../_static/img/analysis/cumulative_return_hold.png 
+.. .. image:: ../_static/img/analysis/cumulative_return_hold.png
 
 
 Usage of `analysis_position.risk_analysis`
-----------------------------------------------
+------------------------------------------
 
 API
-~~~~~~~~~~~~~~~~
+~~~
 
 .. automodule:: qlib.contrib.report.analysis_position.risk_analysis
     :members:
 
 
 Graphical Result
-~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~
 
 .. note::
 
@@ -207,7 +210,7 @@ Graphical Result
                     The `Standard Deviation` series of monthly `CAR` (cumulative abnormal return) without cost.
                 - `excess_return_with_cost_max_drawdown`
                     The `Standard Deviation` series of monthly `CAR` (cumulative abnormal return) with cost.
-                
+
 
 .. image:: ../_static/img/analysis/risk_analysis_annualized_return.png
     :align: center
@@ -218,58 +221,58 @@ Graphical Result
 .. image:: ../_static/img/analysis/risk_analysis_information_ratio.png
     :align: center
 
-.. image:: ../_static/img/analysis/risk_analysis_std.png 
+.. image:: ../_static/img/analysis/risk_analysis_std.png
     :align: center
 
 ..
 .. Usage of `analysis_position.rank_label`
-.. ----------------------------------------------
+.. ---------------------------------------
 ..
 .. API
-.. ~~~~~
+.. ~~~
 ..
 .. .. automodule:: qlib.contrib.report.analysis_position.rank_label
 ..     :members:
 ..
 ..
 .. Graphical Result
-.. ~~~~~~~~~~~~~~~~~
+.. ~~~~~~~~~~~~~~~~
 ..
-.. .. note:: 
+.. .. note::
 ..
 ..     - hold/sell/buy graphics:
 ..         - Axis X: Trading day
-..         - Axis Y: 
+..         - Axis Y:
 ..             Average `ranking ratio`of `label` for stocks that is held/sold/bought on the trading day.
 ..
 ..             In the above example, the `label` is formulated as `Ref($close, -1)/$close - 1`. The `ranking ratio` can be formulated as follows.
 ..             .. math::
-..                 
+..
 ..                 ranking\ ratio = \frac{Ascending\ Ranking\ of\ label}{Number\ of\ Stocks\ in\ the\ Portfolio}
 ..
-.. .. image:: ../_static/img/analysis/rank_label_hold.png 
+.. .. image:: ../_static/img/analysis/rank_label_hold.png
 ..     :align: center
 ..
-.. .. image:: ../_static/img/analysis/rank_label_buy.png 
+.. .. image:: ../_static/img/analysis/rank_label_buy.png
 ..     :align: center
 ..
-.. .. image:: ../_static/img/analysis/rank_label_sell.png 
+.. .. image:: ../_static/img/analysis/rank_label_sell.png
 ..     :align: center
 ..
 ..
 
 Usage of `analysis_model.analysis_model_performance`
------------------------------------------------------
+----------------------------------------------------
 
 API
-~~~~~
+~~~
 
 .. automodule:: qlib.contrib.report.analysis_model.analysis_model_performance
     :members:
 
 
 Graphical Results
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 
 .. note::
 
@@ -288,13 +291,13 @@ Graphical Results
             The Difference series between `Cumulative Return` of `Group1` and of `Group5`
         - `long-average`
             The Difference series between `Cumulative Return` of `Group1` and average `Cumulative Return` for all stocks.
-        
+
         The `ranking ratio` can be formulated as follows.
             .. math::
-                
+
                 ranking\ ratio = \frac{Ascending\ Ranking\ of\ label}{Number\ of\ Stocks\ in\ the\ Portfolio}
 
-.. image:: ../_static/img/analysis/analysis_model_cumulative_return.png 
+.. image:: ../_static/img/analysis/analysis_model_cumulative_return.png
     :align: center
 
 .. note::
@@ -302,7 +305,7 @@ Graphical Results
         The distribution of long-short/long-average returns on each trading day
 
 
-.. image:: ../_static/img/analysis/analysis_model_long_short.png 
+.. image:: ../_static/img/analysis/analysis_model_long_short.png
     :align: center
 
 .. TODO: ask xiao yang for detial
@@ -312,14 +315,14 @@ Graphical Results
         - The `Pearson correlation coefficient` series between `labels` and `prediction scores` of stocks in portfolio.
         - The graphics reports can be used to evaluate the `prediction scores`.
 
-.. image:: ../_static/img/analysis/analysis_model_IC.png 
+.. image:: ../_static/img/analysis/analysis_model_IC.png
     :align: center
 
 .. note::
     - Monthly IC
         Monthly average of the `Information Coefficient`
 
-.. image:: ../_static/img/analysis/analysis_model_monthly_IC.png 
+.. image:: ../_static/img/analysis/analysis_model_monthly_IC.png
     :align: center
 
 .. note::
@@ -328,14 +331,14 @@ Graphical Results
     - IC Normal Dist. Q-Q
         The `Quantile-Quantile Plot` is used for the normal distribution of `Information Coefficient` on each trading day.
 
-.. image:: ../_static/img/analysis/analysis_model_NDQ.png 
+.. image:: ../_static/img/analysis/analysis_model_NDQ.png
     :align: center
 
 .. note::
     - Auto Correlation
-        - The `Pearson correlation coefficient` series between the latest `prediction scores` and the `prediction scores` `lag` days ago of stocks in portfolio on each trading day. 
+        - The `Pearson correlation coefficient` series between the latest `prediction scores` and the `prediction scores` `lag` days ago of stocks in portfolio on each trading day.
         - The graphics reports can be used to estimate the turnover rate.
-         
 
-.. image:: ../_static/img/analysis/analysis_model_auto_correlation.png 
+
+.. image:: ../_static/img/analysis/analysis_model_auto_correlation.png
     :align: center

docs/component/strategy.rst

@@ -6,34 +6,35 @@ Portfolio Strategy: Portfolio Management
 .. currentmodule:: qlib
 
 Introduction
-===================
+============
 
-``Portfolio Strategy`` is designed to adopt different portfolio strategies, which means that users can adopt different algorithms to generate investment portfolios based on the prediction scores of the ``Forecast Model``. Users can use the ``Portfolio Strategy`` in an automatic workflow by ``Workflow`` module, please refer to `Workflow: Workflow Management <workflow.html>`_.  
+``Portfolio Strategy`` is designed to adopt different portfolio strategies, which means that users can adopt different algorithms to generate investment portfolios based on the prediction scores of the ``Forecast Model``. Users can use the ``Portfolio Strategy`` in an automatic workflow by ``Workflow`` module, please refer to `Workflow: Workflow Management <workflow.html>`_.
 
 Because the components in ``Qlib`` are designed in a loosely-coupled way, ``Portfolio Strategy`` can be used as an independent module also.
 
-``Qlib`` provides several implemented portfolio strategies. Also, ``Qlib`` supports custom strategy, users can customize strategies according to their own needs.
+``Qlib`` provides several implemented portfolio strategies. Also, ``Qlib`` supports custom strategy, users can customize strategies according to their own requirements.
+
+After users specifying the models(forecasting signals) and strategies, running backtest will help users to check the performance of a custom model(forecasting signals)/strategy.
 
 Base Class & Interface
 ======================
 
 BaseStrategy
-------------------
+------------
 
-Qlib provides a base class ``qlib.contrib.strategy.BaseStrategy``. All strategy classes need to inherit the base class and implement its interface.
+Qlib provides a base class ``qlib.strategy.base.BaseStrategy``. All strategy classes need to inherit the base class and implement its interface.
 
-- `get_risk_degree`
-    Return the proportion of your total value you will use in investment. Dynamically risk_degree will result in Market timing.
-
-- `generate_order_list`
-    Return the order list. 
+- `generate_trade_decision`
+    generate_trade_decision is a key interface that generates trade decisions in each trading bar.
+    The frequency to call this method depends on the executor frequency("time_per_step"="day" by default). But the trading frequency can be decided by users' implementation.
+    For example, if the user wants to trading in weekly while the `time_per_step` is "day" in executor, user can return non-empty TradeDecision weekly(otherwise return empty like `this <https://github.com/microsoft/qlib/blob/main/qlib/contrib/strategy/signal_strategy.py#L132>`_ ).
 
 Users can inherit `BaseStrategy` to customize their strategy class.
 
 WeightStrategyBase
---------------------
+------------------
 
-Qlib also provides a class ``qlib.contrib.strategy.WeightStrategyBase`` that is a subclass of `BaseStrategy`. 
+Qlib also provides a class ``qlib.contrib.strategy.WeightStrategyBase`` that is a subclass of `BaseStrategy`.
 
 `WeightStrategyBase` only focuses on the target positions, and automatically generates an order list based on positions. It provides the `generate_target_weight_position` interface.
 
@@ -59,62 +60,252 @@ Implemented Strategy
 Qlib provides a implemented strategy classes named `TopkDropoutStrategy`.
 
 TopkDropoutStrategy
-------------------
+-------------------
 `TopkDropoutStrategy` is a subclass of `BaseStrategy` and implement the interface `generate_order_list` whose process is as follows.
 
 - Adopt the ``Topk-Drop`` algorithm to calculate the target amount of each stock
 
     .. note::
-        ``Topk-Drop`` algorithm：
+        There are two parameters for the ``Topk-Drop`` algorithm:
 
         - `Topk`: The number of stocks held
         - `Drop`: The number of stocks sold on each trading day
-        
-        Currently, the number of held stocks is `Topk`.
-        On each trading day, the `Drop` number of held stocks with the worst `prediction score` will be sold, and the same number of unheld stocks with the best `prediction score` will be bought.
-        
+
+        In general, the number of stocks currently held is `Topk`, with the exception of being zero at the beginning period of trading.
+        For each trading day, let $d$ be the number of the instruments currently held and with a rank $\gt K$ when ranked by the prediction scores from high to low.
+        Then `d` number of stocks currently held with the worst `prediction score` will be sold, and the same number of unheld stocks with the best `prediction score` will be bought.
+
+        In general, $d=$`Drop`, especially when the pool of the candidate instruments is large, $K$ is large, and `Drop` is small.
+
+        In most cases, ``TopkDrop`` algorithm sells and buys `Drop` stocks every trading day, which yields a turnover rate of 2$\times$`Drop`/$K$.
+
+        The following images illustrate a typical scenario.
         .. image:: ../_static/img/topk_drop.png
             :alt: Topk-Drop
 
-        ``TopkDrop`` algorithm sells `Drop` stocks every trading day, which guarantees a fixed turnover rate.
-        
+
+
 - Generate the order list from the target amount
 
+EnhancedIndexingStrategy
+------------------------
+`EnhancedIndexingStrategy` Enhanced indexing combines the arts of active management and passive management,
+with the aim of outperforming a benchmark index (e.g., S&P 500) in terms of portfolio return while controlling
+the risk exposure (a.k.a. tracking error).
+
+For more information, please refer to `qlib.contrib.strategy.signal_strategy.EnhancedIndexingStrategy`
+and `qlib.contrib.strategy.optimizer.enhanced_indexing.EnhancedIndexingOptimizer`.
+
+
 Usage & Example
-====================
-``Portfolio Strategy`` can be specified in the ``Intraday Trading(Backtest)``, the example is as follows.
+===============
+
+First, user can create a model to get trading signals(the variable name is ``pred_score`` in following cases).
+
+Prediction Score
+----------------
+
+The `prediction score` is a pandas DataFrame. Its index is <datetime(pd.Timestamp), instrument(str)> and it must
+contains a `score` column.
+
+A prediction sample is shown as follows.
 
 .. code-block:: python
 
-    from qlib.contrib.strategy.strategy import TopkDropoutStrategy
-    from qlib.contrib.evaluate import backtest
-    STRATEGY_CONFIG = {
-        "topk": 50,
-        "n_drop": 5,
-    }
-    BACKTEST_CONFIG = {
-        "verbose": False,
-        "limit_threshold": 0.095,
-        "account": 100000000,
-        "benchmark": BENCHMARK,
-        "deal_price": "close",
-        "open_cost": 0.0005,
-        "close_cost": 0.0015,
-        "min_cost": 5,
-        
-    }
-    # use default strategy
-    strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)
+      datetime instrument     score
+    2019-01-04   SH600000 -0.505488
+    2019-01-04   SZ002531 -0.320391
+    2019-01-04   SZ000999  0.583808
+    2019-01-04   SZ300569  0.819628
+    2019-01-04   SZ001696 -0.137140
+                 ...            ...
+    2019-04-30   SZ000996 -1.027618
+    2019-04-30   SH603127  0.225677
+    2019-04-30   SH603126  0.462443
+    2019-04-30   SH603133 -0.302460
+    2019-04-30   SZ300760 -0.126383
 
-    # pred_score is the `prediction score` output by Model
-    report_normal, positions_normal = backtest(
-        pred_score, strategy=strategy, **BACKTEST_CONFIG
-    )
+``Forecast Model`` module can make predictions, please refer to `Forecast Model: Model Training & Prediction <model.html>`_.
 
-To know more about the `prediction score` `pred_score` output by ``Forecast Model``, please refer to `Forecast Model: Model Training & Prediction <model.html>`_.
+Normally, the prediction score is the output of the models. But some models are learned from a label with a different scale. So the scale of the prediction score may be different from your expectation(e.g. the return of instruments).
+
+Qlib didn't add a step to scale the prediction score to a unified scale due to the following reasons.
+- Because not every trading strategy cares about the scale(e.g. TopkDropoutStrategy only cares about the order).  So the strategy is responsible for rescaling the prediction score(e.g. some portfolio-optimization-based strategies may require a meaningful scale).
+- The model has the flexibility to define the target, loss, and data processing. So we don't think there is a silver bullet to rescale it back directly barely based on the model's outputs. If you want to scale it back to some meaningful values(e.g. stock returns.), an intuitive solution is to create a regression model for the model's recent outputs and your recent target values.
+
+Running backtest
+----------------
+
+- In most cases, users could backtest their portfolio management strategy  with ``backtest_daily``.
+
+    .. code-block:: python
+
+        from pprint import pprint
+
+        import qlib
+        import pandas as pd
+        from qlib.utils.time import Freq
+        from qlib.utils import flatten_dict
+        from qlib.contrib.evaluate import backtest_daily
+        from qlib.contrib.evaluate import risk_analysis
+        from qlib.contrib.strategy import TopkDropoutStrategy
+
+        # init qlib
+        qlib.init(provider_uri=<qlib data dir>)
+
+        CSI300_BENCH = "SH000300"
+        STRATEGY_CONFIG = {
+            "topk": 50,
+            "n_drop": 5,
+            # pred_score, pd.Series
+            "signal": pred_score,
+        }
+
+
+        strategy_obj = TopkDropoutStrategy(**STRATEGY_CONFIG)
+        report_normal, positions_normal = backtest_daily(
+            start_time="2017-01-01", end_time="2020-08-01", strategy=strategy_obj
+        )
+        analysis = dict()
+        # default frequency will be daily (i.e. "day")
+        analysis["excess_return_without_cost"] = risk_analysis(report_normal["return"] - report_normal["bench"])
+        analysis["excess_return_with_cost"] = risk_analysis(report_normal["return"] - report_normal["bench"] - report_normal["cost"])
+
+        analysis_df = pd.concat(analysis)  # type: pd.DataFrame
+        pprint(analysis_df)
+
+
+
+- If users would like to control their strategies in a more detailed(e.g. users have a more advanced version of executor), user could follow this example.
+
+    .. code-block:: python
+
+        from pprint import pprint
+
+        import qlib
+        import pandas as pd
+        from qlib.utils.time import Freq
+        from qlib.utils import flatten_dict
+        from qlib.backtest import backtest, executor
+        from qlib.contrib.evaluate import risk_analysis
+        from qlib.contrib.strategy import TopkDropoutStrategy
+
+        # init qlib
+        qlib.init(provider_uri=<qlib data dir>)
+
+        CSI300_BENCH = "SH000300"
+        # Benchmark is for calculating the excess return of your strategy.
+        # Its data format will be like **ONE normal instrument**.
+        # For example, you can query its data with the code below
+        # `D.features(["SH000300"], ["$close"], start_time='2010-01-01', end_time='2017-12-31', freq='day')`
+        # It is different from the argument `market`, which indicates a universe of stocks (e.g. **A SET** of stocks like csi300)
+        # For example, you can query all data from a stock market with the code below.
+        # ` D.features(D.instruments(market='csi300'), ["$close"], start_time='2010-01-01', end_time='2017-12-31', freq='day')`
+
+        FREQ = "day"
+        STRATEGY_CONFIG = {
+            "topk": 50,
+            "n_drop": 5,
+            # pred_score, pd.Series
+            "signal": pred_score,
+        }
+
+        EXECUTOR_CONFIG = {
+            "time_per_step": "day",
+            "generate_portfolio_metrics": True,
+        }
+
+        backtest_config = {
+            "start_time": "2017-01-01",
+            "end_time": "2020-08-01",
+            "account": 100000000,
+            "benchmark": CSI300_BENCH,
+            "exchange_kwargs": {
+                "freq": FREQ,
+                "limit_threshold": 0.095,
+                "deal_price": "close",
+                "open_cost": 0.0005,
+                "close_cost": 0.0015,
+                "min_cost": 5,
+            },
+        }
+
+        # strategy object
+        strategy_obj = TopkDropoutStrategy(**STRATEGY_CONFIG)
+        # executor object
+        executor_obj = executor.SimulatorExecutor(**EXECUTOR_CONFIG)
+        # backtest
+        portfolio_metric_dict, indicator_dict = backtest(executor=executor_obj, strategy=strategy_obj, **backtest_config)
+        analysis_freq = "{0}{1}".format(*Freq.parse(FREQ))
+        # backtest info
+        report_normal, positions_normal = portfolio_metric_dict.get(analysis_freq)
+
+        # analysis
+        analysis = dict()
+        analysis["excess_return_without_cost"] = risk_analysis(
+            report_normal["return"] - report_normal["bench"], freq=analysis_freq
+        )
+        analysis["excess_return_with_cost"] = risk_analysis(
+            report_normal["return"] - report_normal["bench"] - report_normal["cost"], freq=analysis_freq
+        )
+
+        analysis_df = pd.concat(analysis)  # type: pd.DataFrame
+        # log metrics
+        analysis_dict = flatten_dict(analysis_df["risk"].unstack().T.to_dict())
+        # print out results
+        pprint(f"The following are analysis results of benchmark return({analysis_freq}).")
+        pprint(risk_analysis(report_normal["bench"], freq=analysis_freq))
+        pprint(f"The following are analysis results of the excess return without cost({analysis_freq}).")
+        pprint(analysis["excess_return_without_cost"])
+        pprint(f"The following are analysis results of the excess return with cost({analysis_freq}).")
+        pprint(analysis["excess_return_with_cost"])
+
+
+Result
+------
+
+The backtest results are in the following form:
+
+.. code-block:: python
+
+                                                      risk
+    excess_return_without_cost mean               0.000605
+                               std                0.005481
+                               annualized_return  0.152373
+                               information_ratio  1.751319
+                               max_drawdown      -0.059055
+    excess_return_with_cost    mean               0.000410
+                               std                0.005478
+                               annualized_return  0.103265
+                               information_ratio  1.187411
+                               max_drawdown      -0.075024
+
+
+- `excess_return_without_cost`
+    - `mean`
+        Mean value of the `CAR` (cumulative abnormal return) without cost
+    - `std`
+        The `Standard Deviation` of `CAR` (cumulative abnormal return) without cost.
+    - `annualized_return`
+        The `Annualized Rate` of `CAR` (cumulative abnormal return) without cost.
+    - `information_ratio`
+        The `Information Ratio` without cost. please refer to `Information Ratio – IR <https://www.investopedia.com/terms/i/informationratio.asp>`_.
+    - `max_drawdown`
+        The `Maximum Drawdown` of `CAR` (cumulative abnormal return) without cost, please refer to `Maximum Drawdown (MDD) <https://www.investopedia.com/terms/m/maximum-drawdown-mdd.asp>`_.
+
+- `excess_return_with_cost`
+    - `mean`
+        Mean value of the `CAR` (cumulative abnormal return) series with cost
+    - `std`
+        The `Standard Deviation` of `CAR` (cumulative abnormal return) series with cost.
+    - `annualized_return`
+        The `Annualized Rate` of `CAR` (cumulative abnormal return) with cost.
+    - `information_ratio`
+        The `Information Ratio` with cost. please refer to `Information Ratio – IR <https://www.investopedia.com/terms/i/informationratio.asp>`_.
+    - `max_drawdown`
+        The `Maximum Drawdown` of `CAR` (cumulative abnormal return) with cost, please refer to `Maximum Drawdown (MDD) <https://www.investopedia.com/terms/m/maximum-drawdown-mdd.asp>`_.
 
-To know more about ``Intraday Trading``, please refer to `Intraday Trading: Model&Strategy Testing <backtest.html>`_.
 
 Reference
-===================
-To know more about ``Portfolio Strategy``, please refer to `Strategy API <../reference/api.html#module-qlib.contrib.strategy.strategy>`_.
+=========
+To know more about the `prediction score` `pred_score` output by ``Forecast Model``, please refer to `Forecast Model: Model Training & Prediction <model.html>`_.

docs/component/workflow.rst

@@ -1,12 +1,12 @@
 .. _workflow:
 
-=================================
+=============================
 Workflow: Workflow Management
-=================================
+=============================
 .. currentmodule:: qlib
 
 Introduction
-===================
+============
 
 The components in `Qlib Framework <../introduction/introduction.html#framework>`_ are designed in a loosely-coupled way. Users could build their own Quant research workflow with these components like `Example <https://github.com/microsoft/qlib/blob/main/examples/workflow_by_code.py>`_.
 
@@ -28,7 +28,7 @@ With ``qrun``, user can easily start an `execution`, which includes the followin
 For each `execution`, ``Qlib`` has a complete system to tracking all the information as well as artifacts generated during training, inference and evaluation phase. For more information about how ``Qlib`` handles this, please refer to the related document: `Recorder: Experiment Management <../component/recorder.html>`_.
 
 Complete Example
-===================
+================
 
 Before getting into details, here is a complete example of ``qrun``, which defines the workflow in typical Quant research.
 Below is a typical config file of ``qrun``.
@@ -53,8 +53,10 @@ Below is a typical config file of ``qrun``.
             kwargs:
                 topk: 50
                 n_drop: 5
+                signal:
+                    - <MODEL>
+                    - <DATASET>
         backtest:
-            verbose: False
             limit_threshold: 0.095
             account: 100000000
             benchmark: *benchmark
@@ -88,13 +90,13 @@ Below is a typical config file of ``qrun``.
                     train: [2008-01-01, 2014-12-31]
                     valid: [2015-01-01, 2016-12-31]
                     test: [2017-01-01, 2020-08-01]
-        record: 
+        record:
             - class: SignalRecord
               module_path: qlib.workflow.record_temp
               kwargs: {}
             - class: PortAnaRecord
               module_path: qlib.workflow.record_temp
-              kwargs: 
+              kwargs:
                   config: *port_analysis_config
 
 After saving the config into `configuration.yaml`, users could start the workflow and test their ideas with a single command below.
@@ -109,24 +111,62 @@ If users want to use ``qrun`` under debug mode, please use the following command
 
     python -m pdb qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
 
-.. note:: 
+.. note::
 
     `qrun` will be placed in your $PATH directory when installing ``Qlib``.
 
-.. note:: 
-        
+.. note::
+
     The symbol `&` in `yaml` file stands for an anchor of a field, which is useful when another fields include this parameter as part of the value. Taking the configuration file above as an example, users can directly change the value of `market` and `benchmark` without traversing the entire configuration file.
 
 
 Configuration File
-===================
+==================
 
 Let's get into details of ``qrun`` in this section.
-
 Before using ``qrun``, users need to prepare a configuration file. The following content shows how to prepare each part of the configuration file.
 
+The design logic of the configuration file is very simple. It predefines fixed workflows and provide this yaml interface to users to define how to initialize each component.
+It follow the design of `init_instance_by_config <https://github.com/microsoft/qlib/blob/2aee9e0145decc3e71def70909639b5e5a6f4b58/qlib/utils/__init__.py#L264>`_ .  It defines the initialization of each component of Qlib, which typically include the class and the initialization arguments.
+
+For example, the following yaml and code are equivalent.
+
+.. code-block:: YAML
+
+    model:
+        class: LGBModel
+        module_path: qlib.contrib.model.gbdt
+        kwargs:
+            loss: mse
+            colsample_bytree: 0.8879
+            learning_rate: 0.0421
+            subsample: 0.8789
+            lambda_l1: 205.6999
+            lambda_l2: 580.9768
+            max_depth: 8
+            num_leaves: 210
+            num_threads: 20
+
+
+.. code-block:: python
+
+        from qlib.contrib.model.gbdt import LGBModel
+        kwargs = {
+            "loss": "mse" ,
+            "colsample_bytree": 0.8879,
+            "learning_rate": 0.0421,
+            "subsample": 0.8789,
+            "lambda_l1": 205.6999,
+            "lambda_l2": 580.9768,
+            "max_depth": 8,
+            "num_leaves": 210,
+            "num_threads": 20,
+        }
+        LGBModel(kwargs)
+
+
 Qlib Init Section
---------------------
+-----------------
 
 At first, the configuration file needs to contain several basic parameters which will be used for qlib initialization.
 
@@ -141,21 +181,21 @@ The meaning of each field is as follows:
     Type: str. The URI of the Qlib data. For example, it could be the location where the data loaded by ``get_data.py`` are stored.
 
 - `region`
-    - If `region` == "us", ``Qlib`` will be initialized in US-stock mode. 
+    - If `region` == "us", ``Qlib`` will be initialized in US-stock mode.
     - If `region` == "cn", ``Qlib`` will be initialized in China-stock mode.
 
-    .. note:: 
-        
+    .. note::
+
         The value of `region` should be aligned with the data stored in `provider_uri`.
 
 
 Task Section
---------------------
+------------
 
 The `task` field in the configuration corresponds to a `task`, which contains the parameters of three different subsections: `Model`, `Dataset` and `Record`.
 
 Model Section
-~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~
 
 In the `task` field, the `model` section describes the parameters of the model to be used for training and inference. For more information about the base ``Model`` class, please refer to `Qlib Model <../component/model.html>`_.
 
@@ -184,16 +224,16 @@ The meaning of each field is as follows:
     Type: str. The path for the model in qlib.
 
 - `kwargs`
-    The keywords arguments for the model. Please refer to the specific model implementation for more information: `models <https://github.com/microsoft/qlib/blob/main/qlib/contrib/model>`_. 
+    The keywords arguments for the model. Please refer to the specific model implementation for more information: `models <https://github.com/microsoft/qlib/blob/main/qlib/contrib/model>`_.
+
+.. note::
 
-.. note:: 
-        
     ``Qlib`` provides a util named: ``init_instance_by_config`` to initialize any class inside ``Qlib`` with the configuration includes the fields: `class`, `module_path` and `kwargs`.
 
 Dataset Section
-~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~
 
-The `dataset` field describes the parameters for the ``Dataset`` module in ``Qlib`` as well those for the module ``DataHandler``. For more information about the ``Dataset`` module, please refer to `Qlib Model <../component/data.html#dataset>`_.
+The `dataset` field describes the parameters for the ``Dataset`` module in ``Qlib`` as well those for the module ``DataHandler``. For more information about the ``Dataset`` module, please refer to `Qlib Data <../component/data.html#dataset>`_.
 
 The keywords arguments configuration of the ``DataHandler`` is as follows:
 
@@ -208,7 +248,7 @@ The keywords arguments configuration of the ``DataHandler`` is as follows:
 
 Users can refer to the document of `DataHandler <../component/data.html#datahandler>`_ for more information about the meaning of each field in the configuration.
 
-Here is the configuration for the ``Dataset`` module which will take care of data preprossing and slicing during the training and testing phase.
+Here is the configuration for the ``Dataset`` module which will take care of data preprocessing and slicing during the training and testing phase.
 
 .. code-block:: YAML
 
@@ -226,7 +266,7 @@ Here is the configuration for the ``Dataset`` module which will take care of dat
                 test: [2017-01-01, 2020-08-01]
 
 Record Section
-~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~
 
 The `record` field is about the parameters the ``Record`` module in ``Qlib``. ``Record`` is responsible for tracking training process and results such as `information Coefficient (IC)` and `backtest` in a standard format.
 
@@ -241,8 +281,10 @@ The following script is the configuration of `backtest` and the `strategy` used
             kwargs:
                 topk: 50
                 n_drop: 5
+                signal:
+                    - <MODEL>
+                    - <DATASET>
         backtest:
-            verbose: False
             limit_threshold: 0.095
             account: 100000000
             benchmark: *benchmark
@@ -257,13 +299,13 @@ Here is the configuration details of different `Record Template` such as ``Signa
 
 .. code-block:: YAML
 
-    record: 
+    record:
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
           kwargs: {}
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
-          kwargs: 
+          kwargs:
             config: *port_analysis_config
 
 For more information about the ``Record`` module in ``Qlib``, user can refer to the related document: `Record <../component/recorder.html#record-template>`_.

docs/conf.py

@@ -54,9 +54,9 @@ master_doc = "index"
 
 
 # General information about the project.
-project = u"QLib"
-copyright = u"Microsoft"
-author = u"Microsoft"
+project = "QLib"
+copyright = "Microsoft"
+author = "Microsoft"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -174,7 +174,7 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, "qlib.tex", u"QLib Documentation", u"Microsoft", "manual"),
+    (master_doc, "qlib.tex", "QLib Documentation", "Microsoft", "manual"),
 ]
 
 
@@ -182,7 +182,7 @@ latex_documents = [
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [(master_doc, "qlib", u"QLib Documentation", [author], 1)]
+man_pages = [(master_doc, "qlib", "QLib Documentation", [author], 1)]
 
 
 # -- Options for Texinfo output -------------------------------------------
@@ -194,7 +194,7 @@ texinfo_documents = [
     (
         master_doc,
         "QLib",
-        u"QLib Documentation",
+        "QLib Documentation",
         author,
         "QLib",
         "One line description of project.",

docs/developer/code_standard_and_dev_guide.rst

@@ -0,0 +1,60 @@
+.. _code_standard:
+
+=============
+Code Standard
+=============
+
+Docstring
+=========
+Please use the `Numpydoc Style <https://stackoverflow.com/a/24385103>`_.
+
+Continuous Integration
+======================
+Continuous Integration (CI) tools help you stick to the quality standards by running tests every time you push a new commit and reporting the results to a pull request.
+
+When you submit a PR request, you can check whether your code passes the CI tests in the "check" section at the bottom of the web page.
+
+1. Qlib will check the code format with black. The PR will raise error if your code does not align to the standard of Qlib(e.g. a common error is the mixed use of space and tab).
+ You can fix the bug by inputing the following code in the command line.
+
+.. code-block:: bash
+
+    pip install black
+    python -m black . -l 120
+
+
+2. Qlib will check your code style pylint. The checking command is implemented in [github action workflow](https://github.com/microsoft/qlib/blob/0e8b94a552f1c457cfa6cd2c1bb3b87ebb3fb279/.github/workflows/test.yml#L66).
+   Sometime pylint's restrictions are not that reasonable. You can ignore specific errors like this
+
+.. code-block:: python
+
+    return -ICLoss()(pred, target, index)  # pylint: disable=E1130
+
+
+3. Qlib will check your code style flake8. The checking command is implemented in [github action workflow](https://github.com/microsoft/qlib/blob/0e8b94a552f1c457cfa6cd2c1bb3b87ebb3fb279/.github/workflows/test.yml#L73).
+ You can fix the bug by inputing the following code in the command line.
+
+.. code-block:: bash
+
+    flake8 --ignore E501,F541,E402,F401,W503,E741,E266,E203,E302,E731,E262,F523,F821,F811,F841,E713,E265,W291,E712,E722,W293 qlib
+
+
+4. Qlib has integrated pre-commit, which will make it easier for developers to format their code.
+ Just run the following two commands, and the code will be automatically formatted using black and flake8 when the git commit command is executed.
+
+.. code-block:: bash
+
+    pip install -e .[dev]
+    pre-commit install
+
+
+=================================
+Development Guidance
+=================================
+
+As a developer, you often want make changes to `Qlib` and hope it would reflect directly in your environment without reinstalling it. You can install `Qlib` in editable mode with following command.
+The `[dev]` option will help you to install some related packages when developing `Qlib` (e.g. pytest, sphinx)
+
+.. code-block:: bash
+
+    pip install -e .[dev]

docs/hidden/client.rst

@@ -1,12 +1,12 @@
 .. _client:
 
 Qlib Client-Server Framework
-===================
+============================
 
 .. currentmodule:: qlib
 
 Introduction
------------
+------------
 Client-Server is designed to solve following  problems
 
 - Manage the data in a centralized way. Users don't have to manage data of different versions.
@@ -159,13 +159,11 @@ Limitations
 2. The rolling operation expression with parameter `0` can not be updated rightly under mechanism of the client-server framework.
 
 API
-********************
+***
 
 The client is based on `python-socketio<https://python-socketio.readthedocs.io>`_ which is a framework that supports WebSocket client for Python language. The client can only propose requests and receive results, which do not include any calculating procedure.
 
 Class
---------------------
+-----
 
 .. automodule:: qlib.data.client
-
-

docs/hidden/online.rst

@@ -1,11 +1,11 @@
 .. _online:
 
 Online
-===================
+======
 .. currentmodule:: qlib
 
 Introduction
--------------------
+------------
 
 Welcome to use Online, this module simulates what will be like if we do the real trading use our model and strategy.
 
@@ -31,11 +31,11 @@ The file structure can be viewed at fileStruct_.
 
 
 Example
--------------------
+-------
 
 Let's take an example,
 
-.. note:: Make sure you have the latest version of `qlib` installed. 
+.. note:: Make sure you have the latest version of `qlib` installed.
 
 If you want to use the models and data provided by `qlib`, you only need to do as follows.
 
@@ -93,7 +93,7 @@ If Your account was saved in "./user_data/", you can see the performance of your
 Here 'SH000905' represents csi500 and 'SH000300' represents csi300
 
 Manage your account
---------------------
+-------------------
 
 Any account processed by `online` should be saved in a folder. you can use commands
 defined to manage your accounts.
@@ -161,7 +161,7 @@ be called at each trading date.
         >> online update -date 2019-10-16 -path ./user_data/
 
 API
-------------------
+---
 
 All those operations are based on defined in `qlib.contrib.online.operator`
 
@@ -170,7 +170,7 @@ All those operations are based on defined in `qlib.contrib.online.operator`
 .. _fileStruct:
 
 File structure
-------------------
+--------------
 
 'user_data' indicates the root of folder.
 Name that bold indicates it’s a folder, otherwise it’s a document.
@@ -214,7 +214,7 @@ Configuration file
 The configure file used in `online` should contain the model and strategy information.
 
 About the model
-~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~
 
 First, your configuration file needs to have a field about the model,
 this field and its contents determine the model we used when generating score at predict date.
@@ -243,7 +243,7 @@ contains 2 methods used in `online` module.
 
 
 About the strategy
-~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~
 
 Your need define the strategy used to generate the order list at predict date.
 
@@ -259,7 +259,7 @@ Followings are two examples for a TopkAmountStrategy
             n_drop: 10
 
 Generated files
-------------------
+---------------
 
 The 'online_generate' command will create the order list at {folder_path}/{user_id}/temp/,
 the name of that is orderlist_{YYYY-MM-DD}.json, YYYY-MM-DD is the date that those orders to be executed.

docs/hidden/tuner.rst

@@ -1,11 +1,11 @@
 .. _tuner:
 
 Tuner
-===================
+=====
 .. currentmodule:: qlib
 
 Introduction
--------------------
+------------
 
 Welcome to use Tuner, this document is based on that you can use Estimator proficiently and correctly.
 
@@ -31,7 +31,7 @@ Let's see an example,
 
 First make sure you have the latest version of `qlib` installed.
 
-Then, you need to privide a configuration to setup the experiment.
+Then, you need to provide a configuration to setup the experiment.
 We write a simple configuration example as following,
 
 .. code-block:: YAML
@@ -41,19 +41,19 @@ We write a simple configuration example as following,
         tuner_class: QLibTuner
     qlib_client:
         auto_mount: False
-        logging_level: INFO 
+        logging_level: INFO
     optimization_criteria:
         report_type: model
         report_factor: model_score
         optim_type: max
     tuner_pipeline:
-      - 
-        model: 
+      -
+        model:
             class: SomeModel
             space: SomeModelSpace
-        trainer: 
+        trainer:
             class: RollingTrainer
-        strategy: 
+        strategy:
             class: TopkAmountStrategy
             space: TopkAmountStrategySpace
         max_evals: 2
@@ -93,7 +93,6 @@ We write a simple configuration example as following,
                   fend_time: 2018-12-11
     backtest:
         normal_backtest_args:
-            verbose: False
             limit_threshold: 0.095
             account: 500000
             benchmark: SH000905
@@ -167,13 +166,13 @@ Also, there are some optional fields. The meaning of each field is as follows:
     The class of tuner, str type, must be an already implemented model, such as `QLibTuner` in `qlib`, or a custom tuner, but it must be a subclass of `qlib.contrib.tuner.Tuner`, the default value is `QLibTuner`.
 
 - `tuner_module_path`
-    The module path, str type, absolute url is also supported, indicates the path of the implementation of tuner. The default value is `qlib.contrib.tuner.tuner` 
+    The module path, str type, absolute url is also supported, indicates the path of the implementation of tuner. The default value is `qlib.contrib.tuner.tuner`
 
 About the optimization criteria
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You need to designate a factor to optimize, for tuner need a factor to decide which case is better than other cases.
-Usually, we use the result of `estimator`, such as backtest results and the score of model. 
+Usually, we use the result of `estimator`, such as backtest results and the score of model.
 
 This part needs contain these fields:
 
@@ -204,13 +203,13 @@ The tuner pipeline contains different tuners, and the `tuner` program will proce
 .. code-block:: YAML
 
     tuner_pipeline:
-      - 
-        model: 
+      -
+        model:
             class: SomeModel
             space: SomeModelSpace
-        trainer: 
+        trainer:
             class: RollingTrainer
-        strategy: 
+        strategy:
             class: TopkAmountStrategy
             space: TopkAmountStrategySpace
         max_evals: 2
@@ -218,13 +217,13 @@ The tuner pipeline contains different tuners, and the `tuner` program will proce
 Each part represents a tuner, and its modules which are to be tuned. Space in each part is the hyper-parameters' space of a certain module, you need to create your searching space and modify it in `/qlib/contrib/tuner/space.py`. We use `hyperopt` package to help us to construct the space, you can see the detail of how to use it in https://github.com/hyperopt/hyperopt/wiki/FMin .
 
 - model
-    You need to provide the `class` and the `space` of the model. If the model is user's own implementation, you need to privide the `module_path`. 
+    You need to provide the `class` and the `space` of the model. If the model is user's own implementation, you need to provide the `module_path`.
 
 - trainer
-    You need to proveide the `class` of the trainer. If the trainer is user's own implementation, you need to privide the `module_path`. 
+    You need to provide the `class` of the trainer. If the trainer is user's own implementation, you need to provide the `module_path`.
 
 - strategy
-    You need to provide the `class` and the `space` of the strategy. If the strategy is user's own implementation, you need to privide the `module_path`. 
+    You need to provide the `class` and the `space` of the strategy. If the strategy is user's own implementation, you need to provide the `module_path`.
 
 - data_label
     The label of the data, you can search which kinds of labels will lead to a better result. This part is optional, and you only need to provide `space`.
@@ -250,31 +249,31 @@ You need to use the same dataset to evaluate your different `estimator` experime
         test_start_date: 2016-07-01
         test_end_date: 2018-04-30
 
-- `rolling_period`    
+- `rolling_period`
     The rolling period, integer type, indicates how many time steps need rolling when rolling the data. The default value is `60`. If you use `RollingTrainer`, this config will be used, or it will be ignored.
 
 - `train_start_date`
     Training start time, str type.
 
-- `train_end_date`      
+- `train_end_date`
     Training end time, str type.
 
-- `validate_start_date`    
+- `validate_start_date`
     Validation start time, str type.
 
-- `validate_end_date`    
+- `validate_end_date`
     Validation end time, str type.
 
-- `test_start_date`    
+- `test_start_date`
     Test start time, str type.
 
-- `test_end_date`     
+- `test_end_date`
     Test end time, str type. If `test_end_date` is `-1` or greater than the last date of the data, the last date of the data will be used as `test_end_date`.
 
 About the data and backtest
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-`data` and `backtest` are all same in the whole `tuner` experiment. Different `estimator` experiments must use the same data and backtest method. So, these two parts of config are same with that in `estimator` configuration. You can see the precise defination of these parts in `estimator` introduction. We only provide an example here.
+`data` and `backtest` are all same in the whole `tuner` experiment. Different `estimator` experiments must use the same data and backtest method. So, these two parts of config are same with that in `estimator` configuration. You can see the precise definition of these parts in `estimator` introduction. We only provide an example here.
 
 .. code-block:: YAML
 
@@ -306,7 +305,6 @@ About the data and backtest
                   fend_time: 2018-12-11
     backtest:
         normal_backtest_args:
-            verbose: False
             limit_threshold: 0.095
             account: 500000
             benchmark: SH000905
@@ -317,11 +315,10 @@ About the data and backtest
 Experiment Result
 -----------------
 
-All the results are stored in experiment file directly, you can check them directly in the corresponding files. 
+All the results are stored in experiment file directly, you can check them directly in the corresponding files.
 What we save are as following:
 
 - Global optimal parameters
 - Local optimal parameters of each tuner
 - Config file of this `tuner` experiment
 - Every `estimator` experiments result in the process
-

docs/index.rst

@@ -1,6 +1,6 @@
-============================================================
+======================
 ``Qlib`` Documentation
-============================================================
+======================
 
 ``Qlib`` is an AI-oriented quantitative investment platform, which aims to realize the potential, empower the research, and create the value of AI technologies in quantitative investment.
 
@@ -24,22 +24,23 @@ Document Structure
 .. toctree::
    :maxdepth: 3
    :caption: FIRST STEPS:
-   
+
    Installation <start/installation.rst>
    Initialization <start/initialization.rst>
    Data Retrieval <start/getdata.rst>
    Custom Model Integration <start/integration.rst>
-   
+
 
 .. toctree::
    :maxdepth: 3
    :caption: COMPONENTS:
 
    Workflow: Workflow Management <component/workflow.rst>
-   Data Layer: Data Framework&Usage <component/data.rst>
+   Data Layer: Data Framework & Usage <component/data.rst>
    Forecast Model: Model Training & Prediction <component/model.rst>
-   Strategy: Portfolio Management <component/strategy.rst>
-   Intraday Trading: Model&Strategy Testing <component/backtest.rst>
+   Portfolio Management and Backtest <component/strategy.rst>
+   Nested Decision Execution: High-Frequency Trading <component/highfreq.rst>
+   Meta Controller: Meta-Task & Meta-Dataset & Meta-Model <component/meta.rst>
    Qlib Recorder: Experiment Management <component/recorder.rst>
    Analysis: Evaluation & Results Analysis <component/report.rst>
    Online Serving: Online Management & Strategy & Tool <component/online.rst>
@@ -47,11 +48,12 @@ Document Structure
 .. toctree::
    :maxdepth: 3
    :caption: ADVANCED TOPICS:
-   
+
    Building Formulaic Alphas <advanced/alpha.rst>
    Online & Offline mode <advanced/server.rst>
    Serialization <advanced/serial.rst>
    Task Management <advanced/task_management.rst>
+   Point-In-Time database <advanced/PIT.rst>
 
 .. toctree::
    :maxdepth: 3

docs/introduction/introduction.rst

@@ -3,7 +3,7 @@
 ===============================
 
 Introduction
-===================
+============
 
 .. image:: ../_static/img/logo/white_bg_rec+word.png
     :align: center
@@ -13,9 +13,9 @@ Introduction
 With ``Qlib``, users can easily try their ideas to create better Quant investment strategies.
 
 Framework
-===================
-   
-.. image:: ../_static/img/framework.png
+=========
+
+.. image:: ../_static/img/framework.svg
     :align: center
 
 
@@ -27,16 +27,21 @@ At the module level, Qlib is a platform that consists of above components. The c
 Name                      Description
 ========================  ==============================================================================
 `Infrastructure` layer    `Infrastructure` layer provides underlying support for Quant research.
-                          `DataServer` provides high-performance infrastructure for users to manage 
+                          `DataServer` provides high-performance infrastructure for users to manage
                           and retrieve raw data. `Trainer` provides flexible interface to control
                           the training process of models which enable algorithms controlling the
                           training process.
 
 `Workflow` layer          `Workflow` layer covers the whole workflow of quantitative investment.
                           `Information Extractor` extracts data for models. `Forecast Model` focuses
-                          on producing all kinds of forecast signals (e.g. _alpha_, risk) for other
-                          modules. With these signals `Portfolio Generator` will generate the target
-                          portfolio and produce orders to be executed by `Order Executor`.
+                          on producing all kinds of forecast signals (e.g. *alpha*, risk) for other
+                          modules.  With these signals `Decision Generator` will generate the target
+                          trading decisions(i.e. portfolio, orders)  to be executed by `Execution Env`
+                          (i.e. the trading market).  There may be multiple levels of `Trading Agent`
+                          and `Execution Env` (e.g. an *order executor trading agent and intraday
+                          order execution environment* could behave like an interday trading
+                          environment and nested in  *daily portfolio management trading agent and
+                          interday trading environment*  )
 
 `Interface` layer         `Interface` layer tries to present a user-friendly interface for the underlying
                           system. `Analyser` module will provide users detailed analysis reports of

docs/introduction/quick.rst

@@ -1,10 +1,10 @@
 
-===============================
+===========
 Quick Start
-===============================
+===========
 
 Introduction
-==============
+============
 
 This ``Quick Start`` guide tries to demonstrate
 
@@ -14,7 +14,7 @@ This ``Quick Start`` guide tries to demonstrate
 
 
 Installation
-==================
+============
 
 Users can easily intsall ``Qlib`` according to the following steps:
 
@@ -31,10 +31,10 @@ Users can easily intsall ``Qlib`` according to the following steps:
         git clone https://github.com/microsoft/qlib.git && cd qlib
         python setup.py install
 
-To kown more about `installation`, please refer to `Qlib Installation <../start/installation.html>`_.
+To known more about `installation`, please refer to `Qlib Installation <../start/installation.html>`_.
 
 Prepare Data
-==============
+============
 
 Load and prepare data by running the following code:
 
@@ -44,17 +44,17 @@ Load and prepare data by running the following code:
 
 This dataset is created by public data collected by crawler scripts in ``scripts/data_collector/``, which have been released in the same repository. Users could create the same dataset with it.
 
-To kown more about `prepare data`, please refer to `Data Preparation <../component/data.html#data-preparation>`_.
+To known more about `prepare data`, please refer to `Data Preparation <../component/data.html#data-preparation>`_.
 
 Auto Quant Research Workflow
-====================================
+============================
 
-``Qlib`` provides a tool named ``qrun`` to run the whole workflow automatically (including building dataset, training models, backtest and evaluation). Users can start an auto quant research workflow and have a graphical reports analysis according to the following steps: 
+``Qlib`` provides a tool named ``qrun`` to run the whole workflow automatically (including building dataset, training models, backtest and evaluation). Users can start an auto quant research workflow and have a graphical reports analysis according to the following steps:
 
-- Quant Research Workflow: 
+- Quant Research Workflow:
     - Run  ``qrun`` with a config file of the LightGBM model `workflow_config_lightgbm.yaml` as following.
 
-        .. code-block:: 
+        .. code-block::
 
             cd examples  # Avoid running program under the directory contains `qlib`
             qrun benchmarks/LightGBM/workflow_config_lightgbm.yaml
@@ -64,7 +64,7 @@ Auto Quant Research Workflow
         The result of ``qrun`` is as follows, which is also the typical result of ``Forecast model(alpha)``. Please refer to  `Intraday Trading <../component/backtest.html>`_. for more details about the result.
 
         .. code-block:: python
-        
+
                                                               risk
             excess_return_without_cost mean               0.000605
                                        std                0.005481
@@ -77,7 +77,7 @@ Auto Quant Research Workflow
                                        information_ratio  1.187411
                                        max_drawdown      -0.075024
 
-        
+
     To know more about `workflow` and `qrun`, please refer to `Workflow: Workflow Management <../component/workflow.html>`_.
 
 - Graphical Reports Analysis:
@@ -89,6 +89,6 @@ Auto Quant Research Workflow
 
 
 Custom Model Integration
-===============================================
+========================
 
 ``Qlib`` provides a batch of models (such as ``lightGBM`` and ``MLP`` models) as examples of ``Forecast Model``. In addition to the default model, users can integrate their own custom models into ``Qlib``. If users are interested in the custom model, please refer to `Custom Model Integration <../start/integration.html>`_.

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/reference/api.rst

@@ -1,7 +1,7 @@
 .. _api:
-================================
+=============
 API Reference
-================================
+=============
 
 
 
@@ -9,32 +9,32 @@ Here you can find all ``Qlib`` interfaces.
 
 
 Data
-====================
+====
 
 Provider
---------------------
+--------
 
 .. automodule:: qlib.data.data
     :members:
-		
+
 Filter
---------------------
+------
 
 .. automodule:: qlib.data.filter
     :members:
 
 Class
---------------------
+-----
 .. automodule:: qlib.data.base
     :members:
 
 Operator
---------------------
+--------
 .. automodule:: qlib.data.ops
     :members:
-	       
+
 Cache
-----------------
+-----
 .. autoclass:: qlib.data.cache.MemCacheUnit
     :members:
 
@@ -55,7 +55,7 @@ Cache
 
 
 Storage
--------------
+-------
 .. autoclass:: qlib.data.storage.storage.BaseStorage
     :members:
 
@@ -82,52 +82,52 @@ Storage
 
 
 Dataset
----------------
+-------
 
 Dataset Class
-~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~
 .. automodule:: qlib.data.dataset.__init__
     :members:
 
 Data Loader
-~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~
 .. automodule:: qlib.data.dataset.loader
     :members:
 
 Data Handler
-~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~
 .. automodule:: qlib.data.dataset.handler
     :members:
 
 Processor
-~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~
 .. automodule:: qlib.data.dataset.processor
     :members:
 
 
 Contrib
-====================
+=======
 
 Model
---------------------
+-----
 .. automodule:: qlib.model.base
     :members:
 
 Strategy
--------------------
+--------
 
 .. automodule:: qlib.contrib.strategy.strategy
     :members:
 
 Evaluate
------------------
+--------
 
 .. automodule:: qlib.contrib.evaluate
     :members:
-    
+
 
 Report
------------------
+------
 
 .. automodule:: qlib.contrib.report.analysis_position.report
     :members:
@@ -159,102 +159,100 @@ Report
 
 
 Workflow
-====================
+========
 
 
 Experiment Manager
---------------------
+------------------
 .. autoclass:: qlib.workflow.expm.ExpManager
     :members:
 
 Experiment
---------------------
+----------
 .. autoclass:: qlib.workflow.exp.Experiment
     :members:
 
 Recorder
---------------------
+--------
 .. autoclass:: qlib.workflow.recorder.Recorder
     :members:
 
 Record Template
---------------------
+---------------
 .. automodule:: qlib.workflow.record_temp
     :members:
 
 Task Management
-====================
+===============
 
 
 TaskGen
---------------------
+-------
 .. automodule:: qlib.workflow.task.gen
     :members:
 
 TaskManager
---------------------
+-----------
 .. automodule:: qlib.workflow.task.manage
     :members:
 
 Trainer
---------------------
+-------
 .. automodule:: qlib.model.trainer
     :members:
 
 Collector
---------------------
+---------
 .. automodule:: qlib.workflow.task.collect
     :members:
 
 Group
---------------------
+-----
 .. automodule:: qlib.model.ens.group
     :members:
 
 Ensemble
---------------------
+--------
 .. automodule:: qlib.model.ens.ensemble
     :members:
 
 Utils
---------------------
+-----
 .. automodule:: qlib.workflow.task.utils
     :members:
 
 
 Online Serving
-====================
+==============
 
 
 Online Manager
---------------------
+--------------
 .. automodule:: qlib.workflow.online.manager
     :members:
 
 Online Strategy
---------------------
+---------------
 .. automodule:: qlib.workflow.online.strategy
     :members:
 
 Online Tool
---------------------
+-----------
 .. automodule:: qlib.workflow.online.utils
     :members:
 
+
 RecordUpdater
---------------------
+-------------
 .. automodule:: qlib.workflow.online.update
     :members:
 
 
 Utils
-====================
+=====
 
 Serializable
---------------------
+------------
 
 .. automodule:: qlib.utils.serial.Serializable
     :members:
-
-
-    

docs/requirements.txt

@@ -3,3 +3,4 @@ cmake
 numpy
 scipy
 scikit-learn
+pandas

docs/start/getdata.rst

@@ -1,18 +1,18 @@
 .. _getdata:
 
-=============================
+==============
 Data Retrieval
-=============================
+==============
 
 .. currentmodule:: qlib
 
 Introduction
-====================
+============
 
 Users can get stock data with ``Qlib``. The following examples demonstrate the basic user interface.
 
 Examples
-====================
+========
 
 
 ``QLib`` Initialization:
@@ -30,7 +30,7 @@ If users followed steps in `initialization <initialization.html>`_ and downloade
 Load trading calendar with given time range and frequency:
 
 .. code-block:: python
-		
+
    >> from qlib.data import D
    >> D.calendar(start_time='2010-01-01', end_time='2017-12-31', freq='day')[:2]
    [Timestamp('2010-01-04 00:00:00'), Timestamp('2010-01-05 00:00:00')]
@@ -46,7 +46,7 @@ Parse a given market name into a stock pool config:
 Load instruments of certain stock pool in the given time range:
 
 .. code-block:: python
-		
+
    >> from qlib.data import D
    >> instruments = D.instruments(market='csi300')
    >> D.list_instruments(instruments=instruments, start_time='2010-01-01', end_time='2017-12-31', as_list=True)[:6]
@@ -79,14 +79,14 @@ For more details about filter, please refer `Filter API <../component/data.html>
 Load features of certain instruments in a given time range:
 
 .. code-block:: python
-		
+
    >> from qlib.data import D
    >> instruments = ['SH600000']
    >> fields = ['$close', '$volume', 'Ref($close, 1)', 'Mean($close, 3)', '$high-$low']
    >> D.features(instruments, fields, start_time='2010-01-01', end_time='2017-12-31', freq='day').head()
-                              
+
                               $close     $volume  Ref($close, 1)  Mean($close, 3)  $high-$low
-      instrument  datetime                                                                      
+      instrument  datetime
       SH600000    2010-01-04  86.778313  16162960.0       88.825928        88.061483    2.907631
                   2010-01-05  87.433578  28117442.0       86.778313        87.679273    3.235252
                   2010-01-06  85.713585  23632884.0       87.433578        86.641825    1.720009
@@ -108,7 +108,7 @@ Load features of certain stock pool in a given time range:
    >> D.features(instruments, fields, start_time='2010-01-01', end_time='2017-12-31', freq='day').head()
 
                                  $close        $volume  Ref($close, 1)  Mean($close, 3)  $high-$low
-      instrument  datetime                                                                           
+      instrument  datetime
       SH600655    2010-01-04  2699.567383  158193.328125     2619.070312      2626.097738  124.580566
                   2010-01-08  2612.359619   77501.406250     2584.567627      2623.220133   83.373047
                   2010-01-11  2712.982422  160852.390625     2612.359619      2636.636556  146.621582
@@ -120,6 +120,32 @@ For more details about features, please refer `Feature API <../component/data.ht
 
 .. note:: When calling `D.features()` at the client, use parameter `disk_cache=0` to skip dataset cache, use `disk_cache=1` to generate and use dataset cache. In addition, when calling at the server, users can use `disk_cache=2` to update the dataset cache.
 
+
+When you are building complicated expressions, implementing all the expressions in a single string may not be easy.
+For example, it looks quite long and complicated:
+
+.. code-block:: python
+
+   >> from qlib.data import D
+   >> data = D.features(["sh600519"], ["(($high / $close) + ($open / $close)) * (($high / $close) + ($open / $close)) / (($high / $close) + ($open / $close))"], start_time="20200101")
+
+
+But using string is not the only way to implement the expression. You can also implement expression by code.
+Here is an exmaple which does the same thing as above examples.
+
+
+.. code-block:: python
+
+   >> from qlib.data.ops import *
+   >> f1 = Feature("high") / Feature("close")
+   >> f2 = Feature("open") / Feature("close")
+   >> f3 = f1 + f2
+   >> f4 = f3 * f3 / f3
+
+   >> data = D.features(["sh600519"], [f4], start_time="20200101")
+   >> data.head()
+
+
 API
-====================
+===
 To know more about how to use the Data, go to API Reference: `Data API <../reference/api.html#data>`_

docs/start/initialization.rst

@@ -1,23 +1,23 @@
 .. _initialization:
 
-====================
+===================
 Qlib Initialization
-====================
+===================
 
 .. currentmodule:: qlib
 
 
 Initialization
-=========================
+==============
 
 Please follow the steps below to initialize ``Qlib``.
 
 Download and prepare the Data: execute the following command to download stock data. Please pay `attention` that the data is collected from `Yahoo Finance <https://finance.yahoo.com/lookup>`_ and the data might not be perfect. We recommend users to prepare their own data if they have high-quality datasets. Please refer to `Data <../component/data.html#converting-csv-format-into-qlib-format>`_ for more information about customized dataset.
-    
+
     .. code-block:: bash
-    
+
         python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn
-        
+
 Please refer to `Data Preparation <../component/data.html#data-preparation>`_ for more information about `get_data.py`,
 
 
@@ -27,43 +27,45 @@ Initialize Qlib before calling other APIs: run following code in python.
 
         import qlib
         # region in [REG_CN, REG_US]
-        from qlib.config import REG_CN
+        from qlib.constant import REG_CN
         provider_uri = "~/.qlib/qlib_data/cn_data"  # target_dir
         qlib.init(provider_uri=provider_uri, region=REG_CN)
-    
+
 .. note::
    Do not import qlib package in the repository directory  of ``Qlib``, otherwise, errors may occur.
 
 Parameters
 -------------------
 
-Besides `provider_uri` and `region`, `qlib.init` has other parameters. The following are several important parameters of `qlib.init`:
+Besides `provider_uri` and `region`, `qlib.init` has other parameters.
+The following are several important parameters of `qlib.init` (`Qlib` has a lot of config. Only part of parameters are limited here. More detailed setting can be found `here <https://github.com/microsoft/qlib/blob/main/qlib/config.py>`_):
 
 - `provider_uri`
     Type: str. The URI of the Qlib data. For example, it could be the location where the data loaded by ``get_data.py`` are stored.
 - `region`
-    Type: str, optional parameter(default: `qlib.config.REG_CN`).
-        Currently: ``qlib.config.REG_US`` ('us') and ``qlib.config.REG_CN`` ('cn') is supported. Different value of  `region` will result in different stock market mode.
-        - ``qlib.config.REG_US``: US stock market.
-        - ``qlib.config.REG_CN``: China stock market.
+    Type: str, optional parameter(default: `qlib.constant.REG_CN`).
+        Currently: ``qlib.constant.REG_US`` ('us') and ``qlib.constant.REG_CN`` ('cn') is supported. Different value of  `region` will result in different stock market mode.
+        - ``qlib.constant.REG_US``: US stock market.
+        - ``qlib.constant.REG_CN``: China stock market.
 
         Different modes will result in different trading limitations and costs.
+        The region is just `shortcuts for defining a batch of configurations <https://github.com/microsoft/qlib/blob/528f74af099bf6156e9480bcd2bb28e453231212/qlib/config.py#L249>`_, which include minimal trading order unit (``trade_unit``),  trading limitation (``limit_threshold``) , etc.  It is not a necessary part and users can set the key configurations manually if the existing region setting can't meet their requirements.
 - `redis_host`
     Type: str, optional parameter(default: "127.0.0.1"), host of `redis`
         The lock and cache mechanism relies on redis.
 - `redis_port`
     Type: int, optional parameter(default: 6379), port of `redis`
 
-    .. note:: 
-        
+    .. note::
+
         The value of `region` should be aligned with the data stored in `provider_uri`. Currently, ``scripts/get_data.py`` only provides China stock market data. If users want to use the US stock market data, they should prepare their own US-stock data in `provider_uri` and switch to US-stock mode.
 
     .. note::
-        
+
         If Qlib fails to connect redis via `redis_host` and `redis_port`, cache mechanism will not be used! Please refer to `Cache <../component/data.html#cache>`_ for details.
 - `exp_manager`
     Type: dict, optional parameter, the setting of `experiment manager` to be used in qlib. Users can specify an experiment manager class, as well as the tracking URI for all the experiments. However, please be aware that we only support input of a dictionary in the following style for `exp_manager`. For more information about `exp_manager`, users can refer to `Recorder: Experiment Management <../component/recorder.html>`_.
-    
+
     .. code-block:: Python
 
         # For example, if you want to set your tracking_uri to a <specific folder>, you can initialize qlib below
@@ -76,8 +78,9 @@ Besides `provider_uri` and `region`, `qlib.init` has other parameters. The follo
             }
         })
 - `mongo`
-    Type: dict, optional parameter, the setting of `MongoDB <https://www.mongodb.com/>`_ which will be used in some features such as `Task Management <../advanced/task_management.html>`_, with high performance and clustered processing. 
-    Users need finished `installation <https://www.mongodb.com/try/download/community>`_ firstly, and run it in a fixed URL.
+    Type: dict, optional parameter, the setting of `MongoDB <https://www.mongodb.com/>`_ which will be used in some features such as `Task Management <../advanced/task_management.html>`_, with high performance and clustered processing.
+    Users need to follow the steps in  `installation <https://www.mongodb.com/try/download/community>`_  to install MongoDB firstly and then access it via a URI.
+    Users can access mongodb with credential by setting "task_url"  to a string like `"mongodb://%s:%s@%s" % (user, pwd, host + ":" + port)`.
 
     .. code-block:: Python
 
@@ -86,3 +89,9 @@ Besides `provider_uri` and `region`, `qlib.init` has other parameters. The follo
             "task_url": "mongodb://localhost:27017/",  # your mongo url
             "task_db_name": "rolling_db", # the database name of Task Management
         })
+
+- `logging_level`
+    The logging level for the system.
+
+- `kernels`
+    The number of processes used when calculating features in Qlib's expression engine. It is very helpful to set it to 1 when you are debuggin an expression calculating exception

docs/start/installation.rst

@@ -1,8 +1,8 @@
 .. _installation:
 
-====================
+============
 Installation
-====================
+============
 
 .. currentmodule:: qlib
 
@@ -24,7 +24,7 @@ Also, Users can install ``Qlib`` by the source code according to the following s
 
 - Enter the root directory of ``Qlib``, in which the file ``setup.py`` exists.
 - Then, please execute the following command to install the environment dependencies and install ``Qlib``:
-   
+
    .. code-block:: bash
 
       $ pip install numpy
@@ -34,7 +34,7 @@ Also, Users can install ``Qlib`` by the source code according to the following s
 
 .. note::
    It's recommended to use anaconda/miniconda to setup the environment. ``Qlib`` needs lightgbm and pytorch packages, use pip to install them.
-   
+
 
 
 Use the following code to make sure the installation successful:
@@ -44,6 +44,3 @@ Use the following code to make sure the installation successful:
    >>> import qlib
    >>> qlib.__version__
    <LATEST VERSION>
-
-
-=====================

docs/start/integration.rst

@@ -1,9 +1,9 @@
-=========================================
+========================
 Custom Model Integration
-=========================================
+========================
 
 Introduction
-===================
+============
 
 ``Qlib``'s `Model Zoo` includes models such as ``LightGBM``, ``MLP``, ``LSTM``, etc.. These models are examples of ``Forecast Model``. In addition to the default models ``Qlib`` provide, users can integrate their own custom models into ``Qlib``.
 
@@ -14,7 +14,7 @@ Users can integrate their own custom models according to the following steps.
 - Test the custom model.
 
 Custom Model Class
-===========================
+==================
 The Custom models need to inherit `qlib.model.base.Model <../reference/api.html#module-qlib.model.base>`_ and override the methods in it.
 
 - Override the `__init__` method
@@ -36,7 +36,7 @@ The Custom models need to inherit `qlib.model.base.Model <../reference/api.html#
     - The parameters could include some `optional` parameters with default values, such as `num_boost_round = 1000` for `GBDT`.
     - Code Example: In the following example, `num_boost_round = 1000` is an optional parameter.
     .. code-block:: Python
-    
+
         def fit(self, dataset: DatasetH, num_boost_round = 1000, **kwargs):
 
             # prepare dataset for lgb training and evaluation
@@ -101,14 +101,14 @@ The Custom models need to inherit `qlib.model.base.Model <../reference/api.html#
             )
 
 Configuration File
-=======================
+==================
 
 The configuration file is described in detail in the `Workflow <../component/workflow.html#complete-example>`_ document. In order to integrate the custom model into ``Qlib``, users need to modify the "model" field in the configuration file. The configuration describes which models to use and how we can initialize it.
 
-- Example: The following example describes the `model` field of configuration file about the custom lightgbm model mentioned above, where `module_path` is the module path, `class` is the class name, and `args` is the hyperparameter passed into the __init__ method. All parameters in the field is passed to `self._params` by `\*\*kwargs` in `__init__` except `loss = mse`. 
+- Example: The following example describes the `model` field of configuration file about the custom lightgbm model mentioned above, where `module_path` is the module path, `class` is the class name, and `args` is the hyperparameter passed into the __init__ method. All parameters in the field is passed to `self._params` by `\*\*kwargs` in `__init__` except `loss = mse`.
 
 .. code-block:: YAML
-    
+
     model:
         class: LGBModel
         module_path: qlib.contrib.model.gbdt
@@ -126,7 +126,7 @@ The configuration file is described in detail in the `Workflow <../component/wor
 Users could find configuration file of the baselines of the ``Model`` in ``examples/benchmarks``. All the configurations of different models are listed under the corresponding model folder.
 
 Model Testing
-=====================
+=============
 Assuming that the configuration file is ``examples/benchmarks/LightGBM/workflow_config_lightgbm.yaml``, users can run the following command to test the custom model:
 
 .. code-block:: bash
@@ -136,10 +136,10 @@ Assuming that the configuration file is ``examples/benchmarks/LightGBM/workflow_
 
 .. note:: ``qrun`` is a built-in command of ``Qlib``.
 
-Also, ``Model`` can also be tested as a single module. An example has been given in ``examples/workflow_by_code.ipynb``. 
+Also, ``Model`` can also be tested as a single module. An example has been given in ``examples/workflow_by_code.ipynb``.
 
 
 Reference
-=====================
+=========
 
 To know more about ``Forecast Model``, please refer to `Forecast Model: Model Training & Prediction <../component/model.html>`_ and `Model API <../reference/api.html#module-qlib.model.base>`_.

examples/benchmarks/ADARNN/README.md

@@ -0,0 +1,4 @@
+# AdaRNN
+* Code: [https://github.com/jindongwang/transferlearning/tree/master/code/deep/adarnn](https://github.com/jindongwang/transferlearning/tree/master/code/deep/adarnn)
+* Paper: [AdaRNN: Adaptive Learning and Forecasting for Time Series](https://arxiv.org/pdf/2108.04443.pdf).
+

examples/benchmarks/ADARNN/requirements.txt

@@ -0,0 +1,4 @@
+pandas==1.1.2
+numpy==1.21.0
+scikit_learn==0.23.2
+torch==1.7.0

examples/benchmarks/ADARNN/workflow_config_adarnn_Alpha360.yaml

@@ -0,0 +1,88 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market csi300
+benchmark: &benchmark SH000300
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    infer_processors:
+        - class: RobustZScoreNorm
+          kwargs:
+              fields_group: feature
+              clip_outlier: true
+        - class: Fillna
+          kwargs:
+              fields_group: feature
+    learn_processors:
+        - class: DropnaLabel
+        - class: CSRankNorm
+          kwargs:
+              fields_group: label
+    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            model: <MODEL>
+            dataset: <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: ADARNN
+        module_path: qlib.contrib.model.pytorch_adarnn
+        kwargs:
+            d_feat: 6
+            hidden_size: 64
+            num_layers: 2
+            dropout: 0.0
+            n_epochs: 200
+            lr: 1e-3
+            early_stop: 20
+            batch_size: 800
+            metric: loss
+            loss: mse
+            GPU: 0
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha360
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

examples/benchmarks/ADD/README.md

@@ -0,0 +1,3 @@
+# ADD
+* Paper: [ADD: Augmented Disentanglement Distillation Framework for Improving Stock Trend Forecasting](https://arxiv.org/abs/2012.06289).
+

examples/benchmarks/ADD/requirements.txt

@@ -0,0 +1,4 @@
+numpy==1.21.0
+pandas==1.1.2
+scikit_learn==0.23.2
+torch==1.7.0

examples/benchmarks/ADD/workflow_config_add_Alpha360.yaml

@@ -0,0 +1,94 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market csi300
+benchmark: &benchmark SH000300
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    infer_processors:
+        - class: RobustZScoreNorm
+          kwargs:
+              fields_group: feature
+              clip_outlier: true
+        - class: Fillna
+          kwargs:
+              fields_group: feature
+    learn_processors:
+        - class: DropnaLabel
+        - class: CSRankNorm
+          kwargs:
+              fields_group: label
+    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL>
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: ADD
+        module_path: qlib.contrib.model.pytorch_add
+        kwargs:
+            d_feat: 6
+            hidden_size: 64
+            num_layers: 2
+            dropout: 0.1
+            dec_dropout: 0.0
+            n_epochs: 200
+            lr: 1e-3
+            early_stop: 20
+            batch_size: 5000
+            metric: ic
+            base_model: GRU
+            gamma: 0.1
+            gamma_clip: 0.2
+            optimizer: adam
+            mu: 0.2
+            GPU: 0
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha360
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record:
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs:
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs:
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs:
+            config: *port_analysis_config

examples/benchmarks/ALSTM/README.md

@@ -6,3 +6,4 @@
 
   [https://www.ijcai.org/Proceedings/2017/0366.pdf](https://www.ijcai.org/Proceedings/2017/0366.pdf)
 
+- NOTE: Current version of implementation is just a simplified version of ALSTM. It is an LSTM with attention.

examples/benchmarks/ALSTM/requirements.txt

@@ -1,4 +1,4 @@
-numpy==1.17.4
+numpy==1.21.0
 pandas==1.1.2
 scikit_learn==0.23.2
 torch==1.7.0

examples/benchmarks/ALSTM/workflow_config_alstm_Alpha158.yaml

@@ -34,19 +34,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: ALSTM
@@ -81,7 +86,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/ALSTM/workflow_config_alstm_Alpha360.yaml

@@ -26,19 +26,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: ALSTM
@@ -71,7 +76,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -80,4 +87,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config

examples/benchmarks/CatBoost/requirements.txt

@@ -1,3 +1,3 @@
 pandas==1.1.2
-numpy==1.17.4
+numpy==1.21.0
 catboost==0.24.3

examples/benchmarks/CatBoost/workflow_config_catboost_Alpha158.yaml

@@ -12,19 +12,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: CatBoostModel
@@ -53,7 +58,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/CatBoost/workflow_config_catboost_Alpha158_csi500.yaml

@@ -0,0 +1,72 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market csi500
+benchmark: &benchmark SH000905
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: CatBoostModel
+        module_path: qlib.contrib.model.catboost_model
+        kwargs:
+            loss: RMSE
+            learning_rate: 0.0421
+            subsample: 0.8789
+            max_depth: 6
+            num_leaves: 100
+            thread_count: 20
+            grow_policy: Lossguide
+            bootstrap_type: Poisson
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha158
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

examples/benchmarks/CatBoost/workflow_config_catboost_Alpha360.yaml

@@ -19,19 +19,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: CatBoostModel
@@ -60,7 +65,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/CatBoost/workflow_config_catboost_Alpha360_csi500.yaml

@@ -0,0 +1,79 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market csi500
+benchmark: &benchmark SH000905
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    infer_processors: []
+    learn_processors:
+        - class: DropnaLabel
+        - class: CSRankNorm
+          kwargs:
+              fields_group: label
+    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: CatBoostModel
+        module_path: qlib.contrib.model.catboost_model
+        kwargs:
+            loss: RMSE
+            learning_rate: 0.0421
+            subsample: 0.8789
+            max_depth: 6
+            num_leaves: 100
+            thread_count: 20
+            grow_policy: Lossguide
+            bootstrap_type: Poisson
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha360
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

examples/benchmarks/DoubleEnsemble/requirements.txt

@@ -1,3 +1,3 @@
 pandas==1.1.2
-numpy==1.17.4
+numpy==1.21.0
 lightgbm==3.1.0

examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha158.yaml

@@ -12,19 +12,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: DEnsembleModel
@@ -32,7 +37,7 @@ task:
         kwargs:
             base_model: "gbm"
             loss: mse
-            num_models: 6
+            num_models: 3
             enable_sr: True
             enable_fs: True
             alpha1: 1
@@ -48,11 +53,8 @@ task:
                 - 0.4
             sub_weights:
                 - 1
-                - 0.2
-                - 0.2
-                - 0.2
-                - 0.2
-                - 0.2
+                - 1
+                - 1
             epochs: 28
             colsample_bytree: 0.8879
             learning_rate: 0.2
@@ -75,16 +77,18 @@ task:
                 train: [2008-01-01, 2014-12-31]
                 valid: [2015-01-01, 2016-12-31]
                 test: [2017-01-01, 2020-08-01]
-    record:
+    record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
-          kwargs:
+          kwargs: 
             ana_long_short: False
             ann_scaler: 252
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
-          kwargs:
+          kwargs: 
             config: *port_analysis_config

examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha158_csi500.yaml

@@ -0,0 +1,97 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market csi500
+benchmark: &benchmark SH000905
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: DEnsembleModel
+        module_path: qlib.contrib.model.double_ensemble
+        kwargs:
+            base_model: "gbm"
+            loss: mse
+            num_models: 6
+            enable_sr: True
+            enable_fs: True
+            alpha1: 1
+            alpha2: 1
+            bins_sr: 10
+            bins_fs: 5
+            decay: 0.5
+            sample_ratios:
+                - 0.8
+                - 0.7
+                - 0.6
+                - 0.5
+                - 0.4
+            sub_weights:
+                - 1
+                - 0.2
+                - 0.2
+                - 0.2
+                - 0.2
+                - 0.2
+            epochs: 28
+            colsample_bytree: 0.8879
+            learning_rate: 0.2
+            subsample: 0.8789
+            lambda_l1: 205.6999
+            lambda_l2: 580.9768
+            max_depth: 8
+            num_leaves: 210
+            num_threads: 20
+            verbosity: -1
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha158
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha360.yaml

@@ -19,19 +19,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: DEnsembleModel
@@ -39,7 +44,7 @@ task:
         kwargs:
             base_model: "gbm"
             loss: mse
-            num_models: 6
+            num_models: 3
             enable_sr: True
             enable_fs: True
             alpha1: 1
@@ -55,11 +60,8 @@ task:
                 - 0.4
             sub_weights:
                 - 1
-                - 0.2
-                - 0.2
-                - 0.2
-                - 0.2
-                - 0.2
+                - 1
+                - 1
             epochs: 136
             colsample_bytree: 0.8879
             learning_rate: 0.0421
@@ -82,10 +84,12 @@ task:
                 train: [2008-01-01, 2014-12-31]
                 valid: [2015-01-01, 2016-12-31]
                 test: [2017-01-01, 2020-08-01]
-    record:
+    record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs:
@@ -93,5 +97,5 @@ task:
             ann_scaler: 252
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
-          kwargs:
-            config: *port_analysis_config
+          kwargs: 
+            config: *port_analysis_config

examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha360_csi500.yaml

@@ -0,0 +1,104 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market csi500
+benchmark: &benchmark SH000905
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    infer_processors: []
+    learn_processors:
+        - class: DropnaLabel
+        - class: CSRankNorm
+          kwargs:
+              fields_group: label
+    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: DEnsembleModel
+        module_path: qlib.contrib.model.double_ensemble
+        kwargs:
+            base_model: "gbm"
+            loss: mse
+            num_models: 6
+            enable_sr: True
+            enable_fs: True
+            alpha1: 1
+            alpha2: 1
+            bins_sr: 10
+            bins_fs: 5
+            decay: 0.5
+            sample_ratios:
+                - 0.8
+                - 0.7
+                - 0.6
+                - 0.5
+                - 0.4
+            sub_weights:
+                - 1
+                - 0.2
+                - 0.2
+                - 0.2
+                - 0.2
+                - 0.2
+            epochs: 136
+            colsample_bytree: 0.8879
+            learning_rate: 0.0421
+            subsample: 0.8789
+            lambda_l1: 205.6999
+            lambda_l2: 580.9768
+            max_depth: 8
+            num_leaves: 210
+            num_threads: 20
+            verbosity: -1
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha360
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs:
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

examples/benchmarks/GATs/requirements.txt

@@ -1,4 +1,4 @@
 pandas==1.1.2
-numpy==1.17.4
+numpy==1.21.0
 scikit_learn==0.23.2
 torch==1.7.0

examples/benchmarks/GATs/workflow_config_gats_Alpha158.yaml

@@ -33,19 +33,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: GATs
@@ -79,7 +84,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -88,4 +95,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config

examples/benchmarks/GATs/workflow_config_gats_Alpha360.yaml

@@ -26,19 +26,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: GATs
@@ -71,7 +76,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/GRU/README.md

@@ -0,0 +1,2 @@
+# Gated Recurrent Unit (GRU)
+* Paper: [Learning Phrase Representations using RNN Encoder–Decoder for Statistical Machine Translation](https://aclanthology.org/D14-1179.pdf).

examples/benchmarks/GRU/requirements.txt

@@ -1,4 +1,4 @@
-numpy==1.17.4
+numpy==1.21.0
 pandas==1.1.2
 scikit_learn==0.23.2
 torch==1.7.0

examples/benchmarks/GRU/workflow_config_gru_Alpha158.yaml

@@ -34,19 +34,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: GRU
@@ -80,7 +85,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/GRU/workflow_config_gru_Alpha360.yaml

@@ -26,19 +26,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: GRU
@@ -70,7 +75,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -79,4 +86,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config

examples/benchmarks/HIST/README.md

@@ -0,0 +1,3 @@
+# HIST
+* Code: [https://github.com/Wentao-Xu/HIST](https://github.com/Wentao-Xu/HIST)
+* Paper: [HIST: A Graph-based Framework for Stock Trend Forecasting via Mining Concept-Oriented Shared InformationAdaRNN: Adaptive Learning and Forecasting for Time Series](https://arxiv.org/abs/2110.13716).

examples/benchmarks/HIST/requirements.txt

@@ -0,0 +1,4 @@
+pandas==1.1.2
+numpy==1.21.0
+scikit_learn==0.23.2
+torch==1.7.0

examples/benchmarks/HIST/workflow_config_hist_Alpha360.yaml

@@ -0,0 +1,92 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market csi300
+benchmark: &benchmark SH000300
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    infer_processors:
+        - class: RobustZScoreNorm
+          kwargs:
+              fields_group: feature
+              clip_outlier: true
+        - class: Fillna
+          kwargs:
+              fields_group: feature
+    learn_processors:
+        - class: DropnaLabel
+        - class: CSRankNorm
+          kwargs:
+              fields_group: label
+    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: HIST
+        module_path: qlib.contrib.model.pytorch_hist
+        kwargs:
+            d_feat: 6
+            hidden_size: 64
+            num_layers: 2
+            dropout: 0
+            n_epochs: 200
+            lr: 1e-4
+            early_stop: 20
+            metric: ic
+            loss: mse
+            base_model: LSTM
+            model_path: "benchmarks/LSTM/model_lstm_csi300.pkl"
+            stock2concept: "benchmarks/HIST/qlib_csi300_stock2concept.npy"
+            stock_index: "benchmarks/HIST/qlib_csi300_stock_index.npy"
+            GPU: 0
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha360
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

examples/benchmarks/IGMTF/README.md

@@ -0,0 +1,4 @@
+# IGMTF
+* Code: [https://github.com/Wentao-Xu/IGMTF](https://github.com/Wentao-Xu/IGMTF)
+* Paper: [IGMTF: An Instance-wise Graph-based Framework for
+Multivariate Time Series Forecasting](https://arxiv.org/abs/2109.06489).

examples/benchmarks/IGMTF/requirements.txt

@@ -0,0 +1,4 @@
+pandas==1.1.2
+numpy==1.21.0
+scikit_learn==0.23.2
+torch==1.7.0

examples/benchmarks/IGMTF/workflow_config_igmtf_Alpha360.yaml

@@ -0,0 +1,89 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market csi300
+benchmark: &benchmark SH000300
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    infer_processors:
+        - class: RobustZScoreNorm
+          kwargs:
+              fields_group: feature
+              clip_outlier: true
+        - class: Fillna
+          kwargs:
+              fields_group: feature
+    learn_processors:
+        - class: DropnaLabel
+        - class: CSRankNorm
+          kwargs:
+              fields_group: label
+    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            model: <MODEL>
+            dataset: <DATASET>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: IGMTF
+        module_path: qlib.contrib.model.pytorch_igmtf
+        kwargs:
+            d_feat: 6
+            hidden_size: 64
+            num_layers: 2
+            dropout: 0
+            n_epochs: 200
+            lr: 1e-4
+            early_stop: 20
+            metric: ic
+            loss: mse
+            base_model: LSTM
+            model_path: "benchmarks/LSTM/model_lstm_csi300.pkl"
+            GPU: 0
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha360
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

examples/benchmarks/LSTM/README.md

@@ -0,0 +1,2 @@
+# Long Short-Term Memory (LSTM)
+* Paper: [Long Short-Term Memory](https://direct.mit.edu/neco/article-abstract/9/8/1735/6109/Long-Short-Term-Memory?redirectedFrom=fulltext).

examples/benchmarks/LSTM/requirements.txt

@@ -1,4 +1,4 @@
-numpy==1.17.4
+numpy==1.21.0
 pandas==1.1.2
 scikit_learn==0.23.2
 torch==1.7.0

examples/benchmarks/LSTM/workflow_config_lstm_Alpha158.yaml

@@ -34,19 +34,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: LSTM
@@ -80,7 +85,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 

examples/benchmarks/LSTM/workflow_config_lstm_Alpha360.yaml

@@ -26,19 +26,24 @@ data_handler_config: &data_handler_config
 port_analysis_config: &port_analysis_config
     strategy:
         class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy.strategy
+        module_path: qlib.contrib.strategy
         kwargs:
+            signal:
+                - <MODEL> 
+                - <DATASET>
             topk: 50
             n_drop: 5
     backtest:
-        verbose: False
-        limit_threshold: 0.095
+        start_time: 2017-01-01
+        end_time: 2020-08-01
         account: 100000000
         benchmark: *benchmark
-        deal_price: close
-        open_cost: 0.0005
-        close_cost: 0.0015
-        min_cost: 5
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
 task:
     model:
         class: LSTM
@@ -70,7 +75,9 @@ task:
     record: 
         - class: SignalRecord
           module_path: qlib.workflow.record_temp
-          kwargs: {}
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
         - class: SigAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
@@ -79,4 +86,4 @@ task:
         - class: PortAnaRecord
           module_path: qlib.workflow.record_temp
           kwargs: 
-            config: *port_analysis_config
+            config: *port_analysis_config