Debug MacOS CI

.github/workflows/test.yml

@@ -0,0 +1,131 @@
+name: Test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-18.04, ubuntu-20.04]
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-version: [3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Lint with Black
+      run: |
+        pip install --upgrade pip
+        pip install black wheel
+        black qlib -l 120 --check --diff
+
+    - name: Install Qlib with pip
+      run: |
+        pip install numpy==1.19.5 ruamel.yaml
+        pip install pyqlib --ignore-installed
+
+    - name: Make html with sphnix
+      run: |
+        pip install -U sphinx
+        pip install sphinx_rtd_theme readthedocs_sphinx_ext
+        pip install --exists-action=w --no-cache-dir -r docs/requirements.txt 
+        cd docs 
+        sphinx-build -b html . build
+        cd ..
+        
+    # Check Qlib with pylint
+    # TODO: These problems we will solve in the future. Important among them are: W0221, W0223, W0237, E1102
+      # C0103: invalid-name
+      # C0209: consider-using-f-string
+      # R0402: consider-using-from-import
+      # R1705: no-else-return
+      # R1710: inconsistent-return-statements
+      # R1725: super-with-arguments
+      # R1735: use-dict-literal
+      # W0102: dangerous-default-value
+      # W0212: protected-access
+      # W0221: arguments-differ
+      # W0223: abstract-method
+      # W0231: super-init-not-called
+      # W0237: arguments-renamed
+      # W0612: unused-variable
+      # W0621: redefined-outer-name
+      # W0622: redefined-builtin
+      # FIXME: specify exception type
+      # W0703: broad-except
+      # W1309: f-string-without-interpolation
+      # E1102: not-callable
+      # E1136: unsubscriptable-object
+    # References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962 
+    - name: Check Qlib with pylint
+      run: |
+        pip install --upgrade pip
+        pip install pylint
+        pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0201,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' qlib --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500"
+
+    # The following flake8 error codes were ignored:
+      # E501 line too long
+        # Description: We have used black to limit the length of each line to 120.
+      # F541 f-string is missing placeholders
+        # Description: The same thing is done when using pylint for detection.
+      # E266 too many leading '#' for block comment
+        # Description: To make the code more readable, a lot of "#" is used.
+        # This error code appears centrally in:
+          # qlib/backtest/executor.py
+          # qlib/data/ops.py
+          # qlib/utils/__init__.py
+      # E402 module level import not at top of file
+        # Description: There are times when module level import is not available at the top of the file.
+      # W503 line break before binary operator
+        # Description: Since black formats the length of each line of code, it has to perform a line break when a line of arithmetic is too long.
+      # E731 do not assign a lambda expression, use a def
+        # Description: Restricts the use of lambda expressions, but at some point lambda expressions are required.
+      # E203 whitespace before ':'
+        # Description: If there is whitespace before ":", it cannot pass the black check.
+    - name: Check Qlib with flake8
+      run: |
+        pip install --upgrade pip
+        pip install flake8
+        flake8 --ignore=E501,F541,E266,E402,W503,E731,E203 qlib
+
+    - name: Test data downloads
+      run: |
+        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data_simple --interval 1d --region cn
+        python -c "import os; userpath=os.path.expanduser('~'); os.rename(userpath + '/.qlib/qlib_data/cn_data_simple', userpath + '/.qlib/qlib_data/cn_data')"
+
+    - name: Test workflow by config (install from pip)
+      run: |
+        python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+        python -m pip uninstall -y pyqlib
+
+     # Test Qlib installed from source
+    - name: Install Qlib from source
+      run: |
+        pip install --upgrade cython jupyter jupyter_contrib_nbextensions numpy scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't
+        pip install -e .
+
+    - name: Install test dependencies
+      run: |
+        pip install --upgrade pip
+        pip install black pytest
+
+    - name: Unit tests with Pytest
+      run: |
+        cd tests
+        python -m pytest . --durations=10
+
+    - name: Test workflow by config (install from source)
+      run: |
+        python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+      

.github/workflows/test_macos.yml

@@ -0,0 +1,75 @@
+# There are some issues (in the downloading data phase) on MacOS when running with other tests. So we split it into an individual config.
+name: Test MacOS
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-11, macos-latest]
+        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
+        python-version: [3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Lint with Black
+      run: |
+        cd ..
+        python -m pip install pip --upgrade
+        python -m pip install wheel --upgrade
+        python -m pip install black
+        python -m black qlib -l 120 --check --diff
+    # Test Qlib installed with pip
+
+    - name: Install Qlib with pip
+      run: |
+          python -m pip install numpy==1.19.5
+          python -m pip install pyqlib --ignore-installed ruamel.yaml numpy
+    - name: Install Lightgbm for MacOS
+      run: |
+        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
+        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
+        # FIX MacOS error: Segmentation fault
+        # reference: https://github.com/microsoft/LightGBM/issues/4229
+        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
+        brew unlink libomp
+        brew install libomp.rb
+    - name: Test data downloads
+      run: |
+        python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
+    - name: Test workflow by config (install from pip)
+      run: |
+        python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+        python -m pip uninstall -y pyqlib
+    # Test Qlib installed from source
+    - name: Install Qlib from source
+      run: |
+        python -m pip install --upgrade cython
+        python -m pip install numpy jupyter jupyter_contrib_nbextensions
+        python -m pip install -U scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't
+        pip install -e .
+    - name: Install test dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -U pyopenssl idna
+        python -m pip install black pytest
+    - name: Unit tests with Pytest
+      run: |
+        cd tests
+        python -m pytest . --durations=0
+    - name: Test workflow by config (install from source)
+      run: |
+          python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml

.github/workflows/test_qlib_from_pip.yml

@@ -1,57 +0,0 @@
-name: Test qlib from pip
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-    timeout-minutes: 120
-
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest]
-        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
-        python-version: [3.7, 3.8]
-
-    steps:
-    - name: Test qlib from pip
-      uses: actions/checkout@v2
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Update pip to the latest version
-      run: |
-        python -m pip install --upgrade pip
-
-    - name: Qlib installation test
-      run: |
-        python -m pip install pyqlib
-        # Specify the numpy version because the numpy upgrade caused the CI test to fail, 
-        # and this line of code will be removed when the next version of qlib is released.
-        python -m pip install "numpy<1.23"
-
-    - name: Install Lightgbm for MacOS
-      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
-      run: |
-        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
-        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
-        # FIX MacOS error: Segmentation fault
-        # reference: https://github.com/microsoft/LightGBM/issues/4229
-        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
-        brew unlink libomp
-        brew install libomp.rb
-
-    - name: Downloads dependencies data
-      run: |
-        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
-
-    - name: Test workflow by config
-      run: |
-        qrun examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml

.github/workflows/test_qlib_from_source.yml

@@ -1,156 +0,0 @@
-name: Test qlib from source
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-    timeout-minutes: 180
-    # we may retry for 3 times for `Unit tests with Pytest`
-
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest]
-        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
-        python-version: [3.7, 3.8]
-
-    steps:
-    - name: Test qlib from source
-      uses: actions/checkout@v2
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Update pip to the latest version
-      run: |
-        python -m pip install --upgrade pip
-
-    - name: Installing pytorch for macos
-      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
-      run: |
-        python -m pip install torch torchvision torchaudio
-
-    - name: Installing pytorch for ubuntu
-      if: ${{ matrix.os == 'ubuntu-18.04' || matrix.os == 'ubuntu-20.04' }}
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
-
-    - name: Installing pytorch for windows
-      if: ${{ matrix.os == 'windows-latest' }}
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install torch torchvision torchaudio
-
-    - name: Set up Python tools
-      run: |
-        python -m pip install --upgrade cython
-        python -m pip install -e .[dev]
-
-    - name: Lint with Black
-      run: |
-        black . -l 120 --check --diff
-
-    - name: Make html with sphinx
-      run: |
-        cd docs 
-        sphinx-build -W --keep-going -b html . _build
-        cd ..
-
-    # Check Qlib with pylint
-    # TODO: These problems we will solve in the future. Important among them are: W0221, W0223, W0237, E1102
-      # C0103: invalid-name
-      # C0209: consider-using-f-string
-      # R0402: consider-using-from-import
-      # R1705: no-else-return
-      # R1710: inconsistent-return-statements
-      # R1725: super-with-arguments
-      # R1735: use-dict-literal
-      # W0102: dangerous-default-value
-      # W0212: protected-access
-      # W0221: arguments-differ
-      # W0223: abstract-method
-      # W0231: super-init-not-called
-      # W0237: arguments-renamed
-      # W0612: unused-variable
-      # W0621: redefined-outer-name
-      # W0622: redefined-builtin
-      # FIXME: specify exception type
-      # W0703: broad-except
-      # W1309: f-string-without-interpolation
-      # E1102: not-callable
-      # E1136: unsubscriptable-object
-    # References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962
-    # We use sys.setrecursionlimit(2000) to make the recursion depth larger to ensure that pylint works properly (the default recursion depth is 1000).
-    - name: Check Qlib with pylint
-      run: |
-        pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' qlib --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500; import sys; sys.setrecursionlimit(2000)"
-
-    # The following flake8 error codes were ignored:
-      # E501 line too long
-        # Description: We have used black to limit the length of each line to 120.
-      # F541 f-string is missing placeholders
-        # Description: The same thing is done when using pylint for detection.
-      # E266 too many leading '#' for block comment
-        # Description: To make the code more readable, a lot of "#" is used.
-        # This error code appears centrally in:
-          # qlib/backtest/executor.py
-          # qlib/data/ops.py
-          # qlib/utils/__init__.py
-      # E402 module level import not at top of file
-        # Description: There are times when module level import is not available at the top of the file.
-      # W503 line break before binary operator
-        # Description: Since black formats the length of each line of code, it has to perform a line break when a line of arithmetic is too long.
-      # E731 do not assign a lambda expression, use a def
-        # Description: Restricts the use of lambda expressions, but at some point lambda expressions are required.
-      # E203 whitespace before ':'
-        # Description: If there is whitespace before ":", it cannot pass the black check.
-    - name: Check Qlib with flake8
-      run: |
-        flake8 --ignore=E501,F541,E266,E402,W503,E731,E203 --per-file-ignores="__init__.py:F401,F403" qlib
-
-    # https://github.com/python/mypy/issues/10600
-    - name: Check Qlib with mypy
-      run: |
-        mypy qlib --install-types --non-interactive || true
-        mypy qlib --verbose
-
-    - name: Test data downloads
-      run: |
-        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
-        azcopy copy https://qlibpublic.blob.core.windows.net/data/rl /tmp/qlibpublic/data --recursive
-        mv /tmp/qlibpublic/data tests/.data
-
-    - name: Install Lightgbm for MacOS
-      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
-      run: |
-        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
-        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
-        # FIX MacOS error: Segmentation fault
-        # reference: https://github.com/microsoft/LightGBM/issues/4229
-        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
-        brew unlink libomp
-        brew install libomp.rb
-
-    - name: Test workflow by config (install from source)
-      run: |
-        # Version 0.52.0 of numba must be installed manually in CI, otherwise it will cause incompatibility with the latest version of numpy.
-        python -m pip install numba==0.52.0
-        # You must update numpy manually, because when installing python tools, it will try to uninstall numpy and cause CI to fail.
-        python -m pip install --upgrade numpy
-        python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
-
-    - name: Unit tests with Pytest
-      uses: nick-fields/retry@v2
-      with:
-        timeout_minutes: 60
-        max_attempts: 3
-        command: |
-          cd tests
-          python -m pytest . -m "not slow" --durations=0

.github/workflows/test_qlib_from_source_slow.yml

@@ -1,59 +0,0 @@
-name: Test qlib from source slow
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-    timeout-minutes: 720
-    # we may retry for 3 times for `Unit tests with Pytest`
-
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest]
-        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
-        python-version: [3.7, 3.8]
-
-    steps:
-    - name: Test qlib from source slow
-      uses: actions/checkout@v2
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Set up Python tools
-      run: |
-        python -m pip install --upgrade pip
-        # python -m pip is necessary to upgrade pip.
-        pip install --upgrade cython numpy
-        pip install -e .[dev]
-
-    - name: Downloads dependencies data
-      run: |
-        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
-
-    - name: Install Lightgbm for MacOS
-      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
-      run: |
-        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
-        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
-        # FIX MacOS error: Segmentation fault
-        # reference: https://github.com/microsoft/LightGBM/issues/4229
-        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
-        brew unlink libomp
-        brew install libomp.rb
-
-    - name: Unit tests with Pytest
-      uses: nick-fields/retry@v2
-      with:
-        timeout_minutes: 240
-        max_attempts: 3
-        command: |
-          cd tests
-          python -m pytest . -m "slow" --durations=0

.gitignore

@@ -24,16 +24,9 @@ qlib/VERSION.txt
 qlib/data/_libs/expanding.cpp
 qlib/data/_libs/rolling.cpp
 examples/estimator/estimator_example/
-examples/rl/data/
-examples/rl/checkpoints/
-examples/rl/outputs/
 
 *.egg-info/
 
-# test related
-test-output.xml
-.output
-.data
 
 # special software
 mlruns/
@@ -41,10 +34,8 @@ mlruns/
 tags
 
 .pytest_cache/
-.mypy_cache/
 .vscode/
 
 *.swp
 
 ./pretrain
-.idea/

.mypy.ini

@@ -1,17 +0,0 @@
-[mypy]
-exclude = (?x)(
-    ^qlib/backtest/high_performance_ds\.py$
-    | ^qlib/contrib
-    | ^qlib/data
-    | ^qlib/model
-    | ^qlib/strategy
-    | ^qlib/tests
-    | ^qlib/utils
-    | ^qlib/workflow
-    | ^qlib/config\.py$
-    | ^qlib/log\.py$
-    | ^qlib/__init__\.py$
-  )
-ignore_missing_imports = true
-disallow_incomplete_defs = true
-follow_imports = skip

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/psf/black
-    rev: 22.6.0
+    rev: 22.1.0
     hooks:
     -   id: black
         args: ["qlib", "-l 120"]

CHANGES.rst

@@ -1,63 +1,63 @@
 Changelog
-=========
+====================
 Here you can see the full list of changes between each QLib release.
 
 Version 0.1.0
--------------
+--------------------
 This is the initial release of QLib library.
 
 Version 0.1.1
--------------
+--------------------
 Performance optimize. Add more features and operators.
 
 Version 0.1.2
--------------
+--------------------
 - Support operator syntax. Now ``High() - Low()`` is equivalent to ``Sub(High(), Low())``.   
 - Add more technical indicators.
 
 Version 0.1.3
--------------
+--------------------
 Bug fix and add instruments filtering mechanism.
 
 Version 0.2.0
--------------
+--------------------
 - Redesign ``LocalProvider`` database format for performance improvement.
 - Support load features as string fields.
 - Add scripts for database construction.
 - More operators and technical indicators.
 
 Version 0.2.1
--------------
+--------------------
 - Support registering user-defined ``Provider``.
 - Support use operators in string format, e.g. ``['Ref($close, 1)']`` is valid field format.
 - Support dynamic fields in ``$some_field`` format. And existing fields like ``Close()`` may be deprecated in the future.
 
 Version 0.2.2
--------------
+--------------------
 - Add ``disk_cache`` for reusing features (enabled by default).
 - Add ``qlib.contrib`` for experimental model construction and evaluation.
 
 
 Version 0.2.3
--------------
+--------------------
 - Add ``backtest`` module
 - Decoupling the Strategy, Account, Position, Exchange from the backtest module
 
 Version 0.2.4
--------------
+--------------------
 - Add ``profit attribution`` module
 - Add ``rick_control`` and ``cost_control`` strategies
   
 Version 0.3.0
--------------
+--------------------
 - Add ``estimator`` module
 
 Version 0.3.1
--------------
+--------------------
 - Add ``filter`` module
 
 Version 0.3.2
--------------
+--------------------
 - Add real price trading, if the ``factor`` field in the data set is incomplete, use ``adj_price`` trading
 - Refactor ``handler`` ``launcher`` ``trainer`` code
 - Support ``backtest`` configuration parameters in the configuration file
@@ -65,16 +65,16 @@ Version 0.3.2
 - Fix bug of ``filter`` module
 
 Version 0.3.3
--------------
+-------------------
 - Fix bug of ``filter`` module
 
 Version 0.3.4
--------------
+--------------------
 - Support for ``finetune model``
 - Refactor ``fetcher`` code
 
 Version 0.3.5
--------------
+--------------------
 - Support multi-label training, you can provide multiple label in ``handler``. (But LightGBM doesn't support due to the algorithm itself)
 - Refactor ``handler`` code, dataset.py is no longer used, and you can deploy your own labels and features in ``feature_label_config``
 - Handler only offer DataFrame. Also, ``trainer`` and model.py only receive DataFrame
@@ -82,10 +82,10 @@ Version 0.3.5
 - Move some date config from ``handler`` to ``trainer``
 
 Version 0.4.0
--------------
+--------------------
 - Add `data` package that holds all data-related codes
 - Reform the data provider structure
-- Create a server for data centralized management `qlib-server <https://amc-msra.visualstudio.com/trading-algo/_git/qlib-server>`_
+- Create a server for data centralized management `qlib-server<https://amc-msra.visualstudio.com/trading-algo/_git/qlib-server>`_
 - Add a `ClientProvider` to work with server
 - Add a pluggable cache mechanism
 - Add a recursive backtracking algorithm to inspect the furthest reference date for an expression
@@ -100,7 +100,7 @@ Version 0.4.0
 
 
 Version 0.4.1
--------------
+--------------------
 - Add support Windows
 - Fix ``instruments`` type bug
 - Fix ``features`` is empty bug(It will cause failure in updating)
@@ -112,19 +112,19 @@ Version 0.4.1
 
 
 Version 0.4.2
--------------
+--------------------
 - Refactor DataHandler
 - Add ``Alpha360`` DataHandler
 
 
 Version 0.4.3
--------------
+--------------------
 - Implementing Online Inference and Trading Framework
 - Refactoring The interfaces of backtest and strategy module.
 
 
 Version 0.4.4
--------------
+--------------------
 - Optimize cache generation performance
 - Add report module
 - Fix bug when using ``ServerDatasetCache`` offline.
@@ -138,7 +138,7 @@ Version 0.4.4
 
 
 Version 0.4.5
--------------
+--------------------
 - Add multi-kernel implementation for both client and server.
     - Support a new way to load data from client which skips dataset cache.
     - Change the default dataset method from single kernel implementation to multi kernel implementation.
@@ -146,14 +146,14 @@ Version 0.4.5
 - Support a new method to write config file by using dict.
 
 Version 0.4.6
--------------
+--------------------
 - Some bugs are fixed
     - The default config in `Version 0.4.5` is not friendly to daily frequency data.
     - Backtest error in TopkWeightStrategy when `WithInteract=True`.
 
 
 Version 0.5.0
--------------
+--------------------
 - First opensource version
     - Refine the docs, code
     - Add baselines
@@ -161,19 +161,19 @@ Version 0.5.0
 
 
 Version 0.8.0
--------------
+--------------------
 - The backtest is greatly refactored.
     - Nested decision execution framework is supported
     - There are lots of changes for daily trading, it is hard to list all of them. But a few important changes could be noticed
         - The trading limitation is more accurate;
-            - In `previous version <https://github.com/microsoft/qlib/blob/v0.7.2/qlib/contrib/backtest/exchange.py#L160>`__, longing and shorting actions share the same action.
+            - In `previous version <https://github.com/microsoft/qlib/blob/v0.7.2/qlib/contrib/backtest/exchange.py#L160>`_, longing and shorting actions share the same action.
-            - In `current version <https://github.com/microsoft/qlib/blob/7c31012b507a3823117bddcc693fc64899460b2a/qlib/backtest/exchange.py#L304>`__, the trading limitation is different between logging and shorting action.
+            - In `current version <https://github.com/microsoft/qlib/blob/7c31012b507a3823117bddcc693fc64899460b2a/qlib/backtest/exchange.py#L304>`_, the trading limitation is different between logging and shorting action.
         - The constant is different when calculating annualized metrics.
-            - `Current version <https://github.com/microsoft/qlib/blob/7c31012b507a3823117bddcc693fc64899460b2a/qlib/contrib/evaluate.py#L42>`_ uses more accurate constant than `previous version <https://github.com/microsoft/qlib/blob/v0.7.2/qlib/contrib/evaluate.py#L22>`__
+            - `Current version <https://github.com/microsoft/qlib/blob/7c31012b507a3823117bddcc693fc64899460b2a/qlib/contrib/evaluate.py#L42>`_ uses more accurate constant than `previous version <https://github.com/microsoft/qlib/blob/v0.7.2/qlib/contrib/evaluate.py#L22>`_
-        - `A new version <https://github.com/microsoft/qlib/blob/7c31012b507a3823117bddcc693fc64899460b2a/qlib/tests/data.py#L17>`__ of data is released. Due to the unstability of Yahoo data source, the data may be different after downloading data again.
+        - `A new version <https://github.com/microsoft/qlib/blob/7c31012b507a3823117bddcc693fc64899460b2a/qlib/tests/data.py#L17>`_ of data is released. Due to the unstability of Yahoo data source, the data may be different after downloading data again.
-        - Users could check out the backtesting results between  `Current version <https://github.com/microsoft/qlib/tree/7c31012b507a3823117bddcc693fc64899460b2a/examples/benchmarks>`__ and `previous version <https://github.com/microsoft/qlib/tree/v0.7.2/examples/benchmarks>`__
+        - Users could check out the backtesting results between  `Current version <https://github.com/microsoft/qlib/tree/7c31012b507a3823117bddcc693fc64899460b2a/examples/benchmarks>`_ and `previous version <https://github.com/microsoft/qlib/tree/v0.7.2/examples/benchmarks>`_
 
 
 Other Versions
---------------
+----------------------------------
 Please refer to `Github release Notes <https://github.com/microsoft/qlib/releases>`_

README.md

@@ -11,7 +11,6 @@
 Recent released features
 | Feature | Status |
 | --                      | ------    |
-| RL Learning Framework | :hammer: :chart_with_upwards_trend: Released on Nov 10, 2022. [#1332](https://github.com/microsoft/qlib/pull/1332), [#1322](https://github.com/microsoft/qlib/pull/1322), [#1316](https://github.com/microsoft/qlib/pull/1316),[#1299](https://github.com/microsoft/qlib/pull/1299),[#1263](https://github.com/microsoft/qlib/pull/1263), [#1244](https://github.com/microsoft/qlib/pull/1244), [#1169](https://github.com/microsoft/qlib/pull/1169), [#1125](https://github.com/microsoft/qlib/pull/1125), [#1076](https://github.com/microsoft/qlib/pull/1076)|
 | HIST and IGMTF models | :chart_with_upwards_trend: [Released](https://github.com/microsoft/qlib/pull/1040) on Apr 10, 2022 |
 | Qlib [notebook tutorial](https://github.com/microsoft/qlib/tree/main/examples/tutorial) | 📖 [Released](https://github.com/microsoft/qlib/pull/1037) on Apr 7, 2022 | 
 | Ibovespa index data | :rice: [Released](https://github.com/microsoft/qlib/pull/990) on Apr 6, 2022 |
@@ -68,7 +67,6 @@ For more details, please refer to our paper ["Qlib: An AI-oriented Quantitative
             <li type="circle"><a href="#auto-quant-research-workflow">Auto Quant Research Workflow</a></li>
             <li type="circle"><a href="#building-customized-quant-research-workflow-by-code">Building Customized Quant Research Workflow by Code</a></li></ul>
         <li><a href="#quant-dataset-zoo"><strong>Quant Dataset Zoo</strong></a></li>
-        <li><a href="#learning-framework">Learning Framework</a></li>
         <li><a href="#more-about-qlib">More About Qlib</a></li>
         <li><a href="#offline-mode-and-online-mode">Offline Mode and Online Mode</a>
         <ul>
@@ -107,16 +105,21 @@ Your feedbacks about the features are very important.
 # Framework of Qlib
 
 <div style="align: center">
-<img src="docs/_static/img/framework-abstract.jpg" />
+<img src="docs/_static/img/framework.svg" />
 </div>
 
-The high-level framework of Qlib can be found above(users can find the [detailed framework](https://qlib.readthedocs.io/en/latest/introduction/introduction.html#framework) of Qlib's design when getting into nitty gritty).
+At the module level, Qlib is a platform that consists of the above components. The components are designed as loose-coupled modules, and each component could be used stand-alone.
-The components are designed as loose-coupled modules, and each component could be used stand-alone.
 
-Qlib provides a strong infrastructure to support Quant research. [Data](https://qlib.readthedocs.io/en/latest/component/data.html) is always an important part.
+| Name                   | Description                                                                                                                                                                                                                                                                                                                                                             |
-A strong learning framework is designed to support diverse learning paradigms (e.g. [reinforcement learning](https://qlib.readthedocs.io/en/latest/component/rl.html), [supervised learning](https://qlib.readthedocs.io/en/latest/component/workflow.html#model-section)) and patterns at different levels(e.g. [market dynamic modeling](https://qlib.readthedocs.io/en/latest/component/meta.html)).
+| ------                 | -----                                                                                                                                                                                                                                                                                                                                                                   |
-By modeling the market, [trading strategies](https://qlib.readthedocs.io/en/latest/component/strategy.html) will generate trade decisions that will be executed. Multiple trading strategies and executors in different levels or granularities can be [nested to be optimized and run together](https://qlib.readthedocs.io/en/latest/component/highfreq.html).
+| `Infrastructure` layer | `Infrastructure` layer provides underlying support for Quant research. `DataServer` provides a high-performance infrastructure for users to manage and retrieve raw data. `Trainer` provides a flexible interface to control the training process of models, which enable algorithms to control the training process.                                                       |
-At last, a comprehensive [analysis](https://qlib.readthedocs.io/en/latest/component/report.html) will be provided and the model can be [served online](https://qlib.readthedocs.io/en/latest/component/online.html) in a low cost.
+| `Workflow` layer       | `Workflow` layer covers the whole workflow of quantitative investment. `Information Extractor` extracts data for models. `Forecast Model` focuses on producing all kinds of forecast signals (e.g. _alpha_, risk) for other modules. With these signals `Decision Generator` will generate the target trading decisions(i.e. portfolio, orders)  to be executed by `Execution Env` (i.e. the trading market).  There may be multiple levels of `Trading Agent` and `Execution Env` (e.g. an _order executor trading agent and intraday order execution environment_ could behave like an interday trading environment and nested in  _daily portfolio management trading agent and interday trading environment_  ) |
+| `Interface` layer      | `Interface` layer tries to present a user-friendly interface for the underlying system. `Analyser` module will provide users detailed analysis reports of forecasting signals, portfolios and execution results                                                                                                                                                                 |
+
+* The modules with hand-drawn style are under development and will be released in the future.
+* The modules with dashed borders are highly user-customizable and extendible.
+
+(p.s. framework image is created with https://draw.io/)
 
 
 # Quick Start
@@ -167,25 +170,12 @@ Also, users can install the latest dev version ``Qlib`` by the source code accor
     git clone https://github.com/microsoft/qlib.git && cd qlib
     pip install .
     ```
-  **Note**:  You can install Qlib with `python setup.py install` as well. But it is not the recommended approach. It will skip `pip` and cause obscure problems. For example, **only** the command ``pip install .`` **can** overwrite the stable version installed by ``pip install pyqlib``, while the command ``python setup.py install`` **can't**.
+  **Note**:  You can install Qlib with `python setup.py install` as well. But it is not the recommanded approach. It will skip `pip` and cause obscure problems. For example, **only** the command ``pip install .`` **can** overwrite the stable version installed by ``pip install pyqlib``, while the command ``python setup.py install`` **can't**.
 
-**Tips**: If you fail to install `Qlib` or run the examples in your environment,  comparing your steps and the [CI workflow](.github/workflows/test_qlib_from_source.yml) may help you find the problem.
+**Tips**: If you fail to install `Qlib` or run the examples in your environment,  comparing your steps and the [CI workflow](.github/workflows/test.yml) may help you find the problem.
 
 ## Data Preparation
 Load and prepare data by running the following code:
-
-### Get with module
-  ```bash
-  # get 1d data
-  python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn
-
-  # get 1min data
-  python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data_1min --region cn --interval 1min
-
-  ```
-
-### Get from source
-
   ```bash
   # get 1d data
   python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn
@@ -207,8 +197,6 @@ We recommend users to prepare their own data if they have a high-quality dataset
   > 
   > It is recommended that users update the data manually once (--trading_date 2021-05-25) and then set it to update automatically.
   >
-  > **NOTE**: Users can't incrementally  update data based on the offline data provided by Qlib(some fields are removed to reduce the data size). Users should use [yahoo collector](https://github.com/microsoft/qlib/tree/main/scripts/data_collector/yahoo#automatic-update-of-daily-frequency-datafrom-yahoo-finance) to download Yahoo data from scratch and then incrementally update it.
-  > 
   > For more information, please refer to: [yahoo collector](https://github.com/microsoft/qlib/tree/main/scripts/data_collector/yahoo#automatic-update-of-daily-frequency-datafrom-yahoo-finance)
 
   * Automatic update of data to the "qlib" directory each trading day(Linux)
@@ -401,17 +389,6 @@ Dataset plays a very important role in Quant. Here is a list of the datasets bui
 [Here](https://qlib.readthedocs.io/en/latest/advanced/alpha.html) is a tutorial to build dataset with `Qlib`.
 Your PR to build new Quant dataset is highly welcomed.
 
-
-# Learning Framework
-Qlib is high customizable and a lot of its components are learnable.
-The learnable components are instances of `Forecast Model` and `Trading Agent`. They are learned based on the `Learning Framework` layer and then applied to multiple scenarios in `Workflow` layer.
-The learning framework leverages the `Workflow` layer as well(e.g. sharing `Information Extractor`, creating environments based on `Execution Env`).
-
-Based on learning paradigms, they can be categorized into reinforcement learning and supervised learning.
-- For supervised learning, the detailed docs can be found [here](https://qlib.readthedocs.io/en/latest/component/model.html).
-- For reinforcement learning, the detailed docs can be found [here](https://qlib.readthedocs.io/en/latest/component/rl.html). Qlib's RL learning framework leverages `Execution Env` in `Workflow` layer to create environments.  It's worth noting that `NestedExecutor` is supported as well. This empowers users to optimize different level of strategies/models/agents together (e.g. optimizing an order execution strategy for a specific portfolio management strategy).
-
-
 # More About Qlib
 If you want to have a quick glance at the most frequently used components of qlib, you can try notebooks [here](examples/tutorial/).
 
@@ -481,7 +458,7 @@ Before we released Qlib as an open-source project on Github in Sep 2020, Qlib is
 
 This project welcomes contributions and suggestions.  
 **Here are some 
-[code standards and development guidance](docs/developer/code_standard_and_dev_guide.rst) for submiting a pull request.**
+[code standards](docs/developer/code_standard.rst) for submiting a pull request.**
 
 Making contributions is not a hard thing. Solving an issue(maybe just answering a question raised in [issues list](https://github.com/microsoft/qlib/issues) or [gitter](https://gitter.im/Microsoft/qlib)), fixing/issuing a bug, improving the documents and even fixing a typo are important contributions to Qlib.
 
@@ -497,7 +474,7 @@ If you don't know how to start to contribute, you can refer to the following exa
 | Docs | [Improve docs quality](https://github.com/microsoft/qlib/pull/797/files) ;  [Fix a typo](https://github.com/microsoft/qlib/pull/774) | 
 | Feature |  Implement a [requested feature](https://github.com/microsoft/qlib/projects) like [this](https://github.com/microsoft/qlib/pull/754); [Refactor interfaces](https://github.com/microsoft/qlib/pull/539/files) |
 | Dataset | [Add a dataset](https://github.com/microsoft/qlib/pull/733) | 
-| Models |  [Implement a new model](https://github.com/microsoft/qlib/pull/689), [some instructions to contribute models](https://github.com/microsoft/qlib/tree/main/examples/benchmarks#contributing) |
+| Models |  [Implement a new model](https://github.com/microsoft/qlib/pull/689) | 
 
 [Good first issues](https://github.com/microsoft/qlib/labels/good%20first%20issue) are labelled to indicate that they are easy to start your contributions.
 

docs/FAQ/FAQ.rst

@@ -3,7 +3,7 @@ Qlib FAQ
 ############
 
 Qlib Frequently Asked Questions
-===============================
+================================
 .. contents::
     :depth: 1
     :local:
@@ -13,7 +13,7 @@ Qlib Frequently Asked Questions
 
 
 1. RuntimeError: An attempt has been made to start a new process before the current process has finished its bootstrapping phase...
------------------------------------------------------------------------------------------------------------------------------------
+------------------------------------------------------------------------------------------------------------------------------------
 
 .. code-block:: console
 
@@ -52,7 +52,7 @@ This is caused by the limitation of multiprocessing under windows OS. Please ref
 
 
 2. qlib.data.cache.QlibCacheException: It sees the key(...) of the redis lock has existed in your redis db now.
----------------------------------------------------------------------------------------------------------------
+-----------------------------------------------------------------------------------------------------------------
 
 It sees the key of the redis lock has existed in your redis db now. You can use the following command to clear your redis keys and rerun your commands
 
@@ -72,7 +72,7 @@ If the issue is not resolved, use ``keys *`` to find if multiple keys exist. If
 Also, feel free to post a new issue in our GitHub repository. We always check each issue carefully and try our best to solve them.
 
 3. ModuleNotFoundError: No module named 'qlib.data._libs.rolling'
------------------------------------------------------------------
+------------------------------------------------------------------------------------------------------------------------------------
 
 .. code-block:: python
 
@@ -101,7 +101,7 @@ Also, feel free to post a new issue in our GitHub repository. We always check ea
 
 
 4. BadNamespaceError: / is not a connected namespace
-----------------------------------------------------
+------------------------------------------------------------------------------------------------------------------------------------
 
 .. code-block:: python
 
@@ -125,7 +125,7 @@ Also, feel free to post a new issue in our GitHub repository. We always check ea
 
 
 5. TypeError: send() got an unexpected keyword argument 'binary'
-----------------------------------------------------------------
+------------------------------------------------------------------------------------------------------------------------------------
 
 .. code-block:: python
 

docs/Makefile

@@ -17,5 +17,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	pip install -r requirements.txt
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/advanced/PIT.rst

@@ -1,8 +1,8 @@
 .. _pit:
 
-============================
+===========================
 (P)oint-(I)n-(T)ime Database
-============================
+===========================
 .. currentmodule:: qlib
 
 

docs/advanced/alpha.rst

@@ -1,12 +1,12 @@
 .. _alpha:
 
-=========================
+===========================
 Building Formulaic Alphas 
-=========================
+===========================
 .. currentmodule:: qlib
 
 Introduction
-============
+===================
 
 In quantitative trading practice, designing novel factors that can explain and predict future asset returns are of vital importance to the profitability of a strategy. Such factors are usually called alpha factors, or alphas in short.
 
@@ -15,12 +15,12 @@ A formulaic alpha, as the name suggests, is a kind of alpha that can be presente
 
 
 Building Formulaic Alphas in ``Qlib``
-=====================================
+======================================
 
 In ``Qlib``, users can easily build formulaic alphas.
 
 Example
--------
+-----------------
 
 `MACD`, short for moving average convergence/divergence, is a formulaic alpha used in technical analysis of stock prices. It is designed to reveal changes in the strength, direction, momentum, and duration of a trend in a stock's price.
 
@@ -38,7 +38,7 @@ Example
 
         DIF = \frac{EMA(CLOSE, 12) - EMA(CLOSE, 26)}{CLOSE} 
 
-    `DEA` means a 9-period EMA of the DIF.
+    `DEA`means a 9-period EMA of the DIF.
 
     .. math::
 
@@ -79,7 +79,7 @@ Users can use ``Data Handler`` to build formulaic alphas `MACD` in qlib:
                SZ300315   -0.030557  0.012455
 
 Reference
-=========
+===========
 
 To learn more about ``Data Loader``, please refer to `Data Loader <../component/data.html#data-loader>`_
 

docs/advanced/serial.rst

@@ -1,16 +1,16 @@
 .. _serial:
 
-=============
+=================================
 Serialization
-=============
+=================================
 .. currentmodule:: qlib
 
 Introduction
-============
+===================
 ``Qlib`` supports dumping the state of ``DataHandler``, ``DataSet``, ``Processor`` and ``Model``, etc. into a disk and reloading them. 
 
 Serializable Class
-==================
+========================
 
 ``Qlib`` provides a base class ``qlib.utils.serial.Serializable``, whose state can be dumped into or loaded from disk in `pickle` format. 
 When users dump the state of a ``Serializable`` instance, the attributes of the instance whose name **does not** start with `_` will be saved on the disk.
@@ -19,7 +19,7 @@ However, users can use ``config`` method or override ``default_dump_all`` attrib
 Users can also override ``pickle_backend`` attribute to choose a pickle backend. The supported value is "pickle" (default and common) and "dill" (dump more things such as function, more information in `here <https://pypi.org/project/dill/>`_).
 
 Example
-=======
+==========================
 ``Qlib``'s serializable class includes  ``DataHandler``, ``DataSet``, ``Processor`` and ``Model``, etc., which are subclass of  ``qlib.utils.serial.Serializable``. 
 Specifically, ``qlib.data.dataset.DatasetH`` is one of them. Users can serialize ``DatasetH`` as follows.
 
@@ -41,5 +41,5 @@ A more detailed example is in this `link <https://github.com/microsoft/qlib/tree
 
 
 API
-===
+===================
 Please refer to `Serializable API <../reference/api.html#module-qlib.utils.serial.Serializable>`_.

docs/advanced/server.rst

@@ -1,13 +1,13 @@
 .. _server:
 
-=============================
+=================================
 ``Online`` & ``Offline`` mode
-=============================
+=================================
 .. currentmodule:: qlib
 
 
 Introduction
-============
+=============
 
 ``Qlib`` supports ``Online`` mode and ``Offline`` mode. Only the ``Offline`` mode is introduced in this document. 
 
@@ -18,12 +18,12 @@ The ``Online`` mode is designed to solve the following problems:
 - Make the data can be accessed in a remote way.
 
 Qlib-Server
-===========
+===============
 
 ``Qlib-Server`` is the assorted server system for ``Qlib``, which utilizes ``Qlib`` for basic calculations and provides extensive server system and cache mechanism. With QLibServer, the data provided for ``Qlib`` can be managed in a centralized manner. With ``Qlib-Server``, users can use ``Qlib`` in ``Online`` mode. 
 
 
 
 Reference
-=========
+=================
 If users are interested in ``Qlib-Server`` and ``Online`` mode, please refer to `Qlib-Server Project <https://github.com/microsoft/qlib-server>`_ and `Qlib-Server Document <https://qlib-server.readthedocs.io/en/latest/>`_.

docs/advanced/task_management.rst

@@ -1,13 +1,13 @@
 .. _task_management:
 
-===============
+=================================
 Task Management
-===============
+=================================
 .. currentmodule:: qlib
 
 
 Introduction
-============
+=============
 
 The `Workflow <../component/introduction.html>`_ part introduces how to run research workflow in a loosely-coupled way. But it can only execute one ``task`` when you use ``qrun``.
 To automatically generate and execute different tasks, ``Task Management`` provides a whole process including `Task Generating`_, `Task Storing`_, `Task Training`_ and `Task Collecting`_. 
@@ -18,7 +18,7 @@ With this module, users can run their ``task`` automatically at different period
 
 This whole process can be used in `Online Serving <../component/online.html>`_.
 
-An example of the entire process is shown `here <https://github.com/microsoft/qlib/tree/main/examples/model_rolling/task_manager_rolling.py>`__.
+An example of the entire process is shown `here <https://github.com/microsoft/qlib/tree/main/examples/model_rolling/task_manager_rolling.py>`_.
 
 Task Generating
 ===============
@@ -31,13 +31,12 @@ Here is the base class of ``TaskGen``:
 
 .. autoclass:: qlib.workflow.task.gen.TaskGen
     :members:
-    :noindex:
 
 ``Qlib`` provides a class `RollingGen <https://github.com/microsoft/qlib/tree/main/qlib/workflow/task/gen.py>`_ to generate a list of ``task`` of the dataset in different date segments.
-This class allows users to verify the effect of data from different periods on the model in one experiment. More information is `here <../reference/api.html#TaskGen>`__.
+This class allows users to verify the effect of data from different periods on the model in one experiment. More information is `here <../reference/api.html#TaskGen>`_.
 
 Task Storing
-============
+===============
 To achieve higher efficiency and the possibility of cluster operation, ``Task Manager`` will store all tasks in `MongoDB <https://www.mongodb.com/>`_.
 ``TaskManager`` can fetch undone tasks automatically and manage the lifecycle of a set of tasks with error handling.
 Users **MUST** finish the configuration of `MongoDB <https://www.mongodb.com/>`_ when using this module.
@@ -54,25 +53,22 @@ Users need to provide the MongoDB URL and database name for using ``TaskManager`
 
 .. autoclass:: qlib.workflow.task.manage.TaskManager
     :members:
-    :noindex:
 
-More information of ``Task Manager`` can be found in `here <../reference/api.html#TaskManager>`__.
+More information of ``Task Manager`` can be found in `here <../reference/api.html#TaskManager>`_.
 
 Task Training
-=============
+===============
 After generating and storing those ``task``, it's time to run the ``task`` which is in the *WAITING* status.
 ``Qlib`` provides a method called ``run_task`` to run those ``task`` in task pool, however, users can also customize how tasks are executed.
 An easy way to get the ``task_func`` is using ``qlib.model.trainer.task_train`` directly.
 It will run the whole workflow defined by ``task``, which includes *Model*, *Dataset*, *Record*.
 
 .. autofunction:: qlib.workflow.task.manage.run_task
-    :noindex:
 
 Meanwhile, ``Qlib`` provides a module called ``Trainer``. 
 
 .. autoclass:: qlib.model.trainer.Trainer
     :members:
-    :noindex:
 
 ``Trainer`` will train a list of tasks and return a list of model recorders.
 ``Qlib`` offer two kinds of Trainer, TrainerR is the simplest way and TrainerRM is based on TaskManager to help manager tasks lifecycle automatically. 

docs/changelog/changelog.rst

@@ -1 +1,2 @@
 .. include:: ../../CHANGES.rst
+	     

docs/component/data.rst

@@ -1,11 +1,11 @@
 .. _data:
 
-==================================
+================================
 Data Layer: Data Framework & Usage
-==================================
+================================
 
 Introduction
-============
+============================
 
 ``Data Layer`` provides user-friendly APIs to manage and retrieve data. It provides high-performance data infrastructure. 
 
@@ -24,20 +24,20 @@ The introduction of ``Data Layer`` includes the following parts.
 Here is a typical example of Qlib data workflow
 
 - Users download data and converting data into Qlib format(with filename suffix `.bin`).  In this step, typically only some basic data are stored on disk(such as OHLCV). 
-- Creating some basic features based on Qlib's expression Engine(e.g. "Ref($close, 60) / $close", the return of last 60 trading days). Supported operators in the expression engine can be found `here <https://github.com/microsoft/qlib/blob/main/qlib/data/ops.py>`__. This step is typically implemented in Qlib's `Data Loader <https://qlib.readthedocs.io/en/latest/component/data.html#data-loader>`_ which is a component of `Data Handler <https://qlib.readthedocs.io/en/latest/component/data.html#data-handler>`_ .
+- Creating some basic features based on Qlib's expression Engine(e.g. "Ref($close, 60) / $close", the return of last 60 trading days). Supported operators in the expression engine can be found `here <https://github.com/microsoft/qlib/blob/main/qlib/data/ops.py>`_. This step is typically implemented in Qlib's `Data Loader <https://qlib.readthedocs.io/en/latest/component/data.html#data-loader>`_ which is a component of `Data Handler <https://qlib.readthedocs.io/en/latest/component/data.html#data-handler>`_ .
-- If users require more complicated data processing (e.g. data normalization),  `Data Handler <https://qlib.readthedocs.io/en/latest/component/data.html#data-handler>`_ support user-customized processors to process data(some predefined processors can be found `here <https://github.com/microsoft/qlib/blob/main/qlib/data/dataset/processor.py>`__).  The processors are different from operators in expression engine. It is designed for some complicated data processing methods which is hard to supported in operators in expression engine.
+- If users require more complicated data processing (e.g. data normalization),  `Data Handler <https://qlib.readthedocs.io/en/latest/component/data.html#data-handler>`_ support user-customized processors to process data(some predefined processors can be found `here <https://github.com/microsoft/qlib/blob/main/qlib/data/dataset/processor.py>`_).  The processors are different from operators in expression engine. It is designed for some complicated data processing methods which is hard to supported in operators in expression engine.
 - At last, `Dataset <https://qlib.readthedocs.io/en/latest/component/data.html#dataset>`_ is responsible to prepare model-specific dataset from the processed data of Data Handler
 
 Data Preparation
-================
+============================
 
 Qlib Format Data
-----------------
+------------------
 
 We've specially designed a data structure to manage financial data, please refer to the `File storage design section in Qlib paper <https://arxiv.org/abs/2009.11189>`_ for detailed information.
 Such data will be stored with filename suffix `.bin` (We'll call them `.bin` file, `.bin` format, or qlib format). `.bin` file is designed for scientific computing on finance data.
 
-``Qlib`` provides two different off-the-shelf datasets, which can be accessed through this `link <https://github.com/microsoft/qlib/blob/main/qlib/contrib/data/handler.py>`__:
+``Qlib`` provides two different off-the-shelf datasets, which can be accessed through this `link <https://github.com/microsoft/qlib/blob/main/qlib/contrib/data/handler.py>`_:
 
 ========================  =================  ================
 Dataset                   US Market          China Market
@@ -47,19 +47,14 @@ Alpha360                  √                  √
 Alpha158                  √                  √
 ========================  =================  ================
 
-Also, ``Qlib`` provides a high-frequency dataset. Users can run a high-frequency dataset example through this `link <https://github.com/microsoft/qlib/tree/main/examples/highfreq>`__.
+Also, ``Qlib`` provides a high-frequency dataset. Users can run a high-frequency dataset example through this `link <https://github.com/microsoft/qlib/tree/main/examples/highfreq>`_.
 
 Qlib Format Dataset
--------------------
+--------------------
-``Qlib`` has provided an off-the-shelf dataset in `.bin` format, users could use the script ``scripts/get_data.py`` to download the China-Stock dataset as follows. User can also use numpy to load `.bin` file to validate data.
+``Qlib`` has provided an off-the-shelf dataset in `.bin` format, users could use the script ``scripts/get_data.py`` to download the China-Stock dataset as follows.
 The price volume data look different from the actual dealling price because of they are **adjusted** (`adjusted price <https://www.investopedia.com/terms/a/adjusted_closing_price.asp>`_).  And then you may find that the adjusted price may be different from different data sources. This is because different data sources may vary in the way of adjusting prices. Qlib normalize the price on first trading day of each stock to 1 when adjusting them. 
 Users can leverage `$factor` to get the original trading price (e.g. `$close / $factor` to get the original close price).
 
-Here are some discussions about the price adjusting of Qlib. 
-
-- https://github.com/microsoft/qlib/issues/991#issuecomment-1075252402
-
-
 .. code-block:: bash
 
     # download 1d
@@ -109,7 +104,7 @@ Automatic update of daily frequency data
 
 
 Converting CSV Format into Qlib Format
---------------------------------------
+-------------------------------------------
 
 ``Qlib`` has provided the script ``scripts/dump_bin.py`` to convert **any** data in CSV format into `.bin` files (``Qlib`` format) as long as they are in the correct format.
 
@@ -195,7 +190,7 @@ After conversion, users can find their Qlib format data in the directory `~/.qli
     If you want to use your own alpha-factor which can't be calculate by OCHLV, like PE, EPS and so on, you could add it to the CSV files with OHCLV together and then dump it to the Qlib format data.
 
 Stock Pool (Market)
--------------------
+--------------------------------
 
 ``Qlib`` defines `stock pool <https://github.com/microsoft/qlib/blob/main/examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml#L4>`_ as stock list and their date ranges. Predefined stock pools (e.g. csi300) may be imported as follows.
 
@@ -205,7 +200,7 @@ Stock Pool (Market)
 
 
 Multiple Stock Modes
---------------------
+--------------------------------
 
 ``Qlib`` now provides two different stock modes for users: China-Stock Mode & US-Stock Mode. Here are some different settings of these two modes:
 
@@ -247,14 +242,14 @@ The `trade unit` defines the unit number of stocks can be used in a trade, and t
 
 
 Data API
-========
+========================
 
 Data Retrieval
---------------
+---------------
 Users can use APIs in ``qlib.data`` to retrieve data, please refer to `Data Retrieval <../start/getdata.html>`_.
 
 Feature
--------
+------------------
 
 ``Qlib`` provides `Feature` and `ExpressionOps` to fetch the features according to users' needs.
 
@@ -269,7 +264,7 @@ Feature
 To know more about  ``Feature``, please refer to `Feature API <../reference/api.html#module-qlib.data.base>`_.
 
 Filter
-------
+-------------------
 ``Qlib`` provides `NameDFilter` and `ExpressionDFilter` to filter the instruments according to users' needs.
 
 - `NameDFilter`
@@ -304,51 +299,50 @@ Here is a simple example showing how to use filter in a basic ``Qlib`` workflow
 To know more about ``Filter``, please refer to `Filter API <../reference/api.html#module-qlib.data.filter>`_.
 
 Reference
----------
+-------------
 
 To know more about ``Data API``, please refer to `Data API <../reference/api.html#data>`_.
 
 
 Data Loader
-===========
+=================
 
 ``Data Loader`` in ``Qlib`` is designed to load raw data from the original data source. It will be loaded and used in the ``Data Handler`` module.
 
 QlibDataLoader
---------------
+---------------
 
 The ``QlibDataLoader`` class in ``Qlib`` is such an interface that allows users to load raw data from the ``Qlib`` data source.
 
 StaticDataLoader
-----------------
+---------------
 
 The ``StaticDataLoader`` class in ``Qlib`` is such an interface that allows users to load raw data from file or as provided.
 
 
 Interface
----------
+------------
 
 Here are some interfaces of the ``QlibDataLoader`` class:
 
 .. autoclass:: qlib.data.dataset.loader.DataLoader
     :members:
-    :noindex:
 
 API
----
+-----------
 
 To know more about ``Data Loader``, please refer to `Data Loader API <../reference/api.html#module-qlib.data.dataset.loader>`_.
 
 
 Data Handler
-============
+=================
 
 The ``Data Handler`` module in ``Qlib`` is designed to handler those common data processing methods which will be used by most of the models.
 
 Users can use ``Data Handler`` in an automatic workflow by ``qrun``, refer to `Workflow: Workflow Management <workflow.html>`_ for more details. 
 
 DataHandlerLP
--------------
+--------------
 
 In addition to use ``Data Handler`` in an automatic workflow with ``qrun``, ``Data Handler`` can be used as an independent module, by which users can easily preprocess data (standardization, remove NaN, etc.) and build datasets. 
 
@@ -356,13 +350,12 @@ In order to achieve so, ``Qlib`` provides a base class `qlib.data.dataset.DataHa
 
 
 Interface
----------
+----------------------
 
 Here are some important interfaces that ``DataHandlerLP`` provides:
 
 .. autoclass:: qlib.data.dataset.handler.DataHandlerLP
     :members: __init__, fetch, get_cols
-    :noindex:
 
 
 If users want to load features and labels by config, users can define a new handler and call the static method `parse_config_to_fields` of ``qlib.contrib.data.handler.Alpha158``.
@@ -371,7 +364,7 @@ Also, users can pass ``qlib.contrib.data.processor.ConfigSectionProcessor`` that
 
 
 Processor
----------
+----------
 
 The ``Processor`` module in ``Qlib`` is designed to be learnable and it is responsible for handling data processing such as `normalization` and `drop none/nan features/labels`.
 
@@ -394,7 +387,7 @@ Users can also create their own `processor` by inheriting the base class of ``Pr
 To know more about ``Processor``, please refer to `Processor API <../reference/api.html#module-qlib.data.dataset.processor>`_.
 
 Example
--------
+--------------
 
 ``Data Handler`` can be run with ``qrun`` by modifying the configuration file, and can also be used as a single module. 
 
@@ -434,13 +427,13 @@ Qlib provides implemented data handler `Alpha158`. The following example shows h
 .. note:: In the ``Alpha158``, ``Qlib`` uses the label `Ref($close, -2)/Ref($close, -1) - 1` that means the change from T+1 to T+2, rather than `Ref($close, -1)/$close - 1`, of which the reason is that when getting the T day close price of a china stock, the stock can be bought on T+1 day and sold on T+2 day.
 
 API
----
+---------
 
 To know more about ``Data Handler``, please refer to `Data Handler API <../reference/api.html#module-qlib.data.dataset.handler>`_.
 
 
 Dataset
-=======
+=================
 
 The ``Dataset`` module in ``Qlib`` aims to prepare data for model training and inferencing.
 
@@ -453,35 +446,32 @@ The ``DatasetH`` class is the `dataset` with `Data Handler`. Here is the most im
 
 .. autoclass:: qlib.data.dataset.__init__.DatasetH
     :members:
-    :noindex:
 
 API
----
+---------
 
 To know more about ``Dataset``, please refer to `Dataset API <../reference/api.html#dataset>`_.
 
 
 Cache
-=====
+==========
 
 ``Cache`` is an optional module that helps accelerate providing data by saving some frequently-used data as cache file. ``Qlib`` provides a `Memcache` class to cache the most-frequently-used data in memory, an inheritable `ExpressionCache` class, and an inheritable `DatasetCache` class.
 
 Global Memory Cache
--------------------
+---------------------
 
 `Memcache` is a global memory cache mechanism that composes of three `MemCacheUnit` instances to cache **Calendar**, **Instruments**, and **Features**. The `MemCache` is defined globally in `cache.py` as `H`. Users can use `H['c'], H['i'], H['f']` to get/set `memcache`.
 
 .. autoclass:: qlib.data.cache.MemCacheUnit
     :members:
-    :noindex:
 
 .. autoclass:: qlib.data.cache.MemCache
     :members:
-    :noindex:
 
 
 ExpressionCache
----------------
+-----------------
 
 `ExpressionCache` is a cache mechanism that saves expressions such as **Mean($close, 5)**. Users can inherit this base class to define their own cache mechanism that saves expressions according to the following steps.
 
@@ -492,12 +482,11 @@ The following shows the details about the interfaces:
 
 .. autoclass:: qlib.data.cache.ExpressionCache
     :members:
-    :noindex:
 
 ``Qlib`` has currently provided implemented disk cache `DiskExpressionCache` which inherits from `ExpressionCache` . The expressions data will be stored in the disk.
 
 DatasetCache
-------------
+-----------------
 
 `DatasetCache` is a cache mechanism that saves datasets. A certain dataset is regulated by a stock pool configuration (or a series of instruments, though not recommended), a list of expressions or static feature fields, the start time, and end time for the collected features and the frequency. Users can inherit this base class to define their own cache mechanism that saves datasets according to the following steps.
 
@@ -508,18 +497,17 @@ The following shows the details about the interfaces:
 
 .. autoclass:: qlib.data.cache.DatasetCache
     :members:
-    :noindex:
 
 ``Qlib`` has currently provided implemented disk cache `DiskDatasetCache` which inherits from `DatasetCache` . The datasets' data will be stored in the disk.
 
 
 
 Data and Cache File Structure
-=============================
+==================================
 
 We've specially designed a file structure to manage data and cache, please refer to the `File storage design section in Qlib paper <https://arxiv.org/abs/2009.11189>`_ for detailed information. The file structure of data and cache is listed as follows.
 
-.. code-block::
+.. code-block:: json
 
     - data/
         [raw data] updated by data providers
@@ -548,3 +536,4 @@ We've specially designed a file structure to manage data and cache, please refer
                 - .meta : an assorted meta file recording the stockpool config, field names and visit times
                 - .index : an assorted index file recording the line index of all calendars
             - ...
+

docs/component/highfreq.rst

@@ -1,40 +1,38 @@
 .. _highfreq:
 
-========================================================================
+============================================
 Design of Nested Decision Execution Framework for High-Frequency Trading
-========================================================================
+============================================
 .. currentmodule:: qlib
 
 Introduction
-============
+===================
 
-Daily trading (e.g. portfolio management) and intraday trading (e.g. orders execution) are two hot topics in Quant investment and are usually studied separately.
+Daily trading (e.g. portfolio management) and intraday trading (e.g. orders execution) are two hot topics in Quant investment and usually studied separately.
 
 To get the join trading performance of daily and intraday trading, they must interact with each other and run backtest jointly.
-In order to support the joint backtest strategies at multiple levels, a corresponding framework is required. None of the publicly available high-frequency trading frameworks considers multi-level joint trading, which makes the backtesting aforementioned inaccurate.
+In order to support the joint backtest strategies in multiple levels, a corresponding framework is required. None of the publicly available high-frequency trading frameworks considers multi-level joint trading, which make the backtesting aforementioned inaccurate.
 
 Besides backtesting, the optimization of strategies from different levels is not standalone and can be affected by each other.
-For example, the best portfolio management strategy may change with the performance of order executions(e.g. a portfolio with higher turnover may become a better choice when we improve the order execution strategies).
+For example, the best portfolio management strategy may change with the performance of order executions(e.g. a portfolio with higher turnover may becomes a better choice when we improve the order execution strategies).
-To achieve overall good performance, it is necessary to consider the interaction of strategies at a different levels.
+To achieve the overall good performance , it is necessary to consider the interaction of strategies in different level. 
 
-Therefore, building a new framework for trading on multiple levels becomes necessary to solve the various problems mentioned above, for which we designed a nested decision execution framework that considers the interaction of strategies.
+Therefore, building a new framework for trading in multiple levels becomes necessary to solve the various problems mentioned above, for which we designed a nested decision execution framework that consider the interaction of strategies.
 
 .. image:: ../_static/img/framework.svg
 
 The design of the framework is shown in the yellow part in the middle of the figure above. Each level consists of ``Trading Agent`` and ``Execution Env``. ``Trading Agent`` has its own data processing module (``Information Extractor``), forecasting module (``Forecast Model``) and decision generator (``Decision Generator``). The trading algorithm generates the decisions by the ``Decision Generator`` based on the forecast signals output by the ``Forecast Module``, and the decisions generated by the trading algorithm are passed to the ``Execution Env``, which returns the execution results. 
 
-The frequency of the trading algorithm, decision content and execution environment can be customized by users (e.g. intraday trading, daily-frequency trading, weekly-frequency trading), and the execution environment can be nested with finer-grained trading algorithm and execution environment inside (i.e. sub-workflow in the figure, e.g. daily-frequency orders can be turned into finer-grained decisions by splitting orders within the day). The flexibility of the nested decision execution framework makes it easy for users to explore the effects of combining different levels of trading strategies and break down the optimization barriers between different levels of the trading algorithm.
+The frequency of trading algorithm, decision content and execution environment can be customized by users (e.g. intraday trading, daily-frequency trading, weekly-frequency trading), and the execution environment can be nested with finer-grained trading algorithm and execution environment inside (i.e. sub-workflow in the figure, e.g. daily-frequency orders can be turned into finer-grained decisions by splitting orders within the day). The flexibility of nested decision execution framework makes it easy for users to explore the effects of combining different levels of trading strategies and break down the optimization barriers between different levels of trading algorithm.
-
-The optimization for the nested decision execution framework can be implemented with the support of `QlibRL <https://qlib.readthedocs.io/en/latest/component/rl.html>`_. To know more about how to use the QlibRL, go to API Reference: `RL API <../reference/api.html#rl>`_. 
 
 Example
-=======
+===========================
 
-An example of a nested decision execution framework for high-frequency can be found `here <https://github.com/microsoft/qlib/blob/main/examples/nested_decision_execution/workflow.py>`_.
+An example of nested decision execution framework for high-frequency can be found `here <https://github.com/microsoft/qlib/blob/main/examples/nested_decision_execution/workflow.py>`_.
 
 
-Besides, the above examples, here are some other related works about high-frequency trading in Qlib.
+Besides, the above examples, here are some other related work about high-frequency trading in Qlib.
 
 - `Prediction with high-frequency data <https://github.com/microsoft/qlib/tree/main/examples/highfreq#benchmarks-performance-predicting-the-price-trend-in-high-frequency-data>`_
-- `Examples <https://github.com/microsoft/qlib/blob/main/examples/orderbook_data/>`_ to extract features from high-frequency data without fixed frequency.
+- `Examples <https://github.com/microsoft/qlib/blob/main/examples/orderbook_data/>`_ to extract features form high-frequency data without fixed frequency.
 - `A paper <https://github.com/microsoft/qlib/tree/high-freq-execution#high-frequency-execution>`_ for high-frequency trading.

docs/component/meta.rst

@@ -1,17 +1,17 @@
 .. _meta:
 
-======================================================
+=================================
 Meta Controller: Meta-Task & Meta-Dataset & Meta-Model
-======================================================
+=================================
 .. currentmodule:: qlib
 
 
 Introduction
-============
+=============
 ``Meta Controller`` provides guidance to ``Forecast Model``, which aims to learn regular patterns among a series of forecasting tasks and use learned patterns to guide forthcoming forecasting tasks. Users can implement their own meta-model instance based on ``Meta Controller`` module.
 
 Meta Task
-=========
+=============
 
 A `Meta Task` instance is the basic element in the meta-learning framework. It saves the data that can be used for the `Meta Model`. Multiple `Meta Task` instances may share the same `Data Handler`, controlled by `Meta Dataset`. Users should use `prepare_task_data()` to obtain the data that can be directly fed into the `Meta Model`.
 
@@ -19,7 +19,7 @@ A `Meta Task` instance is the basic element in the meta-learning framework. It s
     :members:
 
 Meta Dataset
-============
+=============
 
 `Meta Dataset` controls the meta-information generating process. It is on the duty of providing data for training the `Meta Model`. Users should use `prepare_tasks` to retrieve a list of `Meta Task` instances.
 
@@ -27,7 +27,7 @@ Meta Dataset
     :members:
 
 Meta Model
-==========
+=============
 
 General Meta Model
 ------------------
@@ -39,14 +39,14 @@ General Meta Model
     :members:
 
 Meta Task Model
----------------
+------------------
 This type of meta-model may interact with task definitions directly. Then, the `Meta Task Model` is the class for them to inherit from. They guide the base tasks by modifying the base task definitions. The function `prepare_tasks` can be used to obtain the modified base task definitions.
 
 .. autoclass:: qlib.model.meta.model.MetaTaskModel
     :members:
 
 Meta Guide Model
-----------------
+------------------
 This type of meta-model participates in the training process of the base forecasting model. The meta-model may guide the base forecasting models during their training to improve their performances.
 
 .. autoclass:: qlib.model.meta.model.MetaGuideModel
@@ -54,7 +54,7 @@ This type of meta-model participates in the training process of the base forecas
 
 
 Example
-=======
+=============
 ``Qlib`` provides an implementation of ``Meta Model`` module, ``DDG-DA``, 
 which adapts to the market dynamics. 
 

docs/component/model.rst

@@ -1,11 +1,11 @@
 .. _model:
 
-===========================================
+============================================
 Forecast Model: Model Training & Prediction
-===========================================
+============================================
 
 Introduction
-============
+===================
 
 ``Forecast Model`` is designed to make the `prediction score` about stocks. Users can use the ``Forecast Model`` in an automatic workflow by ``qrun``, please refer to `Workflow: Workflow Management <workflow.html>`_.  
 
@@ -20,14 +20,13 @@ The base class provides the following interfaces:
 
 .. autoclass:: qlib.model.base.Model
     :members:
-    :noindex:
 
 ``Qlib`` also provides a base class `qlib.model.base.ModelFT <../reference/api.html#qlib.model.base.ModelFT>`_, which includes the method for finetuning the model.
     
 For other interfaces such as `finetune`, please refer to `Model API <../reference/api.html#module-qlib.model.base>`_.
 
 Example
-=======
+==================
 
 ``Qlib``'s `Model Zoo` includes models such as ``LightGBM``, ``MLP``, ``LSTM``, etc.. These models are treated as the baselines of ``Forecast Model``. The following steps show how to run`` LightGBM`` as an independent module.
 
@@ -112,11 +111,11 @@ By default, the meaning of the score is normally the rating of the instruments b
 
 
 Custom Model
-============
+===================
 
 Qlib supports custom models. If users are interested in customizing their own models and integrating the models into ``Qlib``, please refer to `Custom Model Integration <../start/integration.html>`_.
 
 
 API
-===
+===================
 Please refer to `Model API <../reference/api.html#module-qlib.model.base>`_.

docs/component/online.rst

@@ -1,13 +1,13 @@
-.. _online_serving:
+.. _online:
 
-==============
+=================================
 Online Serving
-==============
+=================================
 .. currentmodule:: qlib
 
 
 Introduction
-============
+=============
 
 .. image:: ../_static/img/online_serving.png
     :align: center
@@ -28,29 +28,25 @@ Known limitations currently
 
 
 Online Manager
-==============
+=============
 
 .. automodule:: qlib.workflow.online.manager
     :members:
-    :noindex:
 
 Online Strategy
-===============
+=============
 
 .. automodule:: qlib.workflow.online.strategy
     :members:
-    :noindex:
 
 Online Tool
-===========
+=============
 
 .. automodule:: qlib.workflow.online.utils
     :members:
-    :noindex:
 
 Updater
-=======
+=============
 
 .. automodule:: qlib.workflow.online.update
     :members:
-    :noindex:

docs/component/recorder.rst

@@ -6,7 +6,7 @@ Qlib Recorder: Experiment Management
 .. currentmodule:: qlib
 
 Introduction
-============
+===================
 ``Qlib`` contains an experiment management system named ``QlibRecorder``, which is designed to help users handle experiment and analyse results in an efficient way. 
 
 There are three components of the system:
@@ -40,7 +40,7 @@ This experiment management system defines a set of interface and provided a conc
 If users set the implementation of ``ExpManager`` to be ``MLflowExpManager``, they can use the command `mlflow ui` to visualize and check the experiment results. For more information, please refer to the related documents `here <https://www.mlflow.org/docs/latest/cli.html#mlflow-ui>`_.
 
 Qlib Recorder
-=============
+===================
 ``QlibRecorder`` provides a high level API for users to use the experiment management system. The interfaces are wrapped in the variable ``R`` in ``Qlib``, and users can directly use ``R`` to interact with the system. The following command shows how to import ``R`` in Python:
 
 .. code-block:: Python
@@ -55,31 +55,29 @@ Here are the available interfaces of ``QlibRecorder``:
     :members:
 
 Experiment Manager
-==================
+===================
 
 The ``ExpManager`` module in ``Qlib`` is responsible for managing different experiments. Most of the APIs of ``ExpManager`` are similar to ``QlibRecorder``, and the most important API will be the ``get_exp`` method. User can directly refer to the documents above for some detailed information about how to use the ``get_exp`` method.
 
 .. autoclass:: qlib.workflow.expm.ExpManager
     :members: get_exp, list_experiments
-    :noindex:
 
 For other interfaces such as `create_exp`, `delete_exp`, please refer to `Experiment Manager API <../reference/api.html#experiment-manager>`_.
 
 Experiment
-==========
+===================
 
 The ``Experiment`` class is solely responsible for a single experiment, and it will handle any operations that are related to an experiment. Basic methods such as `start`, `end` an experiment are included. Besides, methods related to `recorders` are also available: such methods include `get_recorder` and `list_recorders`.
 
 .. autoclass:: qlib.workflow.exp.Experiment
     :members: get_recorder, list_recorders
-    :noindex:
 
 For other interfaces such as `search_records`, `delete_recorder`, please refer to `Experiment API <../reference/api.html#experiment>`_.
 
 ``Qlib`` also provides a default ``Experiment``, which will be created and used under certain situations when users use the APIs such as `log_metrics` or `get_exp`. If the default ``Experiment`` is used, there will be related logged information when running ``Qlib``. Users are able to change the name of the default ``Experiment`` in the config file of ``Qlib`` or during ``Qlib``'s `initialization <../start/initialization.html#parameters>`_, which is set to be '`Experiment`'.
 
 Recorder
-========
+===================
 
 The ``Recorder`` class is responsible for a single recorder. It will handle some detailed operations such as ``log_metrics``, ``log_params`` of a single run. It is designed to help user to easily track results and things being generated during a run.
 
@@ -87,12 +85,11 @@ Here are some important APIs that are not included in the ``QlibRecorder``:
 
 .. autoclass:: qlib.workflow.recorder.Recorder
     :members: list_artifacts, list_metrics, list_params, list_tags
-    :noindex:
 
 For other interfaces such as `save_objects`, `load_object`, please refer to `Recorder API <../reference/api.html#recorder>`_.
 
 Record Template
-===============
+===================
 
 The ``RecordTemp`` class is a class that enables generate experiment results such as IC and backtest in a certain format. We have provided three different `Record Template` class:
 
@@ -110,7 +107,7 @@ Here is a simple example of what is done in ``SigAnaRecord``, which users can re
 
 - ``PortAnaRecord``: This class generates the results of `backtest`. The detailed information about `backtest` as well as the available `strategy`, users can refer to `Strategy <../component/strategy.html>`_ and `Backtest <../component/backtest.html>`_.
 
-Here is a simple example of what is done in ``PortAnaRecord``, which users can refer to if they want to do backtest based on their own prediction and label.
+Here is a simple exampke of what is done in ``PortAnaRecord``, which users can refer to if they want to do backtest based on their own prediction and label.
 
 .. code-block:: Python
 

docs/component/report.rst

@@ -1,11 +1,11 @@
 .. _report:
 
-=======================================
+==========================================
 Analysis: Evaluation & Results Analysis
-=======================================
+==========================================
 
 Introduction
-============
+===================
 
 ``Analysis`` is designed to show the graphical reports of ``Intraday Trading`` , which helps users to evaluate and analyse investment portfolios visually. The following are some graphics to view:
 
@@ -24,7 +24,7 @@ All of the accumulated profit metrics(e.g. return, max drawdown) in Qlib are cal
 This avoids the metrics or the plots being skewed exponentially over time.
 
 Graphical Reports
-=================
+===================
 
 Users can run the following code to get all supported reports.
 
@@ -41,17 +41,16 @@ Users can run the following code to get all supported reports.
 
 
 Usage & Example
-===============
+===================
 
 Usage of `analysis_position.report`
 -----------------------------------
 
 API
-~~~
+~~~~~~~~~~~~~~~~
 
 .. automodule:: qlib.contrib.report.analysis_position.report
     :members:
-    :noindex:
 
 Graphical Result
 ~~~~~~~~~~~~~~~~
@@ -90,15 +89,14 @@ Usage of `analysis_position.score_ic`
 -------------------------------------
 
 API
-~~~
+~~~~~~~~~~~~~~~~
 
 .. automodule:: qlib.contrib.report.analysis_position.score_ic
     :members:
-    :noindex:
 
 
 Graphical Result
-~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 
 .. note:: 
 
@@ -106,7 +104,7 @@ Graphical Result
     - Axis Y: 
         - `ic`
             The `Pearson correlation coefficient` series between `label` and `prediction score`.
-            In the above example, the `label` is formulated as `Ref($close, -2)/Ref($close, -1)-1`. Please refer to `Data Feature <data.html#feature>`_ for more details.
+            In the above example, the `label` is formulated as `Ref($close, -1)/$close - 1`. Please refer to `Data Feature <data.html#feature>`_ for more details.
                 
         - `rank_ic`
             The `Spearman's rank correlation coefficient` series between `label` and `prediction score`.
@@ -146,18 +144,17 @@ Graphical Result
 
 
 Usage of `analysis_position.risk_analysis`
-------------------------------------------
+----------------------------------------------
 
 API
-~~~
+~~~~~~~~~~~~~~~~
 
 .. automodule:: qlib.contrib.report.analysis_position.risk_analysis
     :members:
-    :noindex:
 
 
 Graphical Result
-~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 
 .. note::
 
@@ -177,7 +174,6 @@ Graphical Result
                 The `Information Ratio` without cost.
             - `excess_return_with_cost`
                 The `Information Ratio` with cost.
-
             To know more about `Information Ratio`, please refer to `Information Ratio – IR <https://www.investopedia.com/terms/i/informationratio.asp>`_.
         -  `max_drawdown`
             - `excess_return_without_cost`
@@ -230,17 +226,17 @@ Graphical Result
 
 ..
 .. Usage of `analysis_position.rank_label`
-.. ---------------------------------------
+.. ----------------------------------------------
 ..
 .. API
-.. ~~~
+.. ~~~~~
 ..
 .. .. automodule:: qlib.contrib.report.analysis_position.rank_label
 ..     :members:
 ..
 ..
 .. Graphical Result
-.. ~~~~~~~~~~~~~~~~
+.. ~~~~~~~~~~~~~~~~~
 ..
 .. .. note:: 
 ..
@@ -266,18 +262,17 @@ Graphical Result
 ..
 
 Usage of `analysis_model.analysis_model_performance`
-----------------------------------------------------
+-----------------------------------------------------
 
 API
-~~~
+~~~~~
 
 .. automodule:: qlib.contrib.report.analysis_model.analysis_model_performance
     :members:
-    :noindex:
 
 
 Graphical Results
-~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~
 
 .. note::
 

docs/component/rl/framework.rst

@@ -1,45 +0,0 @@
-The Framework of QlibRL
-=======================
-
-QlibRL contains a full set of components that cover the entire lifecycle of an RL pipeline, including building the simulator of the market, shaping states & actions, training policies (strategies), and backtesting strategies in the simulated environment.
-
-QlibRL is basically implemented with the support of Tianshou and Gym frameworks. The high-level structure of QlibRL is demonstrated below:
-
-.. image:: ../../_static/img/QlibRL_framework.png
-   :width: 600
-   :align: center
-
-Here, we briefly introduce each component in the figure.
-
-EnvWrapper
-------------
-EnvWrapper is the complete capsulation of the simulated environment. It receives actions from outside (policy/strategy/agent), simulates the changes in the market, and then replies rewards and updated states, thus forming an interaction loop.
-
-In QlibRL, EnvWrapper is a subclass of gym.Env, so it implements all necessary interfaces of gym.Env. Any classes or pipelines that accept gym.Env should also accept EnvWrapper. Developers do not need to implement their own EnvWrapper to build their own environment. Instead, they only need to implement 4 components of the EnvWrapper:
-
-- `Simulator`
-    The simulator is the core component responsible for the environment simulation. Developers could implement all the logic that is directly related to the environment simulation in the Simulator in any way they like. In QlibRL, there are already two implementations of Simulator for single asset trading: 1) ``SingleAssetOrderExecution``, which is built based on Qlib's backtest toolkits and hence considers a lot of practical trading details but is slow. 2) ``SimpleSingleAssetOrderExecution``, which is built based on a simplified trading simulator, which ignores a lot of details (e.g. trading limitations, rounding) but is quite fast.
-- `State interpreter` 
-    The state interpreter is responsible for "interpret" states in the original format (format provided by the simulator) into states in a format that the policy could understand. For example, transform unstructured raw features into numerical tensors.
-- `Action interpreter` 
-    The action interpreter is similar to the state interpreter. But instead of states, it interprets actions generated by the policy, from the format provided by the policy to the format that is acceptable to the simulator.
-- `Reward function` 
-    The reward function returns a numerical reward to the policy after each time the policy takes an action. 
-
-EnvWrapper will organically organize these components. Such decomposition allows for better flexibility in development. For example, if the developers want to train multiple types of policies in the same environment, they only need to design one simulator and design different state interpreters/action interpreters/reward functions for different types of policies.
-
-QlibRL has well-defined base classes for all these 4 components. All the developers need to do is define their own components by inheriting the base classes and then implementing all interfaces required by the base classes. The API for the above base components can be found `here <../../reference/api.html#module-qlib.rl>`__.
-
-Policy
-------------
-QlibRL directly uses Tianshou's policy. Developers could use policies provided by Tianshou off the shelf, or implement their own policies by inheriting Tianshou's policies.
-
-Training Vessel & Trainer
--------------------------
-As stated by their names, training vessels and trainers are helper classes used in training. A training vessel is a ship that contains a simulator/interpreters/reward function/policy, and it controls algorithm-related parts of training. Correspondingly, the trainer is responsible for controlling the runtime parts of training.
-
-As you may have noticed, a training vessel itself holds all the required components to build an EnvWrapper rather than holding an instance of EnvWrapper directly. This allows the training vessel to create duplicates of EnvWrapper dynamically when necessary (for example, under parallel training).
-
-With a training vessel, the trainer could finally launch the training pipeline by simple, Scikit-learn-like interfaces (i.e., ``trainer.fit()``).
-
-The API for Trainer and TrainingVessel and can be found `here <../../reference/api.html#module-qlib.rl.trainer>`__.

docs/component/rl/overall.rst

@@ -1,50 +0,0 @@
-=====================================================
-Reinforcement Learning in Quantitative Trading
-=====================================================
-
-Reinforcement Learning
-======================
-Different from supervised learning tasks such as classification tasks and regression tasks. Another important paradigm in machine learning is Reinforcement Learning, 
-which attempts to optimize an accumulative numerical reward signal by directly interacting with the environment under a few assumptions such as Markov Decision Process(MDP).
-
-As demonstrated in the following figure, an RL system consists of four elements, 1)the agent 2) the environment the agent interacts with 3) the policy that the agent follows to take actions on the environment and 4)the reward signal from the environment to the agent. 
-In general, the agent can perceive and interpret its environment, take actions and learn through reward, to seek long-term and maximum overall reward to achieve an optimal solution.
-
-.. image:: ../../_static/img/RL_framework.png
-   :width: 300
-   :align: center 
-
-RL attempts to learn to produce actions by trial and error. 
-By sampling actions and then observing which one leads to our desired outcome, a policy is obtained to generate optimal actions. 
-In contrast to supervised learning, RL learns this not from a label but from a time-delayed label called a reward. 
-This scalar value lets us know whether the current outcome is good or bad. 
-In a word, the target of RL is to take actions to maximize reward.
-
-The Qlib Reinforcement Learning toolkit (QlibRL) is an RL platform for quantitative investment, which provides support to implement the RL algorithms in Qlib.
-
-
-Potential Application Scenarios in Quantitative Trading
-=======================================================
-RL methods have already achieved outstanding achievement in many applications, such as game playing, resource allocating, recommendation, marketing and advertising, etc.
-Investment is always a continuous process, taking the stock market as an example, investors need to control their positions and stock holdings by one or more buying and selling behaviors, to maximize the investment returns.
-Besides, each buy and sell decision is made by investors after fully considering the overall market information and stock information. 
-From the view of an investor, the process could be described as a continuous decision-making process generated according to interaction with the market, such problems could be solved by the RL algorithms. 
-Following are some scenarios where RL can potentially be used in quantitative investment.
-
-Portfolio Construction
-----------------------
-Portfolio construction is a process of selecting securities optimally by taking a minimum risk to achieve maximum returns. With an RL-based solution, an agent allocates stocks at every time step by obtaining information for each stock and the market. The key is to develop of policy for building a portfolio and make the policy able to pick the optimal portfolio. 
-
-Order Execution
----------------
-As a fundamental problem in algorithmic trading, order execution aims at fulfilling a specific trading order, either liquidation or acquirement, for a given instrument. Essentially, the goal of order execution is twofold: it not only requires to fulfill the whole order but also targets a more economical execution with maximizing profit gain (or minimizing capital loss). The order execution with only one order of liquidation or acquirement is called single-asset order execution.
-
-Considering stock investment always aim to pursue long-term maximized profits, it usually manifests as a sequential process of continuously adjusting the asset portfolios, execution for multiple orders, including order of liquidation and acquirement, brings more constraints and makes the sequence of execution for different orders should be considered, e.g. before executing an order to buy some stocks, we have to sell at least one stock. The order execution with multiple assets is called multi-asset order execution. 
-
-According to the order execution’s trait of sequential decision-making, an RL-based solution could be applied to solve the order execution. With an RL-based solution, an agent optimizes execution strategy by interacting with the market environment. 
-
-With QlibRL, the RL algorithm in the above scenarios can be easily implemented.
-
-Nested Portfolio Construction and Order Executor
-------------------------------------------------
-QlibRL makes it possible to jointly optimize different levels of strategies/models/agents. Take `Nested Decision Execution Framework <https://github.com/microsoft/qlib/blob/main/examples/nested_decision_execution>`_ as an example, the optimization of order execution strategy and portfolio management strategies can interact with each other to maximize returns.

docs/component/rl/quickstart.rst

@@ -1,175 +0,0 @@
-
-Quick Start
-============
-.. currentmodule:: qlib
-
-QlibRL provides an example of an implementation of a single asset order execution task and the following is an example of the config file to train with QlibRL.
-
-.. code-block:: yaml
-
-    simulator:
-        # Each step contains 30mins
-        time_per_step: 30
-        # Upper bound of volume, should be null or a float between 0 and 1, if it is a float, represent upper bound is calculated by the percentage of the market volume
-        vol_limit: null
-    env:
-        # Concurrent environment workers.
-        concurrency: 1
-        # dummy or subproc or shmem. Corresponding to `parallelism in tianshou <https://tianshou.readthedocs.io/en/master/api/tianshou.env.html#vectorenv>`_.
-        parallel_mode: dummy
-    action_interpreter:
-        class: CategoricalActionInterpreter
-        kwargs:
-            # Candidate actions, it can be a list with length L: [a_1, a_2,..., a_L] or an integer n, in which case the list of length n+1 is auto-generated, i.e., [0, 1/n, 2/n,..., n/n].
-            values: 14
-            # Total number of steps (an upper-bound estimation)
-            max_step: 8
-        module_path: qlib.rl.order_execution.interpreter
-    state_interpreter:
-        class: FullHistoryStateInterpreter
-        kwargs:
-            # Number of dimensions in data.
-            data_dim: 6
-            # Equal to the total number of records. For example, in SAOE per minute, data_ticks is the length of the day in minutes.
-            data_ticks: 240
-            # The total number of steps (an upper-bound estimation). For example, 390min / 30min-per-step = 13 steps.
-            max_step: 8
-            # Provider of the processed data.
-            processed_data_provider:
-                class: PickleProcessedDataProvider
-                module_path: qlib.rl.data.pickle_styled
-                kwargs:
-                    data_dir: ./data/pickle_dataframe/feature
-        module_path: qlib.rl.order_execution.interpreter
-    reward:
-        class: PAPenaltyReward
-        kwargs:
-            # The penalty for a large volume in a short time.
-            penalty: 100.0
-        module_path: qlib.rl.order_execution.reward
-    data:
-        source:
-            order_dir: ./data/training_order_split
-            data_dir: ./data/pickle_dataframe/backtest
-            # number of time indexes
-            total_time: 240
-            # start time index
-            default_start_time: 0
-            # end time index
-            default_end_time: 240
-            proc_data_dim: 6
-        num_workers: 0
-        queue_size: 20
-    network:
-        class: Recurrent
-        module_path: qlib.rl.order_execution.network
-    policy:
-        class: PPO
-        kwargs:
-            lr: 0.0001
-        module_path: qlib.rl.order_execution.policy
-    runtime:
-        seed: 42
-        use_cuda: false
-    trainer:
-        max_epoch: 2
-        # Number of episodes collected in each training iteration
-        repeat_per_collect: 5
-        earlystop_patience: 2
-        # Episodes per collect at training.
-        episode_per_collect: 20
-        batch_size: 16
-        # Perform validation every n iterations
-        val_every_n_epoch: 1
-        checkpoint_path: ./checkpoints
-        checkpoint_every_n_iters: 1
-
-
-And the config file for backtesting:
-
-.. code-block:: yaml
-
-    order_file: ./data/backtest_orders.csv
-    start_time: "9:45"
-    end_time: "14:44"
-    qlib:
-        provider_uri_1min: ./data/bin
-        feature_root_dir: ./data/pickle
-        # feature generated by today's information
-        feature_columns_today: [
-            "$open", "$high", "$low", "$close", "$vwap", "$volume",
-        ]
-        # feature generated by yesterday's information
-        feature_columns_yesterday: [
-            "$open_v1", "$high_v1", "$low_v1", "$close_v1", "$vwap_v1", "$volume_v1",
-        ]
-    exchange:
-        # the expression for buying and selling stock limitation
-        limit_threshold: ['$close == 0', '$close == 0']
-        # deal price for buying and selling
-        deal_price: ["If($close == 0, $vwap, $close)", "If($close == 0, $vwap, $close)"]
-    volume_threshold:
-        # volume limits are both buying and selling, "cum" means that this is a cumulative value over time
-        all: ["cum", "0.2 * DayCumsum($volume, '9:45', '14:44')"]
-        # the volume limits of buying
-        buy: ["current", "$close"]
-        # the volume limits of selling, "current" means that this is a real-time value and will not accumulate over time
-        sell: ["current", "$close"]
-    strategies: 
-        30min: 
-            class: TWAPStrategy
-            module_path: qlib.contrib.strategy.rule_strategy
-            kwargs: {}
-        1day: 
-            class: SAOEIntStrategy
-            module_path: qlib.rl.order_execution.strategy
-            kwargs:
-            state_interpreter:
-                class: FullHistoryStateInterpreter
-                module_path: qlib.rl.order_execution.interpreter
-                kwargs:
-                max_step: 8
-                data_ticks: 240
-                data_dim: 6
-                processed_data_provider:
-                    class: PickleProcessedDataProvider
-                    module_path: qlib.rl.data.pickle_styled
-                    kwargs:
-                    data_dir: ./data/pickle_dataframe/feature
-            action_interpreter: 
-                class: CategoricalActionInterpreter
-                module_path: qlib.rl.order_execution.interpreter
-                kwargs: 
-                values: 14
-                max_step: 8
-            network: 
-                class: Recurrent
-                module_path: qlib.rl.order_execution.network
-                kwargs: {}
-            policy: 
-                class: PPO
-                module_path: qlib.rl.order_execution.policy
-                kwargs: 
-                    lr: 1.0e-4
-                    # Local path to the latest model. The model is generated during training, so please run training first if you want to run backtest with a trained policy. You could also remove this parameter file to run backtest with a randomly initialized policy.
-                    weight_file: ./checkpoints/latest.pth
-    # Concurrent environment workers.
-    concurrency: 5
-
-With the above config files, you can start training the agent by the following command:
-
-.. code-block:: console
-
-    $ python -m qlib.rl.contrib.train_onpolicy.py --config_path train_config.yml
-
-After the training, you can backtest with the following command:
-
-.. code-block:: console
-
-    $ python -m qlib.rl.contrib.backtest.py --config_path backtest_config.yml
-
-In that case, :class:`~qlib.rl.order_execution.simulator_qlib.SingleAssetOrderExecution` and :class:`~qlib.rl.order_execution.simulator_simple.SingleAssetOrderExecutionSimple` as examples for simulator, :class:`qlib.rl.order_execution.interpreter.FullHistoryStateInterpreter` and :class:`qlib.rl.order_execution.interpreter.CategoricalActionInterpreter` as examples for interpreter, :class:`qlib.rl.order_execution.policy.PPO` as an example for policy, and :class:`qlib.rl.order_execution.reward.PAPenaltyReward` as an example for reward.
-For the single asset order execution task, if developers have already defined their simulator/interpreters/reward function/policy, they could launch the training and backtest pipeline by simply modifying the corresponding settings in the config files.
-The details about the example can be found `here <https://github.com/microsoft/qlib/blob/main/examples/rl/README.md>`_. 
-
-In the future, we will provide more examples for different scenarios such as RL-based portfolio construction.

docs/component/rl/toctree.rst

@@ -1,10 +0,0 @@
-.. _rl:
-
-========================================================================
-Reinforcement Learning in Quantitative Trading
-========================================================================
-
-.. toctree::
-    Overall <overall>
-    Quick Start <quickstart>
-    Framework <framework>

docs/component/strategy.rst

@@ -6,7 +6,7 @@ Portfolio Strategy: Portfolio Management
 .. currentmodule:: qlib
 
 Introduction
-============
+===================
 
 ``Portfolio Strategy`` is designed to adopt different portfolio strategies, which means that users can adopt different algorithms to generate investment portfolios based on the prediction scores of the ``Forecast Model``. Users can use the ``Portfolio Strategy`` in an automatic workflow by ``Workflow`` module, please refer to `Workflow: Workflow Management <workflow.html>`_.
 
@@ -20,7 +20,7 @@ Base Class & Interface
 ======================
 
 BaseStrategy
-------------
+------------------
 
 Qlib provides a base class ``qlib.strategy.base.BaseStrategy``. All strategy classes need to inherit the base class and implement its interface.
 
@@ -32,7 +32,7 @@ Qlib provides a base class ``qlib.strategy.base.BaseStrategy``. All strategy cla
 Users can inherit `BaseStrategy` to customize their strategy class.
 
 WeightStrategyBase
-------------------
+--------------------
 
 Qlib also provides a class ``qlib.contrib.strategy.WeightStrategyBase`` that is a subclass of `BaseStrategy`.
 
@@ -60,13 +60,13 @@ Implemented Strategy
 Qlib provides a implemented strategy classes named `TopkDropoutStrategy`.
 
 TopkDropoutStrategy
--------------------
+------------------
 `TopkDropoutStrategy` is a subclass of `BaseStrategy` and implement the interface `generate_order_list` whose process is as follows.
 
 - Adopt the ``Topk-Drop`` algorithm to calculate the target amount of each stock
 
     .. note::
-        There are two parameters for the ``Topk-Drop`` algorithm:
+        There are two parameters for the ``Topk-Drop`` algorithm：
 
         - `Topk`: The number of stocks held
         - `Drop`: The number of stocks sold on each trading day
@@ -80,7 +80,6 @@ TopkDropoutStrategy
         In most cases, ``TopkDrop`` algorithm sells and buys `Drop` stocks every trading day, which yields a turnover rate of 2$\times$`Drop`/$K$.
          
         The following images illustrate a typical scenario.
-
         .. image:: ../_static/img/topk_drop.png
             :alt: Topk-Drop
 
@@ -99,12 +98,12 @@ and `qlib.contrib.strategy.optimizer.enhanced_indexing.EnhancedIndexingOptimizer
 
 
 Usage & Example
-===============
+====================
 
 First, user can create a model to get trading signals(the variable name is ``pred_score`` in following cases).
 
 Prediction Score
-----------------
+-----------------
 
 The `prediction score` is a pandas DataFrame. Its index is <datetime(pd.Timestamp), instrument(str)> and it must
 contains a `score` column.
@@ -135,7 +134,7 @@ Qlib didn't add a step to scale the prediction score to a unified scale due to t
 - The model has the flexibility to define the target, loss, and data processing. So we don't think there is a silver bullet to rescale it back directly barely based on the model's outputs. If you want to scale it back to some meaningful values(e.g. stock returns.), an intuitive solution is to create a regression model for the model's recent outputs and your recent target values.
 
 Running backtest
-----------------
+-----------------
 
 - In most cases, users could backtest their portfolio management strategy  with ``backtest_daily``.
 
@@ -263,7 +262,7 @@ Running backtest
 
 
 Result
-------
+------------------
 
 The backtest results are in the following form:
 
@@ -308,5 +307,5 @@ The backtest results are in the following form:
 
 
 Reference
-=========
+===================
 To know more about the `prediction score` `pred_score` output by ``Forecast Model``, please refer to `Forecast Model: Model Training & Prediction <model.html>`_.

docs/component/workflow.rst

@@ -1,12 +1,12 @@
 .. _workflow:
 
-=============================
+=================================
 Workflow: Workflow Management
-=============================
+=================================
 .. currentmodule:: qlib
 
 Introduction
-============
+===================
 
 The components in `Qlib Framework <../introduction/introduction.html#framework>`_ are designed in a loosely-coupled way. Users could build their own Quant research workflow with these components like `Example <https://github.com/microsoft/qlib/blob/main/examples/workflow_by_code.py>`_.
 
@@ -28,7 +28,7 @@ With ``qrun``, user can easily start an `execution`, which includes the followin
 For each `execution`, ``Qlib`` has a complete system to tracking all the information as well as artifacts generated during training, inference and evaluation phase. For more information about how ``Qlib`` handles this, please refer to the related document: `Recorder: Experiment Management <../component/recorder.html>`_.
 
 Complete Example
-================
+===================
 
 Before getting into details, here is a complete example of ``qrun``, which defines the workflow in typical Quant research.
 Below is a typical config file of ``qrun``.
@@ -121,7 +121,7 @@ If users want to use ``qrun`` under debug mode, please use the following command
 
 
 Configuration File
-==================
+===================
 
 Let's get into details of ``qrun`` in this section.
 Before using ``qrun``, users need to prepare a configuration file. The following content shows how to prepare each part of the configuration file.
@@ -166,7 +166,7 @@ For example, the following yaml and code are equivalent.
 
 
 Qlib Init Section
------------------
+--------------------
 
 At first, the configuration file needs to contain several basic parameters which will be used for qlib initialization.
 
@@ -190,12 +190,12 @@ The meaning of each field is as follows:
 
 
 Task Section
-------------
+--------------------
 
 The `task` field in the configuration corresponds to a `task`, which contains the parameters of three different subsections: `Model`, `Dataset` and `Record`.
 
 Model Section
-~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~
 
 In the `task` field, the `model` section describes the parameters of the model to be used for training and inference. For more information about the base ``Model`` class, please refer to `Qlib Model <../component/model.html>`_.
 
@@ -231,7 +231,7 @@ The meaning of each field is as follows:
     ``Qlib`` provides a util named: ``init_instance_by_config`` to initialize any class inside ``Qlib`` with the configuration includes the fields: `class`, `module_path` and `kwargs`.
 
 Dataset Section
-~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~
 
 The `dataset` field describes the parameters for the ``Dataset`` module in ``Qlib`` as well those for the module ``DataHandler``. For more information about the ``Dataset`` module, please refer to `Qlib Data <../component/data.html#dataset>`_.
 
@@ -266,7 +266,7 @@ Here is the configuration for the ``Dataset`` module which will take care of dat
                 test: [2017-01-01, 2020-08-01]
 
 Record Section
-~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~
 
 The `record` field is about the parameters the ``Record`` module in ``Qlib``. ``Record`` is responsible for tracking training process and results such as `information Coefficient (IC)` and `backtest` in a standard format.
 

docs/conf.py

@@ -77,7 +77,7 @@ language = "en_US"
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "hidden"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = "sphinx"

docs/developer/code_standard.rst

@@ -1,22 +1,21 @@
 .. _code_standard:
 
-=============
+=================================
 Code Standard
-=============
+=================================
 
 Docstring
-=========
+=================================
 Please use the `Numpydoc Style <https://stackoverflow.com/a/24385103>`_.
 
 Continuous Integration
-======================
+=================================
 Continuous Integration (CI) tools help you stick to the quality standards by running tests every time you push a new commit and reporting the results to a pull request. 
 
 When you submit a PR request, you can check whether your code passes the CI tests in the "check" section at the bottom of the web page.
 
 1. Qlib will check the code format with black. The PR will raise error if your code does not align to the standard of Qlib(e.g. a common error is the mixed use of space and tab).
-
+ You can fix the bug by inputing the following code in the command line.
-   You can fix the bug by inputting the following code in the command line.
 
 .. code-block:: bash
 
@@ -33,8 +32,7 @@ When you submit a PR request, you can check whether your code passes the CI test
 
 
 3. Qlib will check your code style flake8. The checking command is implemented in [github action workflow](https://github.com/microsoft/qlib/blob/0e8b94a552f1c457cfa6cd2c1bb3b87ebb3fb279/.github/workflows/test.yml#L73).
-
+ You can fix the bug by inputing the following code in the command line.
-   You can fix the bug by inputing the following code in the command line.
 
 .. code-block:: bash
 
@@ -42,22 +40,9 @@ When you submit a PR request, you can check whether your code passes the CI test
 
 
 4. Qlib has integrated pre-commit, which will make it easier for developers to format their code.
-
+ Just run the following two commands, and the code will be automatically formatted using black and flake8 when the git commit command is executed.
-   Just run the following two commands, and the code will be automatically formatted using black and flake8 when the git commit command is executed.
 
 .. code-block:: bash
 
     pip install -e .[dev]
     pre-commit install
-
-
-=================================
-Development Guidance
-=================================
-
-As a developer, you often want make changes to `Qlib` and hope it would reflect directly in your environment without reinstalling it. You can install `Qlib` in editable mode with following command.
-The `[dev]` option will help you to install some related packages when developing `Qlib` (e.g. pytest, sphinx)
-
-.. code-block:: bash
-
-    pip install -e .[dev]

docs/hidden/client.rst

@@ -1,12 +1,12 @@
 .. _client:
 
 Qlib Client-Server Framework
-============================
+===================
 
 .. currentmodule:: qlib
 
 Introduction
-------------
+-----------
 Client-Server is designed to solve following  problems
 
 - Manage the data in a centralized way. Users don't have to manage data of different versions.
@@ -81,7 +81,6 @@ If running on Windows, open **NFS** features and write correct **mount_path**, i
     * Open ``Programs and Features``.
     * Click ``Turn Windows features on or off``.
     * Scroll down and check the option ``Services for NFS``, then click OK
-
     Reference address: https://graspingtech.com/mount-nfs-share-windows-10/
 2.config correct mount_path
     * In windows, mount path must be not exist path and root path,
@@ -160,11 +159,13 @@ Limitations
 2. The rolling operation expression with parameter `0` can not be updated rightly under mechanism of the client-server framework.
 
 API
-***
+********************
 
-The client is based on `python-socketio <https://python-socketio.readthedocs.io>`_ which is a framework that supports WebSocket client for Python language. The client can only propose requests and receive results, which do not include any calculating procedure.
+The client is based on `python-socketio<https://python-socketio.readthedocs.io>`_ which is a framework that supports WebSocket client for Python language. The client can only propose requests and receive results, which do not include any calculating procedure.
 
 Class
------
+--------------------
 
 .. automodule:: qlib.data.client
+
+

docs/hidden/online.rst

@@ -1,11 +1,11 @@
 .. _online:
 
 Online
-======
+===================
 .. currentmodule:: qlib
 
 Introduction
-------------
+-------------------
 
 Welcome to use Online, this module simulates what will be like if we do the real trading use our model and strategy.
 
@@ -31,7 +31,7 @@ The file structure can be viewed at fileStruct_.
 
 
 Example
--------
+-------------------
 
 Let's take an example,
 
@@ -93,7 +93,7 @@ If Your account was saved in "./user_data/", you can see the performance of your
 Here 'SH000905' represents csi500 and 'SH000300' represents csi300
 
 Manage your account
--------------------
+--------------------
 
 Any account processed by `online` should be saved in a folder. you can use commands
 defined to manage your accounts.
@@ -161,7 +161,7 @@ be called at each trading date.
         >> online update -date 2019-10-16 -path ./user_data/
 
 API
----
+------------------
 
 All those operations are based on defined in `qlib.contrib.online.operator`
 
@@ -170,7 +170,7 @@ All those operations are based on defined in `qlib.contrib.online.operator`
 .. _fileStruct:
 
 File structure
---------------
+------------------
 
 'user_data' indicates the root of folder.
 Name that bold indicates it’s a folder, otherwise it’s a document.
@@ -214,7 +214,7 @@ Configuration file
 The configure file used in `online` should contain the model and strategy information.
 
 About the model
-~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~
 
 First, your configuration file needs to have a field about the model,
 this field and its contents determine the model we used when generating score at predict date.
@@ -243,7 +243,7 @@ contains 2 methods used in `online` module.
 
 
 About the strategy
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~
 
 Your need define the strategy used to generate the order list at predict date.
 
@@ -259,7 +259,7 @@ Followings are two examples for a TopkAmountStrategy
             n_drop: 10
 
 Generated files
----------------
+------------------
 
 The 'online_generate' command will create the order list at {folder_path}/{user_id}/temp/,
 the name of that is orderlist_{YYYY-MM-DD}.json, YYYY-MM-DD is the date that those orders to be executed.

docs/hidden/tuner.rst

@@ -1,11 +1,11 @@
 .. _tuner:
 
 Tuner
-=====
+===================
 .. currentmodule:: qlib
 
 Introduction
-------------
+-------------------
 
 Welcome to use Tuner, this document is based on that you can use Estimator proficiently and correctly.
 
@@ -322,3 +322,4 @@ What we save are as following:
 - Local optimal parameters of each tuner
 - Config file of this `tuner` experiment
 - Every `estimator` experiments result in the process
+

docs/index.rst

@@ -1,6 +1,6 @@
-======================
+============================================================
 ``Qlib`` Documentation
-======================
+============================================================
 
 ``Qlib`` is an AI-oriented quantitative investment platform, which aims to realize the potential, empower the research, and create the value of AI technologies in quantitative investment.
 
@@ -33,7 +33,7 @@ Document Structure
 
 .. toctree::
    :maxdepth: 3
-   :caption: MAIN COMPONENTS:
+   :caption: COMPONENTS:
 
    Workflow: Workflow Management <component/workflow.rst>
    Data Layer: Data Framework & Usage <component/data.rst>
@@ -44,11 +44,10 @@ Document Structure
    Qlib Recorder: Experiment Management <component/recorder.rst>
    Analysis: Evaluation & Results Analysis <component/report.rst>
    Online Serving: Online Management & Strategy & Tool <component/online.rst>
-   Reinforcement Learning <component/rl/toctree>
 
 .. toctree::
    :maxdepth: 3
-   :caption: OTHER COMPONENTS/FEATURES/TOPICS:
+   :caption: ADVANCED TOPICS:
    
    Building Formulaic Alphas <advanced/alpha.rst>
    Online & Offline mode <advanced/server.rst>
@@ -56,12 +55,6 @@ Document Structure
    Task Management <advanced/task_management.rst>
    Point-In-Time database <advanced/PIT.rst>
 
-.. toctree::
-   :maxdepth: 3
-   :caption: FOR DEVELOPERS:
-
-   Code Standard & Development Guidance <developer/code_standard_and_dev_guide.rst>
-
 .. toctree::
    :maxdepth: 3
    :caption: REFERENCE:

docs/introduction/introduction.rst

@@ -3,7 +3,7 @@
 ===============================
 
 Introduction
-============
+===================
 
 .. image:: ../_static/img/logo/white_bg_rec+word.png
     :align: center
@@ -13,8 +13,7 @@ Introduction
 With ``Qlib``, users can easily try their ideas to create better Quant investment strategies.
 
 Framework
-=========
+===================
-
    
 .. image:: ../_static/img/framework.svg
     :align: center
@@ -22,49 +21,32 @@ Framework
 
 At the module level, Qlib is a platform that consists of above components. The components are designed as loose-coupled modules and each component could be used stand-alone.
 
-This framework may be intimidating for new users to Qlib. It tries to accurately include a lot of details of Qlib's design.
-For users new to Qlib, you can skip it first and read it later.
 
 
+========================  ==============================================================================
+Name                      Description
+========================  ==============================================================================
+`Infrastructure` layer    `Infrastructure` layer provides underlying support for Quant research.
+                          `DataServer` provides high-performance infrastructure for users to manage 
+                          and retrieve raw data. `Trainer` provides flexible interface to control
+                          the training process of models which enable algorithms controlling the
+                          training process.
 
-===========================  ==============================================================================
+`Workflow` layer          `Workflow` layer covers the whole workflow of quantitative investment.
-Name                         Description
+                          `Information Extractor` extracts data for models. `Forecast Model` focuses
-===========================  ==============================================================================
+                          on producing all kinds of forecast signals (e.g. *alpha*, risk) for other
-`Infrastructure` layer       `Infrastructure` layer provides underlying support for Quant research.
+                          modules.  With these signals `Decision Generator` will generate the target 
-                             `DataServer` provides high-performance infrastructure for users to manage
+                          trading decisions(i.e. portfolio, orders)  to be executed by `Execution Env`
-                             and retrieve raw data. `Trainer` provides flexible interface to control
+                          (i.e. the trading market).  There may be multiple levels of `Trading Agent`
-                             the training process of models which enable algorithms controlling the
+                          and `Execution Env` (e.g. an *order executor trading agent and intraday
-                             training process.
+                          order execution environment* could behave like an interday trading
+                          environment and nested in  *daily portfolio management trading agent and
+                          interday trading environment*  ) 
 
-`Learning Framework` layer   The `Forecast Model` and `Trading Agent` are learnable. They are learned
+`Interface` layer         `Interface` layer tries to present a user-friendly interface for the underlying
-                             based on the `Learning Framework` layer and then applied to multiple scenarios
+                          system. `Analyser` module will provide users detailed analysis reports of
-                             in `Workflow` layer. The supported learning paradigms can be categorized into
+                          forecasting signals, portfolios and execution results
-                             reinforcement learning and supervised learning.  The learning framework
+========================  ==============================================================================
-                             leverages the `Workflow` layer as well(e.g. sharing `Information Extractor`,
-                             creating environments based on `Execution Env`).
-
-`Workflow` layer             `Workflow` layer covers the whole workflow of quantitative investment.
-                             Both supervised-learning-based strategies and RL-based Strategies
-                             are supported.
-                             `Information Extractor` extracts data for models. `Forecast Model` focuses
-                             on producing all kinds of forecast signals (e.g. *alpha*, risk) for other
-                             modules.  With these signals `Decision Generator` will generate the target
-                             trading decisions(i.e. portfolio, orders)
-                             If RL-based Strategies are adopted, the `Policy` is learned in a end-to-end way,
-                             the trading deicsions are generated directly.
-                             Decisions will be executed by `Execution Env`
-                             (i.e. the trading market).  There may be multiple levels of `Strategy`
-                             and `Executor` (e.g. an *order executor trading strategy and intraday order executor*
-                             could behave like an interday trading loop and be nested in
-                             *daily portfolio management trading strategy and interday trading executor*
-                             trading loop)
-
-`Interface` layer            `Interface` layer tries to present a user-friendly interface for the underlying
-                             system. `Analyser` module will provide users detailed analysis reports of
-                             forecasting signals, portfolios and execution results
-===========================  ==============================================================================
 
 - The modules with hand-drawn style are under development and will be released in the future.
 - The modules with dashed borders are highly user-customizable and extendible.
-
-(p.s. framework image is created with https://draw.io/)

docs/introduction/quick.rst

@@ -1,10 +1,10 @@
 
-===========
+===============================
 Quick Start
-===========
+===============================
 
 Introduction
-============
+==============
 
 This ``Quick Start`` guide tries to demonstrate
 
@@ -14,14 +14,13 @@ This ``Quick Start`` guide tries to demonstrate
 
 
 Installation
-============
+==================
 
 Users can easily intsall ``Qlib`` according to the following steps:
 
 - Before installing ``Qlib`` from source, users need to install some dependencies:
 
     .. code-block::
-
         pip install numpy
         pip install --upgrade  cython
 
@@ -35,7 +34,7 @@ Users can easily intsall ``Qlib`` according to the following steps:
 To known more about `installation`, please refer to `Qlib Installation <../start/installation.html>`_.
 
 Prepare Data
-============
+==============
 
 Load and prepare data by running the following code:
 
@@ -48,7 +47,7 @@ This dataset is created by public data collected by crawler scripts in ``scripts
 To known more about `prepare data`, please refer to `Data Preparation <../component/data.html#data-preparation>`_.
 
 Auto Quant Research Workflow
-============================
+====================================
 
 ``Qlib`` provides a tool named ``qrun`` to run the whole workflow automatically (including building dataset, training models, backtest and evaluation). Users can start an auto quant research workflow and have a graphical reports analysis according to the following steps: 
 
@@ -90,6 +89,6 @@ Auto Quant Research Workflow
 
 
 Custom Model Integration
-========================
+===============================================
 
 ``Qlib`` provides a batch of models (such as ``lightGBM`` and ``MLP`` models) as examples of ``Forecast Model``. In addition to the default model, users can integrate their own custom models into ``Qlib``. If users are interested in the custom model, please refer to `Custom Model Integration <../start/integration.html>`_.

docs/make.bat

@@ -1,35 +0,0 @@
-@ECHO OFF
-
-pushd %~dp0
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=sphinx-build
-)
-set SOURCEDIR=.
-set BUILDDIR=_build
-
-%SPHINXBUILD% >NUL 2>NUL
-if errorlevel 9009 (
-	echo.
-	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
-	echo.installed, then set the SPHINXBUILD environment variable to point
-	echo.to the full path of the 'sphinx-build' executable. Alternatively you
-	echo.may add the Sphinx directory to PATH.
-	echo.
-	echo.If you don't have Sphinx installed, grab it from
-	echo.https://www.sphinx-doc.org/
-	exit /b 1
-)
-
-if "%1" == "" goto help
-
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-goto end
-
-:help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-
-:end
-popd

docs/reference/api.rst

@@ -1,8 +1,7 @@
 .. _api:
-
+================================
-=============
 API Reference
-=============
+================================
 
 
 
@@ -10,32 +9,32 @@ Here you can find all ``Qlib`` interfaces.
 
 
 Data
-====
+====================
 
 Provider
---------
+--------------------
 
 .. automodule:: qlib.data.data
     :members:
 		
 Filter
-------
+--------------------
 
 .. automodule:: qlib.data.filter
     :members:
 
 Class
------
+--------------------
 .. automodule:: qlib.data.base
     :members:
 
 Operator
---------
+--------------------
 .. automodule:: qlib.data.ops
     :members:
 	       
 Cache
------
+----------------
 .. autoclass:: qlib.data.cache.MemCacheUnit
     :members:
 
@@ -56,7 +55,7 @@ Cache
 
 
 Storage
--------
+-------------
 .. autoclass:: qlib.data.storage.storage.BaseStorage
     :members:
 
@@ -83,52 +82,52 @@ Storage
 
 
 Dataset
--------
+---------------
 
 Dataset Class
-~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~
 .. automodule:: qlib.data.dataset.__init__
     :members:
 
 Data Loader
-~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~
 .. automodule:: qlib.data.dataset.loader
     :members:
 
 Data Handler
-~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~
 .. automodule:: qlib.data.dataset.handler
     :members:
 
 Processor
-~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~
 .. automodule:: qlib.data.dataset.processor
     :members:
 
 
 Contrib
-=======
+====================
 
 Model
------
+--------------------
 .. automodule:: qlib.model.base
     :members:
 
 Strategy
---------
+-------------------
 
-.. automodule:: qlib.contrib.strategy
+.. automodule:: qlib.contrib.strategy.strategy
     :members:
 
 Evaluate
---------
+-----------------
 
 .. automodule:: qlib.contrib.evaluate
     :members:
     
 
 Report
-------
+-----------------
 
 .. automodule:: qlib.contrib.report.analysis_position.report
     :members:
@@ -160,133 +159,103 @@ Report
 
 
 Workflow
-========
+====================
 
 
 Experiment Manager
-------------------
+--------------------
 .. autoclass:: qlib.workflow.expm.ExpManager
     :members:
 
 Experiment
-----------
+--------------------
 .. autoclass:: qlib.workflow.exp.Experiment
     :members:
 
 Recorder
---------
+--------------------
 .. autoclass:: qlib.workflow.recorder.Recorder
     :members:
 
 Record Template
----------------
+--------------------
 .. automodule:: qlib.workflow.record_temp
     :members:
 
 Task Management
-===============
+====================
 
 
 TaskGen
--------
+--------------------
 .. automodule:: qlib.workflow.task.gen
     :members:
 
 TaskManager
------------
+--------------------
 .. automodule:: qlib.workflow.task.manage
     :members:
 
 Trainer
--------
+--------------------
 .. automodule:: qlib.model.trainer
     :members:
 
 Collector
----------
+--------------------
 .. automodule:: qlib.workflow.task.collect
     :members:
 
 Group
------
+--------------------
 .. automodule:: qlib.model.ens.group
     :members:
 
 Ensemble
---------
+--------------------
 .. automodule:: qlib.model.ens.ensemble
     :members:
 
 Utils
------
+--------------------
 .. automodule:: qlib.workflow.task.utils
     :members:
 
 
 Online Serving
-==============
+====================
 
 
 Online Manager
---------------
+--------------------
 .. automodule:: qlib.workflow.online.manager
     :members:
 
 Online Strategy
----------------
+--------------------
 .. automodule:: qlib.workflow.online.strategy
     :members:
 
 Online Tool
------------
+--------------------
 .. automodule:: qlib.workflow.online.utils
     :members:
 
 
 RecordUpdater
--------------
+--------------------
 .. automodule:: qlib.workflow.online.update
     :members:
 
 
 Utils
-=====
+====================
 
 Serializable
-------------
+--------------------
 
-.. automodule:: qlib.utils.serial
+.. automodule:: qlib.utils.serial.Serializable
     :members:
 
-RL
-==============
 
-Base Component
---------------
-.. automodule:: qlib.rl
-    :members:
-    :imported-members:
     
-Strategy
---------
-.. automodule:: qlib.rl.strategy
-    :members:
-    :imported-members:
-
-Trainer
--------
-.. automodule:: qlib.rl.trainer
-    :members:
-    :imported-members:
-
-Order Execution
----------------
-.. automodule:: qlib.rl.order_execution
-    :members:
-    :imported-members:
-
-Utils
----------------
-.. automodule:: qlib.rl.utils
-    :members:
-    :imported-members:

docs/requirements.txt

@@ -4,4 +4,3 @@ numpy
 scipy
 scikit-learn
 pandas
-tianshou

docs/start/getdata.rst

@@ -1,18 +1,18 @@
 .. _getdata:
 
-==============
+=============================
 Data Retrieval
-==============
+=============================
 
 .. currentmodule:: qlib
 
 Introduction
-============
+====================
 
 Users can get stock data with ``Qlib``. The following examples demonstrate the basic user interface.
 
 Examples
-========
+====================
 
 
 ``QLib`` Initialization:
@@ -83,14 +83,15 @@ Load features of certain instruments in a given time range:
    >> from qlib.data import D
    >> instruments = ['SH600000']
    >> fields = ['$close', '$volume', 'Ref($close, 1)', 'Mean($close, 3)', '$high-$low']
-   >> D.features(instruments, fields, start_time='2010-01-01', end_time='2017-12-31', freq='day').head().to_string()
+   >> D.features(instruments, fields, start_time='2010-01-01', end_time='2017-12-31', freq='day').head()
-   '                           $close     $volume  Ref($close, 1)  Mean($close, 3)  $high-$low
+                              
-   ... instrument  datetime
+                              $close     $volume  Ref($close, 1)  Mean($close, 3)  $high-$low
-   ... SH600000    2010-01-04  86.778313  16162960.0       88.825928        88.061483    2.907631
+      instrument  datetime                                                                      
-   ...             2010-01-05  87.433578  28117442.0       86.778313        87.679273    3.235252
+      SH600000    2010-01-04  86.778313  16162960.0       88.825928        88.061483    2.907631
-   ...             2010-01-06  85.713585  23632884.0       87.433578        86.641825    1.720009
+                  2010-01-05  87.433578  28117442.0       86.778313        87.679273    3.235252
-   ...             2010-01-07  83.788803  20813402.0       85.713585        85.645322    3.030487
+                  2010-01-06  85.713585  23632884.0       87.433578        86.641825    1.720009
-   ...             2010-01-08  84.730675  16044853.0       83.788803        84.744354    2.047623'
+                  2010-01-07  83.788803  20813402.0       85.713585        85.645322    3.030487
+                  2010-01-08  84.730675  16044853.0       83.788803        84.744354    2.047623
 
 Load features of certain stock pool in a given time range:
 
@@ -104,14 +105,15 @@ Load features of certain stock pool in a given time range:
    >> expressionDFilter = ExpressionDFilter(rule_expression='$close>Ref($close,1)')
    >> instruments = D.instruments(market='csi300', filter_pipe=[nameDFilter, expressionDFilter])
    >> fields = ['$close', '$volume', 'Ref($close, 1)', 'Mean($close, 3)', '$high-$low']
-   >> D.features(instruments, fields, start_time='2010-01-01', end_time='2017-12-31', freq='day').head().to_string()
+   >> D.features(instruments, fields, start_time='2010-01-01', end_time='2017-12-31', freq='day').head()
-   '                              $close        $volume  Ref($close, 1)  Mean($close, 3)  $high-$low
+
-   ... instrument  datetime
+                                 $close        $volume  Ref($close, 1)  Mean($close, 3)  $high-$low
-   ... SH600655    2010-01-04  2699.567383  158193.328125     2619.070312      2626.097738  124.580566
+      instrument  datetime                                                                           
-   ...             2010-01-08  2612.359619   77501.406250     2584.567627      2623.220133   83.373047
+      SH600655    2010-01-04  2699.567383  158193.328125     2619.070312      2626.097738  124.580566
-   ...             2010-01-11  2712.982422  160852.390625     2612.359619      2636.636556  146.621582
+                  2010-01-08  2612.359619   77501.406250     2584.567627      2623.220133   83.373047
-   ...             2010-01-12  2788.688232  164587.937500     2712.982422      2704.676758  128.413818
+                  2010-01-11  2712.982422  160852.390625     2612.359619      2636.636556  146.621582
-   ...             2010-01-13  2790.604004  145460.453125     2788.688232      2764.091553  128.413818'
+                  2010-01-12  2788.688232  164587.937500     2712.982422      2704.676758  128.413818
+                  2010-01-13  2790.604004  145460.453125     2788.688232      2764.091553  128.413818
 
 
 For more details about features, please refer `Feature API <../component/data.html>`_.
@@ -125,7 +127,7 @@ For example, it looks quite long and complicated:
 .. code-block:: python
 
    >> from qlib.data import D
-   >> data = D.features(["sh600519"], ["(($high / $close) + ($open / $close)) * (($high / $close) + ($open / $close)) / (($high / $close) + ($open / $close))"], start_time="20200101")
+   >> data = D.features(["sh600519"], ["(($high / $close) + ($open / $close)) * (($high / $close) + ($open / $close)) / ($high / $close) + ($open / $close)"], start_time="20200101")
 
 
 But using string is not the only way to implement the expression. You can also implement expression by code.
@@ -145,5 +147,5 @@ Here is an exmaple which does the same thing as above examples.
 
 
 API
-===
+====================
 To know more about how to use the Data, go to API Reference: `Data API <../reference/api.html#data>`_

docs/start/initialization.rst

@@ -1,14 +1,14 @@
 .. _initialization:
 
-===================
+====================
 Qlib Initialization
-===================
+====================
 
 .. currentmodule:: qlib
 
 
 Initialization
-==============
+=========================
 
 Please follow the steps below to initialize ``Qlib``.
 

docs/start/installation.rst

@@ -1,8 +1,8 @@
 .. _installation:
 
-============
+====================
 Installation
-============
+====================
 
 .. currentmodule:: qlib
 
@@ -44,3 +44,6 @@ Use the following code to make sure the installation successful:
    >>> import qlib
    >>> qlib.__version__
    <LATEST VERSION>
+
+
+=====================

docs/start/integration.rst

@@ -1,9 +1,9 @@
-========================
+=========================================
 Custom Model Integration
-========================
+=========================================
 
 Introduction
-============
+===================
 
 ``Qlib``'s `Model Zoo` includes models such as ``LightGBM``, ``MLP``, ``LSTM``, etc.. These models are examples of ``Forecast Model``. In addition to the default models ``Qlib`` provide, users can integrate their own custom models into ``Qlib``.
 
@@ -14,123 +14,119 @@ Users can integrate their own custom models according to the following steps.
 - Test the custom model.
 
 Custom Model Class
-==================
+===========================
 The Custom models need to inherit `qlib.model.base.Model <../reference/api.html#module-qlib.model.base>`_ and override the methods in it.
 
 - Override the `__init__` method
     - ``Qlib`` passes the initialized parameters to the \_\_init\_\_ method.
     - The hyperparameters of model in the configuration must be consistent with those defined in the `__init__` method.
     - Code Example: In the following example, the hyperparameters of model in the configuration file should contain parameters such as `loss:mse`.
+    .. code-block:: Python
 
-        .. code-block:: Python
+        def __init__(self, loss='mse', **kwargs):
-
+            if loss not in {'mse', 'binary'}:
-            def __init__(self, loss='mse', **kwargs):
+                raise NotImplementedError
-                if loss not in {'mse', 'binary'}:
+            self._scorer = mean_squared_error if loss == 'mse' else roc_auc_score
-                    raise NotImplementedError
+            self._params.update(objective=loss, **kwargs)
-                self._scorer = mean_squared_error if loss == 'mse' else roc_auc_score
+            self._model = None
-                self._params.update(objective=loss, **kwargs)
-                self._model = None
 
 - Override the `fit` method
     - ``Qlib`` calls the fit method to train the model.
     - The parameters must include training feature `dataset`, which is designed in the interface.
     - The parameters could include some `optional` parameters with default values, such as `num_boost_round = 1000` for `GBDT`.
     - Code Example: In the following example, `num_boost_round = 1000` is an optional parameter.
+    .. code-block:: Python
     
-        .. code-block:: Python
+        def fit(self, dataset: DatasetH, num_boost_round = 1000, **kwargs):
 
-            def fit(self, dataset: DatasetH, num_boost_round = 1000, **kwargs):
+            # prepare dataset for lgb training and evaluation
+            df_train, df_valid = dataset.prepare(
+                ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
+            )
+            x_train, y_train = df_train["feature"], df_train["label"]
+            x_valid, y_valid = df_valid["feature"], df_valid["label"]
 
-                # prepare dataset for lgb training and evaluation
+            # Lightgbm need 1D array as its label
-                df_train, df_valid = dataset.prepare(
+            if y_train.values.ndim == 2 and y_train.values.shape[1] == 1:
-                    ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
+                y_train, y_valid = np.squeeze(y_train.values), np.squeeze(y_valid.values)
-                )
+            else:
-                x_train, y_train = df_train["feature"], df_train["label"]
+                raise ValueError("LightGBM doesn't support multi-label training")
-                x_valid, y_valid = df_valid["feature"], df_valid["label"]
 
-                # Lightgbm need 1D array as its label
+            dtrain = lgb.Dataset(x_train.values, label=y_train)
-                if y_train.values.ndim == 2 and y_train.values.shape[1] == 1:
+            dvalid = lgb.Dataset(x_valid.values, label=y_valid)
-                    y_train, y_valid = np.squeeze(y_train.values), np.squeeze(y_valid.values)
-                else:
-                    raise ValueError("LightGBM doesn't support multi-label training")
 
-                dtrain = lgb.Dataset(x_train.values, label=y_train)
+            # fit the model
-                dvalid = lgb.Dataset(x_valid.values, label=y_valid)
+            self.model = lgb.train(
-
+                self.params,
-                # fit the model
+                dtrain,
-                self.model = lgb.train(
+                num_boost_round=num_boost_round,
-                    self.params,
+                valid_sets=[dtrain, dvalid],
-                    dtrain,
+                valid_names=["train", "valid"],
-                    num_boost_round=num_boost_round,
+                early_stopping_rounds=early_stopping_rounds,
-                    valid_sets=[dtrain, dvalid],
+                verbose_eval=verbose_eval,
-                    valid_names=["train", "valid"],
+                evals_result=evals_result,
-                    early_stopping_rounds=early_stopping_rounds,
+                **kwargs
-                    verbose_eval=verbose_eval,
+            )
-                    evals_result=evals_result,
-                    **kwargs
-                )
 
 - Override the `predict` method
     - The parameters must include the parameter `dataset`, which will be userd to get the test dataset.
     - Return the `prediction score`.
     - Please refer to `Model API <../reference/api.html#module-qlib.model.base>`_ for the parameter types of the fit method.
     - Code Example: In the following example, users need to use `LightGBM` to predict the label(such as `preds`) of test data `x_test` and return it.
+    .. code-block:: Python
 
-        .. code-block:: Python
+        def predict(self, dataset: DatasetH, **kwargs)-> pandas.Series:
-
+            if self.model is None:
-            def predict(self, dataset: DatasetH, **kwargs)-> pandas.Series:
+                raise ValueError("model is not fitted yet!")
-                if self.model is None:
+            x_test = dataset.prepare("test", col_set="feature", data_key=DataHandlerLP.DK_I)
-                    raise ValueError("model is not fitted yet!")
+            return pd.Series(self.model.predict(x_test.values), index=x_test.index)
-                x_test = dataset.prepare("test", col_set="feature", data_key=DataHandlerLP.DK_I)
-                return pd.Series(self.model.predict(x_test.values), index=x_test.index)
 
 - Override the `finetune` method (Optional)
     - This method is optional to the users. When users want to use this method on their own models, they should inherit the ``ModelFT`` base class, which includes the interface of `finetune`.
     - The parameters must include the parameter `dataset`.
     - Code Example: In the following example, users will use `LightGBM` as the model and finetune it.
+    .. code-block:: Python
 
-        .. code-block:: Python
+        def finetune(self, dataset: DatasetH, num_boost_round=10, verbose_eval=20):
-
+            # Based on existing model and finetune by train more rounds
-            def finetune(self, dataset: DatasetH, num_boost_round=10, verbose_eval=20):
+            dtrain, _ = self._prepare_data(dataset)
-                # Based on existing model and finetune by train more rounds
+            self.model = lgb.train(
-                dtrain, _ = self._prepare_data(dataset)
+                self.params,
-                self.model = lgb.train(
+                dtrain,
-                    self.params,
+                num_boost_round=num_boost_round,
-                    dtrain,
+                init_model=self.model,
-                    num_boost_round=num_boost_round,
+                valid_sets=[dtrain],
-                    init_model=self.model,
+                valid_names=["train"],
-                    valid_sets=[dtrain],
+                verbose_eval=verbose_eval,
-                    valid_names=["train"],
+            )
-                    verbose_eval=verbose_eval,
-                )
 
 Configuration File
-==================
+=======================
 
 The configuration file is described in detail in the `Workflow <../component/workflow.html#complete-example>`_ document. In order to integrate the custom model into ``Qlib``, users need to modify the "model" field in the configuration file. The configuration describes which models to use and how we can initialize it.
 
 - Example: The following example describes the `model` field of configuration file about the custom lightgbm model mentioned above, where `module_path` is the module path, `class` is the class name, and `args` is the hyperparameter passed into the __init__ method. All parameters in the field is passed to `self._params` by `\*\*kwargs` in `__init__` except `loss = mse`. 
 
-    .. code-block:: YAML
+.. code-block:: YAML
     
-        model:
+    model:
-            class: LGBModel
+        class: LGBModel
-            module_path: qlib.contrib.model.gbdt
+        module_path: qlib.contrib.model.gbdt
-            args:
+        args:
-                loss: mse
+            loss: mse
-                colsample_bytree: 0.8879
+            colsample_bytree: 0.8879
-                learning_rate: 0.0421
+            learning_rate: 0.0421
-                subsample: 0.8789
+            subsample: 0.8789
-                lambda_l1: 205.6999
+            lambda_l1: 205.6999
-                lambda_l2: 580.9768
+            lambda_l2: 580.9768
-                max_depth: 8
+            max_depth: 8
-                num_leaves: 210
+            num_leaves: 210
-                num_threads: 20
+            num_threads: 20
 
 Users could find configuration file of the baselines of the ``Model`` in ``examples/benchmarks``. All the configurations of different models are listed under the corresponding model folder.
 
 Model Testing
-=============
+=====================
 Assuming that the configuration file is ``examples/benchmarks/LightGBM/workflow_config_lightgbm.yaml``, users can run the following command to test the custom model:
 
 .. code-block:: bash
@@ -144,6 +140,6 @@ Also, ``Model`` can also be tested as a single module. An example has been given
 
 
 Reference
-=========
+=====================
 
 To know more about ``Forecast Model``, please refer to `Forecast Model: Model Training & Prediction <../component/model.html>`_ and `Model API <../reference/api.html#module-qlib.model.base>`_.

examples/benchmarks/ALSTM/README.md

@@ -6,4 +6,3 @@
 
   [https://www.ijcai.org/Proceedings/2017/0366.pdf](https://www.ijcai.org/Proceedings/2017/0366.pdf)
 
-- NOTE: Current version of implementation is just a simplified version of ALSTM. It is an LSTM with attention.

examples/benchmarks/CatBoost/workflow_config_catboost_Alpha158_csi500.yaml

@@ -1,72 +0,0 @@
-qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
-    region: cn
-market: &market csi500
-benchmark: &benchmark SH000905
-data_handler_config: &data_handler_config
-    start_time: 2008-01-01
-    end_time: 2020-08-01
-    fit_start_time: 2008-01-01
-    fit_end_time: 2014-12-31
-    instruments: *market
-port_analysis_config: &port_analysis_config
-    strategy:
-        class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy
-        kwargs:
-            signal:
-                - <MODEL> 
-                - <DATASET>
-            topk: 50
-            n_drop: 5
-    backtest:
-        start_time: 2017-01-01
-        end_time: 2020-08-01
-        account: 100000000
-        benchmark: *benchmark
-        exchange_kwargs:
-            limit_threshold: 0.095
-            deal_price: close
-            open_cost: 0.0005
-            close_cost: 0.0015
-            min_cost: 5
-task:
-    model:
-        class: CatBoostModel
-        module_path: qlib.contrib.model.catboost_model
-        kwargs:
-            loss: RMSE
-            learning_rate: 0.0421
-            subsample: 0.8789
-            max_depth: 6
-            num_leaves: 100
-            thread_count: 20
-            grow_policy: Lossguide
-            bootstrap_type: Poisson
-    dataset:
-        class: DatasetH
-        module_path: qlib.data.dataset
-        kwargs:
-            handler:
-                class: Alpha158
-                module_path: qlib.contrib.data.handler
-                kwargs: *data_handler_config
-            segments:
-                train: [2008-01-01, 2014-12-31]
-                valid: [2015-01-01, 2016-12-31]
-                test: [2017-01-01, 2020-08-01]
-    record: 
-        - class: SignalRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            model: <MODEL>
-            dataset: <DATASET>
-        - class: SigAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            ana_long_short: False
-            ann_scaler: 252
-        - class: PortAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            config: *port_analysis_config

examples/benchmarks/CatBoost/workflow_config_catboost_Alpha360_csi500.yaml

@@ -1,79 +0,0 @@
-qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
-    region: cn
-market: &market csi500
-benchmark: &benchmark SH000905
-data_handler_config: &data_handler_config
-    start_time: 2008-01-01
-    end_time: 2020-08-01
-    fit_start_time: 2008-01-01
-    fit_end_time: 2014-12-31
-    instruments: *market
-    infer_processors: []
-    learn_processors:
-        - class: DropnaLabel
-        - class: CSRankNorm
-          kwargs:
-              fields_group: label
-    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
-port_analysis_config: &port_analysis_config
-    strategy:
-        class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy
-        kwargs:
-            signal:
-                - <MODEL> 
-                - <DATASET>
-            topk: 50
-            n_drop: 5
-    backtest:
-        start_time: 2017-01-01
-        end_time: 2020-08-01
-        account: 100000000
-        benchmark: *benchmark
-        exchange_kwargs:
-            limit_threshold: 0.095
-            deal_price: close
-            open_cost: 0.0005
-            close_cost: 0.0015
-            min_cost: 5
-task:
-    model:
-        class: CatBoostModel
-        module_path: qlib.contrib.model.catboost_model
-        kwargs:
-            loss: RMSE
-            learning_rate: 0.0421
-            subsample: 0.8789
-            max_depth: 6
-            num_leaves: 100
-            thread_count: 20
-            grow_policy: Lossguide
-            bootstrap_type: Poisson
-    dataset:
-        class: DatasetH
-        module_path: qlib.data.dataset
-        kwargs:
-            handler:
-                class: Alpha360
-                module_path: qlib.contrib.data.handler
-                kwargs: *data_handler_config
-            segments:
-                train: [2008-01-01, 2014-12-31]
-                valid: [2015-01-01, 2016-12-31]
-                test: [2017-01-01, 2020-08-01]
-    record: 
-        - class: SignalRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            model: <MODEL>
-            dataset: <DATASET>
-        - class: SigAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            ana_long_short: False
-            ann_scaler: 252
-        - class: PortAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            config: *port_analysis_config

examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha158.yaml

@@ -37,7 +37,7 @@ task:
         kwargs:
             base_model: "gbm"
             loss: mse
-            num_models: 3
+            num_models: 6
             enable_sr: True
             enable_fs: True
             alpha1: 1
@@ -53,8 +53,11 @@ task:
                 - 0.4
             sub_weights:
                 - 1
-                - 1
+                - 0.2
-                - 1
+                - 0.2
+                - 0.2
+                - 0.2
+                - 0.2
             epochs: 28
             colsample_bytree: 0.8879
             learning_rate: 0.2

examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha158_csi500.yaml

@@ -1,97 +0,0 @@
-qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
-    region: cn
-market: &market csi500
-benchmark: &benchmark SH000905
-data_handler_config: &data_handler_config
-    start_time: 2008-01-01
-    end_time: 2020-08-01
-    fit_start_time: 2008-01-01
-    fit_end_time: 2014-12-31
-    instruments: *market
-port_analysis_config: &port_analysis_config
-    strategy:
-        class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy
-        kwargs:
-            signal:
-                - <MODEL> 
-                - <DATASET>
-            topk: 50
-            n_drop: 5
-    backtest:
-        start_time: 2017-01-01
-        end_time: 2020-08-01
-        account: 100000000
-        benchmark: *benchmark
-        exchange_kwargs:
-            limit_threshold: 0.095
-            deal_price: close
-            open_cost: 0.0005
-            close_cost: 0.0015
-            min_cost: 5
-task:
-    model:
-        class: DEnsembleModel
-        module_path: qlib.contrib.model.double_ensemble
-        kwargs:
-            base_model: "gbm"
-            loss: mse
-            num_models: 6
-            enable_sr: True
-            enable_fs: True
-            alpha1: 1
-            alpha2: 1
-            bins_sr: 10
-            bins_fs: 5
-            decay: 0.5
-            sample_ratios:
-                - 0.8
-                - 0.7
-                - 0.6
-                - 0.5
-                - 0.4
-            sub_weights:
-                - 1
-                - 0.2
-                - 0.2
-                - 0.2
-                - 0.2
-                - 0.2
-            epochs: 28
-            colsample_bytree: 0.8879
-            learning_rate: 0.2
-            subsample: 0.8789
-            lambda_l1: 205.6999
-            lambda_l2: 580.9768
-            max_depth: 8
-            num_leaves: 210
-            num_threads: 20
-            verbosity: -1
-    dataset:
-        class: DatasetH
-        module_path: qlib.data.dataset
-        kwargs:
-            handler:
-                class: Alpha158
-                module_path: qlib.contrib.data.handler
-                kwargs: *data_handler_config
-            segments:
-                train: [2008-01-01, 2014-12-31]
-                valid: [2015-01-01, 2016-12-31]
-                test: [2017-01-01, 2020-08-01]
-    record: 
-        - class: SignalRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            model: <MODEL>
-            dataset: <DATASET>
-        - class: SigAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            ana_long_short: False
-            ann_scaler: 252
-        - class: PortAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            config: *port_analysis_config

examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha360.yaml

@@ -44,7 +44,7 @@ task:
         kwargs:
             base_model: "gbm"
             loss: mse
-            num_models: 3
+            num_models: 6
             enable_sr: True
             enable_fs: True
             alpha1: 1
@@ -60,8 +60,11 @@ task:
                 - 0.4
             sub_weights:
                 - 1
-                - 1
+                - 0.2
-                - 1
+                - 0.2
+                - 0.2
+                - 0.2
+                - 0.2
             epochs: 136
             colsample_bytree: 0.8879
             learning_rate: 0.0421

examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_Alpha360_csi500.yaml

@@ -1,104 +0,0 @@
-qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
-    region: cn
-market: &market csi500
-benchmark: &benchmark SH000905
-data_handler_config: &data_handler_config
-    start_time: 2008-01-01
-    end_time: 2020-08-01
-    fit_start_time: 2008-01-01
-    fit_end_time: 2014-12-31
-    instruments: *market
-    infer_processors: []
-    learn_processors:
-        - class: DropnaLabel
-        - class: CSRankNorm
-          kwargs:
-              fields_group: label
-    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
-port_analysis_config: &port_analysis_config
-    strategy:
-        class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy
-        kwargs:
-            signal:
-                - <MODEL> 
-                - <DATASET>
-            topk: 50
-            n_drop: 5
-    backtest:
-        start_time: 2017-01-01
-        end_time: 2020-08-01
-        account: 100000000
-        benchmark: *benchmark
-        exchange_kwargs:
-            limit_threshold: 0.095
-            deal_price: close
-            open_cost: 0.0005
-            close_cost: 0.0015
-            min_cost: 5
-task:
-    model:
-        class: DEnsembleModel
-        module_path: qlib.contrib.model.double_ensemble
-        kwargs:
-            base_model: "gbm"
-            loss: mse
-            num_models: 6
-            enable_sr: True
-            enable_fs: True
-            alpha1: 1
-            alpha2: 1
-            bins_sr: 10
-            bins_fs: 5
-            decay: 0.5
-            sample_ratios:
-                - 0.8
-                - 0.7
-                - 0.6
-                - 0.5
-                - 0.4
-            sub_weights:
-                - 1
-                - 0.2
-                - 0.2
-                - 0.2
-                - 0.2
-                - 0.2
-            epochs: 136
-            colsample_bytree: 0.8879
-            learning_rate: 0.0421
-            subsample: 0.8789
-            lambda_l1: 205.6999
-            lambda_l2: 580.9768
-            max_depth: 8
-            num_leaves: 210
-            num_threads: 20
-            verbosity: -1
-    dataset:
-        class: DatasetH
-        module_path: qlib.data.dataset
-        kwargs:
-            handler:
-                class: Alpha360
-                module_path: qlib.contrib.data.handler
-                kwargs: *data_handler_config
-            segments:
-                train: [2008-01-01, 2014-12-31]
-                valid: [2015-01-01, 2016-12-31]
-                test: [2017-01-01, 2020-08-01]
-    record: 
-        - class: SignalRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            model: <MODEL>
-            dataset: <DATASET>
-        - class: SigAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs:
-            ana_long_short: False
-            ann_scaler: 252
-        - class: PortAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            config: *port_analysis_config

examples/benchmarks/DoubleEnsemble/workflow_config_doubleensemble_early_stop_Alpha158.yaml

@@ -1,95 +0,0 @@
-qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
-    region: cn
-market: &market csi300
-benchmark: &benchmark SH000300
-data_handler_config: &data_handler_config
-    start_time: 2008-01-01
-    end_time: 2020-08-01
-    fit_start_time: 2008-01-01
-    fit_end_time: 2014-12-31
-    instruments: *market
-port_analysis_config: &port_analysis_config
-    strategy:
-        class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy
-        kwargs:
-            signal:
-                - <MODEL> 
-                - <DATASET>
-            topk: 50
-            n_drop: 5
-    backtest:
-        start_time: 2017-01-01
-        end_time: 2020-08-01
-        account: 100000000
-        benchmark: *benchmark
-        exchange_kwargs:
-            limit_threshold: 0.095
-            deal_price: close
-            open_cost: 0.0005
-            close_cost: 0.0015
-            min_cost: 5
-task:
-    model:
-        class: DEnsembleModel
-        module_path: qlib.contrib.model.double_ensemble
-        kwargs:
-            base_model: "gbm"
-            loss: mse
-            num_models: 3
-            enable_sr: True
-            enable_fs: True
-            alpha1: 1
-            alpha2: 1
-            bins_sr: 10
-            bins_fs: 5
-            decay: 0.5
-            sample_ratios:
-                - 0.8
-                - 0.7
-                - 0.6
-                - 0.5
-                - 0.4
-            sub_weights:
-                - 1
-                - 1
-                - 1
-            epochs: 1000
-            early_stopping_rounds: 50
-            colsample_bytree: 0.8879
-            learning_rate: 0.2
-            subsample: 0.8789
-            lambda_l1: 205.6999
-            lambda_l2: 580.9768
-            max_depth: 8
-            num_leaves: 210
-            num_threads: 20
-            verbosity: -1
-    dataset:
-        class: DatasetH
-        module_path: qlib.data.dataset
-        kwargs:
-            handler:
-                class: Alpha158
-                module_path: qlib.contrib.data.handler
-                kwargs: *data_handler_config
-            segments:
-                train: [2008-01-01, 2014-12-31]
-                valid: [2015-01-01, 2016-12-31]
-                test: [2017-01-01, 2020-08-01]
-    record: 
-        - class: SignalRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            model: <MODEL>
-            dataset: <DATASET>
-        - class: SigAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            ana_long_short: False
-            ann_scaler: 252
-        - class: PortAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            config: *port_analysis_config

examples/benchmarks/LightGBM/README.md

@@ -2,9 +2,3 @@
 * Code: [https://github.com/microsoft/LightGBM](https://github.com/microsoft/LightGBM)
 * Paper: LightGBM: A Highly Efficient Gradient Boosting
 Decision Tree. [https://proceedings.neurips.cc/paper/2017/file/6449f44a102fde848669bdd9eb6b76fa-Paper.pdf](https://proceedings.neurips.cc/paper/2017/file/6449f44a102fde848669bdd9eb6b76fa-Paper.pdf).
-
-
-# Introductions about the settings/configs.
-
-`workflow_config_lightgbm_multi_freq.yaml`
-- It uses data sources of different frequencies (i.e. multiple frequencies) for daily prediction.

examples/benchmarks/LightGBM/features_sample.py

@@ -5,8 +5,6 @@ from qlib.data.inst_processor import InstProcessor
 
 
 class Resample1minProcessor(InstProcessor):
-    """This processor tries to resample the data. It will reasmple the data from 1min freq to day freq by selecting a specific miniute"""
-
     def __init__(self, hour: int, minute: int, **kwargs):
         self.hour = hour
         self.minute = minute

examples/benchmarks/LightGBM/requirements.txt

@@ -1,3 +1,3 @@
 pandas==1.1.2
 numpy==1.21.0
-lightgbm
+lightgbm==3.1.0

examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_csi500.yaml

@@ -1,72 +0,0 @@
-qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
-    region: cn
-market: &market csi500
-benchmark: &benchmark SH000905
-data_handler_config: &data_handler_config
-    start_time: 2008-01-01
-    end_time: 2020-08-01
-    fit_start_time: 2008-01-01
-    fit_end_time: 2014-12-31
-    instruments: *market
-port_analysis_config: &port_analysis_config
-    strategy:
-        class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy
-        kwargs:
-            model: <MODEL> 
-            dataset: <DATASET>
-            topk: 50
-            n_drop: 5
-    backtest:
-        start_time: 2017-01-01
-        end_time: 2020-08-01
-        account: 100000000
-        benchmark: *benchmark
-        exchange_kwargs:
-            limit_threshold: 0.095
-            deal_price: close
-            open_cost: 0.0005
-            close_cost: 0.0015
-            min_cost: 5
-task:
-    model:
-        class: LGBModel
-        module_path: qlib.contrib.model.gbdt
-        kwargs:
-            loss: mse
-            colsample_bytree: 0.9
-            learning_rate: 0.1
-            subsample: 0.9
-            lambda_l1: 205.6999
-            lambda_l2: 580.9768
-            max_depth: 8
-            num_leaves: 250
-            num_threads: 20
-    dataset:
-        class: DatasetH
-        module_path: qlib.data.dataset
-        kwargs:
-            handler:
-                class: Alpha158
-                module_path: qlib.contrib.data.handler
-                kwargs: *data_handler_config
-            segments:
-                train: [2008-01-01, 2014-12-31]
-                valid: [2015-01-01, 2016-12-31]
-                test: [2017-01-01, 2020-08-01]
-    record: 
-        - class: SignalRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            model: <MODEL>
-            dataset: <DATASET>
-        - class: SigAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            ana_long_short: False
-            ann_scaler: 252
-        - class: PortAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            config: *port_analysis_config

examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha360_csi500.yaml

@@ -1,80 +0,0 @@
-qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
-    region: cn
-market: &market csi500
-benchmark: &benchmark SH000905
-data_handler_config: &data_handler_config
-    start_time: 2008-01-01
-    end_time: 2020-08-01
-    fit_start_time: 2008-01-01
-    fit_end_time: 2014-12-31
-    instruments: *market
-    infer_processors: []
-    learn_processors:
-        - class: DropnaLabel
-        - class: CSRankNorm
-          kwargs:
-              fields_group: label
-    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
-port_analysis_config: &port_analysis_config
-    strategy:
-        class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy
-        kwargs:
-            signal:
-                - <MODEL> 
-                - <DATASET>
-            topk: 50
-            n_drop: 5
-    backtest:
-        start_time: 2017-01-01
-        end_time: 2020-08-01
-        account: 100000000
-        benchmark: *benchmark
-        exchange_kwargs:
-            limit_threshold: 0.095
-            deal_price: close
-            open_cost: 0.0005
-            close_cost: 0.0015
-            min_cost: 5
-task:
-    model:
-        class: LGBModel
-        module_path: qlib.contrib.model.gbdt
-        kwargs:
-            loss: mse
-            colsample_bytree: 0.8879
-            learning_rate: 0.0421
-            subsample: 0.8789
-            lambda_l1: 205.6999
-            lambda_l2: 580.9768
-            max_depth: 8
-            num_leaves: 210
-            num_threads: 20
-    dataset:
-        class: DatasetH
-        module_path: qlib.data.dataset
-        kwargs:
-            handler:
-                class: Alpha360
-                module_path: qlib.contrib.data.handler
-                kwargs: *data_handler_config
-            segments:
-                train: [2008-01-01, 2014-12-31]
-                valid: [2015-01-01, 2016-12-31]
-                test: [2017-01-01, 2020-08-01]
-    record: 
-        - class: SignalRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            model: <MODEL>
-            dataset: <DATASET>
-        - class: SigAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            ana_long_short: False
-            ann_scaler: 252
-        - class: PortAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            config: *port_analysis_config

examples/benchmarks/Linear/workflow_config_linear_Alpha158_csi500.yaml

@@ -1,78 +0,0 @@
-qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
-    region: cn
-market: &market csi500
-benchmark: &benchmark SH000905
-data_handler_config: &data_handler_config
-    start_time: 2008-01-01
-    end_time: 2020-08-01
-    fit_start_time: 2008-01-01
-    fit_end_time: 2014-12-31
-    instruments: *market
-    infer_processors:
-        - class: RobustZScoreNorm
-          kwargs:
-              fields_group: feature
-              clip_outlier: true
-        - class: Fillna
-          kwargs:
-              fields_group: feature
-    learn_processors:
-        - class: DropnaLabel
-        - class: CSRankNorm
-          kwargs:
-              fields_group: label
-port_analysis_config: &port_analysis_config
-    strategy:
-        class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy
-        kwargs:
-            signal:
-                - <MODEL> 
-                - <DATASET>
-            topk: 50
-            n_drop: 5
-    backtest:
-        start_time: 2017-01-01
-        end_time: 2020-08-01
-        account: 100000000
-        benchmark: *benchmark
-        exchange_kwargs:
-            limit_threshold: 0.095
-            deal_price: close
-            open_cost: 0.0005
-            close_cost: 0.0015
-            min_cost: 5
-task:
-    model:
-        class: LinearModel
-        module_path: qlib.contrib.model.linear
-        kwargs:
-            estimator: ols
-    dataset:
-        class: DatasetH
-        module_path: qlib.data.dataset
-        kwargs:
-            handler:
-                class: Alpha158
-                module_path: qlib.contrib.data.handler
-                kwargs: *data_handler_config
-            segments:
-                train: [2008-01-01, 2014-12-31]
-                valid: [2015-01-01, 2016-12-31]
-                test: [2017-01-01, 2020-08-01]
-    record: 
-        - class: SignalRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            model: <MODEL>
-            dataset: <DATASET>
-        - class: SigAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            ana_long_short: True
-            ann_scaler: 252
-        - class: PortAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            config: *port_analysis_config

examples/benchmarks/MLP/workflow_config_mlp_Alpha158_csi500.yaml

@@ -1,102 +0,0 @@
-qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
-    region: cn
-market: &market csi500
-benchmark: &benchmark SH000905
-data_handler_config: &data_handler_config
-    start_time: 2008-01-01
-    end_time: 2020-08-01
-    fit_start_time: 2008-01-01
-    fit_end_time: 2014-12-31
-    instruments: *market
-    infer_processors: [
-        {
-            "class" : "DropCol", 
-            "kwargs":{"col_list": ["VWAP0"]}
-        },
-        {
-             "class" : "CSZFillna", 
-             "kwargs":{"fields_group": "feature"}
-        }
-    ]
-    learn_processors: [
-        {
-            "class" : "DropCol", 
-            "kwargs":{"col_list": ["VWAP0"]}
-        },
-        {
-            "class" : "DropnaProcessor", 
-            "kwargs":{"fields_group": "feature"}
-        },
-        "DropnaLabel",
-        {
-            "class": "CSZScoreNorm", 
-            "kwargs": {"fields_group": "label"}
-        }
-    ]
-    process_type: "independent"
-
-port_analysis_config: &port_analysis_config
-    strategy:
-        class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy
-        kwargs:
-            signal:
-                - <MODEL> 
-                - <DATASET>
-            topk: 50
-            n_drop: 5
-    backtest:
-        start_time: 2017-01-01
-        end_time: 2020-08-01
-        account: 100000000
-        benchmark: *benchmark
-        exchange_kwargs:
-            limit_threshold: 0.095
-            deal_price: close
-            open_cost: 0.0005
-            close_cost: 0.0015
-            min_cost: 5
-task:
-    model:
-        class: DNNModelPytorch
-        module_path: qlib.contrib.model.pytorch_nn
-        kwargs:
-            loss: mse
-            lr: 0.002
-            lr_decay: 0.96
-            lr_decay_steps: 100
-            optimizer: adam
-            max_steps: 8000
-            batch_size: 8192
-            GPU: 0
-            weight_decay: 0.0002
-            pt_model_kwargs:
-              input_dim: 157
-    dataset:
-        class: DatasetH
-        module_path: qlib.data.dataset
-        kwargs:
-            handler:
-                class: Alpha158
-                module_path: qlib.contrib.data.handler
-                kwargs: *data_handler_config
-            segments:
-                train: [2008-01-01, 2014-12-31]
-                valid: [2015-01-01, 2016-12-31]
-                test: [2017-01-01, 2020-08-01]
-    record: 
-        - class: SignalRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            model: <MODEL>
-            dataset: <DATASET>
-        - class: SigAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            ana_long_short: False
-            ann_scaler: 252
-        - class: PortAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            config: *port_analysis_config

examples/benchmarks/MLP/workflow_config_mlp_Alpha360_csi500.yaml

@@ -1,89 +0,0 @@
-qlib_init:
-    provider_uri: "~/.qlib/qlib_data/cn_data"
-    region: cn
-market: &market csi500
-benchmark: &benchmark SH000905
-data_handler_config: &data_handler_config
-    start_time: 2008-01-01
-    end_time: 2020-08-01
-    fit_start_time: 2008-01-01
-    fit_end_time: 2014-12-31
-    instruments: *market
-    infer_processors:
-        - class: RobustZScoreNorm
-          kwargs:
-              fields_group: feature
-              clip_outlier: true
-        - class: Fillna
-          kwargs:
-              fields_group: feature
-    learn_processors:
-        - class: DropnaLabel
-        - class: CSRankNorm
-          kwargs:
-              fields_group: label
-    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
-
-port_analysis_config: &port_analysis_config
-    strategy:
-        class: TopkDropoutStrategy
-        module_path: qlib.contrib.strategy
-        kwargs:
-            signal:
-                - <MODEL> 
-                - <DATASET>
-            topk: 50
-            n_drop: 5
-    backtest:
-        start_time: 2017-01-01
-        end_time: 2020-08-01
-        account: 100000000
-        benchmark: *benchmark
-        exchange_kwargs:
-            limit_threshold: 0.095
-            deal_price: close
-            open_cost: 0.0005
-            close_cost: 0.0015
-            min_cost: 5
-task:
-    model:
-        class: DNNModelPytorch
-        module_path: qlib.contrib.model.pytorch_nn
-        kwargs:
-            loss: mse
-            lr: 0.002
-            lr_decay: 0.96
-            lr_decay_steps: 100
-            optimizer: adam
-            max_steps: 8000
-            batch_size: 4096
-            GPU: 0
-            pt_model_kwargs:
-              input_dim: 360
-    dataset:
-        class: DatasetH
-        module_path: qlib.data.dataset
-        kwargs:
-            handler:
-                class: Alpha360
-                module_path: qlib.contrib.data.handler
-                kwargs: *data_handler_config
-            segments:
-                train: [2008-01-01, 2014-12-31]
-                valid: [2015-01-01, 2016-12-31]
-                test: [2017-01-01, 2020-08-01]
-    record: 
-        - class: SignalRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            model: <MODEL>
-            dataset: <DATASET>
-        - class: SigAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            ana_long_short: False
-            ann_scaler: 252
-        - class: PortAnaRecord
-          module_path: qlib.workflow.record_temp
-          kwargs: 
-            config: *port_analysis_config

examples/benchmarks/README.md

@@ -20,9 +20,7 @@ The numbers shown below demonstrate the performance of the entire `workflow` of
 > NOTE:
 > We have very limited resources to implement and finetune the models. We tried our best effort to fairly compare these models.  But some models may have greater potential than what it looks like in the table below.  Your contribution is highly welcomed to explore their potential.
 
-## Results on CSI300
+## Alpha158 dataset
-
-### Alpha158 dataset
 
 | Model Name                               | Dataset                             | IC          | ICIR        | Rank IC     | Rank ICIR   | Annualized Return | Information Ratio | Max Drawdown |
 |------------------------------------------|-------------------------------------|-------------|-------------|-------------|-------------|-------------------|-------------------|--------------|
@@ -43,9 +41,10 @@ The numbers shown below demonstrate the performance of the entire `workflow` of
 | TFT (Bryan Lim, et al.)                  | Alpha158(with selected 20 features) | 0.0358±0.00 | 0.2160±0.03 | 0.0116±0.01 | 0.0720±0.03 | 0.0847±0.02       | 0.8131±0.19       | -0.1824±0.03 |
 | MLP                                      | Alpha158                            | 0.0376±0.00 | 0.2846±0.02 | 0.0429±0.00 | 0.3220±0.01 | 0.0895±0.02       | 1.1408±0.23       | -0.1103±0.02 |
 | LightGBM(Guolin Ke, et al.)              | Alpha158                            | 0.0448±0.00 | 0.3660±0.00 | 0.0469±0.00 | 0.3877±0.00 | 0.0901±0.00       | 1.0164±0.00       | -0.1038±0.00 |
-| DoubleEnsemble(Chuheng Zhang, et al.)    | Alpha158                            | 0.0521±0.00 | 0.4223±0.01 | 0.0502±0.00 | 0.4117±0.01 | 0.1158±0.01       | 1.3432±0.11       | -0.0920±0.01 |
+| DoubleEnsemble(Chuheng Zhang, et al.)    | Alpha158                            | 0.0544±0.00 | 0.4340±0.00 | 0.0523±0.00 | 0.4284±0.01 | 0.1168±0.01       | 1.3384±0.12       | -0.1036±0.01 |
 
-### Alpha360 dataset
+
+## Alpha360 dataset
 
 | Model Name                                | Dataset  | IC          | ICIR        | Rank IC     | Rank ICIR   | Annualized Return | Information Ratio | Max Drawdown |
 |-------------------------------------------|----------|-------------|-------------|-------------|-------------|-------------------|-------------------|--------------|
@@ -55,7 +54,7 @@ The numbers shown below demonstrate the performance of the entire `workflow` of
 | Localformer(Juyong Jiang, et al.)         | Alpha360 | 0.0404±0.00 | 0.2932±0.04 | 0.0542±0.00 | 0.4110±0.03 | 0.0246±0.02       | 0.3211±0.21       | -0.1095±0.02 |
 | CatBoost((Liudmila Prokhorenkova, et al.) | Alpha360 | 0.0378±0.00 | 0.2714±0.00 | 0.0467±0.00 | 0.3659±0.00 | 0.0292±0.00       | 0.3781±0.00       | -0.0862±0.00 |
 | XGBoost(Tianqi Chen, et al.)              | Alpha360 | 0.0394±0.00 | 0.2909±0.00 | 0.0448±0.00 | 0.3679±0.00 | 0.0344±0.00       | 0.4527±0.02       | -0.1004±0.00 |
-| DoubleEnsemble(Chuheng Zhang, et al.)     | Alpha360 | 0.0390±0.00 | 0.2946±0.01 | 0.0486±0.00 | 0.3836±0.01 | 0.0462±0.01       | 0.6151±0.18       | -0.0915±0.01 |
+| DoubleEnsemble(Chuheng Zhang, et al.)     | Alpha360 | 0.0404±0.00 | 0.3023±0.00 | 0.0495±0.00 | 0.3898±0.00 | 0.0468±0.01       | 0.6302±0.20       | -0.0860±0.01 |
 | LightGBM(Guolin Ke, et al.)               | Alpha360 | 0.0400±0.00 | 0.3037±0.00 | 0.0499±0.00 | 0.4042±0.00 | 0.0558±0.00       | 0.7632±0.00       | -0.0659±0.00 |
 | TCN(Shaojie Bai, et al.)                  | Alpha360 | 0.0441±0.00 | 0.3301±0.02 | 0.0519±0.00 | 0.4130±0.01 | 0.0604±0.02       | 0.8295±0.34       | -0.1018±0.03 |
 | ALSTM (Yao Qin, et al.)                   | Alpha360 | 0.0497±0.00 | 0.3829±0.04 | 0.0599±0.00 | 0.4736±0.03 | 0.0626±0.02       | 0.8651±0.31       | -0.0994±0.03 |
@@ -74,74 +73,8 @@ The numbers shown below demonstrate the performance of the entire `workflow` of
 - The base model of DoubleEnsemble is LGBM.
 - The base model of TCTS is GRU.
 - About the datasets
-  - Alpha158 is a tabular dataset. There are less spatial relationships between different features. Each feature are carefully designed by human (a.k.a feature engineering)
+  - Alpha158 is a tabular dataset. There are less spatial relationships between different features. Each feature are carefully desgined by human (a.k.a feature engineering)
   - Alpha360 contains raw price and volue data without much feature engineering. There are strong strong spatial relationships between the features in the time dimension.
 - The metrics can be categorized into two
    - Signal-based evaluation:  IC, ICIR, Rank IC, Rank ICIR
-      - ![equation](https://latex.codecogs.com/gif.latex?%5Ctext%7Bcorr%7D%28%5Ctextbf%7Bx%7D%2C%5Ctextbf%7By%7D%29%3D%5Cfrac%7B%5Csum_i%20%28x_i-%5Cbar%7Bx%7D%29%28y_i-%5Cbar%7By%7D%29%7D%7B%5Csqrt%7B%5Csum_i%28x_i-%5Cbar%7Bx%7D%29%5E2%5Csum_i%28y_i-%5Cbar%7By%7D%29%5E2%7D%7D)
-      - ![equation](https://latex.codecogs.com/gif.latex?%5Ctext%7BIC%7D%5E%7B%28t%29%7D%20%3D%20%5Ctext%7Bcorr%7D%28%5Chat%7B%5Ctextbf%7By%7D%7D%5E%7B%28t%29%7D%2C%20%5Ctextbf%7Bret%7D%5E%7B%28t%29%7D%29)
-      - ![equation](https://latex.codecogs.com/gif.latex?%5Ctext%7BICIR%7D%20%3D%20%5Cfrac%20%7B%5Ctext%7Bmean%7D%28%5Ctextbf%7BIC%7D%29%7D%20%7B%5Ctext%7Bstd%7D%28%5Ctextbf%7BIC%7D%29%7D)
-      - ![equation](https://latex.codecogs.com/gif.latex?%5Ctext%7BRank%20IC%7D%5E%7B%28t%29%7D%20%3D%20%5Ctext%7Bcorr%7D%28%5Ctext%7Brank%7D%28%5Chat%7B%5Ctextbf%7By%7D%7D%5E%7B%28t%29%7D%29%2C%20%5Ctext%7Brank%7D%28%5Ctextbf%7Bret%7D%5E%7B%28t%29%7D%29%29)
-      - ![equation](https://latex.codecogs.com/gif.latex?%5Ctext%7BRank%20ICIR%7D%20%3D%20%5Cfrac%20%7B%5Ctext%7Bmean%7D%28%5Ctextbf%7BRank%20IC%7D%29%7D%20%7B%5Ctext%7Bstd%7D%28%5Ctextbf%7BRankIC%7D%29%7D)
    - Portfolio-based metrics:  Annualized Return, Information Ratio, Max Drawdown
-
-## Results on CSI500
-The results on CSI500 is not complete. PR's for models on csi500 are welcome!
-
-Transfer previous models in CSI300 to CSI500 is quite easy.  You can try models with just a few commands below.
-```
-cd examples/benchmarks/LightGBM
-pip install -r requirements.txt
-
-# create new config and set the benchmark to csi500
-cp workflow_config_lightgbm_Alpha158.yaml workflow_config_lightgbm_Alpha158_csi500.yaml
-sed -i "s/csi300/csi500/g"  workflow_config_lightgbm_Alpha158_csi500.yaml
-sed -i "s/SH000300/SH000905/g"  workflow_config_lightgbm_Alpha158_csi500.yaml
-
-# you can either run the model once
-qrun workflow_config_lightgbm_Alpha158_csi500.yaml
-
-# or run it for multiple times automatically and get the summarized results.
-cd  ../../
-python run_all_model.py run 3 lightgbm Alpha158 csi500  # for models with randomness.  please run it for 20 times.
-```
-
-### Alpha158 dataset
-| Model Name | Dataset  | IC          | ICIR        | Rank IC     | Rank ICIR   | Annualized Return | Information Ratio | Max Drawdown |
-|------------|----------|-------------|-------------|-------------|-------------|-------------------|-------------------|--------------|
-| Linear     | Alpha158 | 0.0332±0.00 | 0.3044±0.00 | 0.0462±0.00 | 0.4326±0.00 | 0.0382±0.00       | 0.1723±0.00       | -0.4876±0.00 |
-| MLP        | Alpha158 | 0.0229±0.01 | 0.2181±0.05 | 0.0360±0.00 | 0.3409±0.02 | 0.0043±0.02       | 0.0602±0.27       | -0.2184±0.04 |
-| LightGBM   | Alpha158 | 0.0399±0.00 | 0.4065±0.00 | 0.0482±0.00 | 0.5101±0.00 | 0.1284±0.00       | 1.5650±0.00       | -0.0635±0.00 |
-| CatBoost   | Alpha158 | 0.0345±0.00 | 0.2855±0.00 | 0.0417±0.00 | 0.3740±0.00 | 0.0496±0.00       | 0.5977±0.00       | -0.1496±0.00 |
-| DoubleEnsemble  | Alpha158 | 0.0380±0.00 | 0.3659±0.00 | 0.0442±0.00 | 0.4324±0.00 | 0.0382±0.00       | 0.1723±0.00       | -0.4876±0.00 |
-
-### Alpha360 dataset
-| Model Name | Dataset  | IC          | ICIR        | Rank IC     | Rank ICIR   | Annualized Return | Information Ratio | Max Drawdown |
-|------------|----------|-------------|-------------|-------------|-------------|-------------------|-------------------|--------------|
-| MLP        | Alpha360 | 0.0258±0.00 | 0.2021±0.02 | 0.0426±0.00 | 0.3840±0.02 | 0.0022±0.02       | 0.0301±0.26       | -0.2064±0.02 |
-| LightGBM   | Alpha360 | 0.0400±0.00 | 0.3605±0.00 | 0.0536±0.00 | 0.5431±0.00 | 0.0505±0.00       | 0.7658±0.02       | -0.1880±0.00 |
-| CatBoost   | Alpha360 | 0.0382±0.00 | 0.3229±0.00 | 0.0489±0.00 | 0.4649±0.00 | 0.0297±0.00       | 0.4227±0.02       | -0.1499±0.01 |
-| DoubleEnsemble  | Alpha360 | 0.0361±0.00 | 0.3092±0.00 | 0.0499±0.00 | 0.4793±0.00 | 0.0382±0.00       | 0.1723±0.02       | -0.4876±0.00 |
-
-# Contributing
-
-Your contributions to new models are highly welcome!
-
-If you want to contribute your new models, you can follow the steps below.
-1. Create a folder for your model
-2. The folder contains following items(you can refer to [this example](https://github.com/microsoft/qlib/tree/main/examples/benchmarks/TCTS)).
-    - `requirements.txt`: required dependencies.
-    - `README.md`: a brief introduction to your models
-    - `workflow_config_<model name>_<dataset>.yaml`: a configuration which can read by `qrun`. You are encouraged to run your model in all datasets.
-3. You can integrate your model as a module [in this folder](https://github.com/microsoft/qlib/tree/main/qlib/contrib/model).
-4. Please updated your results in the benchmark tables, e.g. [Alpha360](#alpha158-dataset), [Alpha158](#alpha158-dataset)(the values of each metric are the mean and std calculated based on 20 runs with different random seeds, if you don't have enough computational resource, you can ask for help in the PR).
-5. Update the info in the index page in the [news list](https://github.com/microsoft/qlib#newspaper-whats-new----sparkling_heart) and [model list](https://github.com/microsoft/qlib#quant-model-paper-zoo).
-
-Finally, you can send PR for review. ([here is an example](https://github.com/microsoft/qlib/pull/1040))
-
-
-# FAQ
-
-Q: What's the difference between models with name `*.py` and `*_ts.py`?
-
-A: Models with name `*_ts.py` are designed for `TSDatasetH` (`TSDatasetH` will create time-series automatically from tabular data).  Models with name `*.py` are designed for `DatasetH` (`DatasetH` is usually used in tabular data.  But users still can apply time-series models on tabular datasets if the columns has time-series relationships). 

examples/benchmarks_dynamic/DDG-DA/README.md

@@ -28,8 +28,6 @@ The default forecasting models are `Linear`. Users can choose other forecasting
 The results of related methods in Qlib's public dataset can be found [here](../)
 
 # Requirements
-Here are the minimal hardware requirements to run the ``workflow.py`` of DDG-DA.
+Here is the minimal hardware requirements to run the ``workflow.py`` of DDG-DA.
 * Memory: 45G
 * Disk: 4G
-
-Pytorch with CPU & RAM will be enough for this example.

examples/rl/README.md

@@ -1,55 +0,0 @@
-This folder contains a simple example of how to run Qlib RL. It contains:
-
-```
-.
-├── experiment_config
-│   ├── backtest       # Backtest config
-│   └── training       # Training config
-├── README.md          # Readme (the current file)
-└── scripts            # Scripts for data pre-processing
-```
-
-## Data preparation
-
-Use [AzCopy](https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10) to download data:
-
-```
-azcopy copy https://qlibpublic.blob.core.windows.net/data/rl/qlib_rl_example_data ./ --recursive
-mv qlib_rl_example_data data
-```
-
-The downloaded data will be placed at `./data`. The original data are in `data/csv`. To create all data needed by the case, run:
-
-```
-bash scripts/data_pipeline.sh
-```
-
-After the execution finishes, the `data/` directory should be like:
-
-```
-data
-├── backtest_orders.csv
-├── bin
-├── csv
-├── pickle
-├── pickle_dataframe
-└── training_order_split
-```
-
-## Run training
-
-Run:
-
-```
-python -m qlib.rl.contrib.train_onpolicy --config_path ./experiment_config/training/config.yml
-```
-
-After training, checkpoints will be stored under `checkpoints/`.
-
-## Run backtest
-
-```
-python -m qlib.rl.contrib.backtest --config_path ./experiment_config/backtest/config.yml
-```
-
-The backtest workflow will use the trained model in `checkpoints/`. The backtest summary can be found in `outputs/`.

examples/rl/experiment_config/backtest/config.yml

@@ -1,57 +0,0 @@
-order_file: ./data/backtest_orders.csv
-start_time: "9:45"
-end_time: "14:44"
-qlib:
-  provider_uri_1min: ./data/bin
-  feature_root_dir: ./data/pickle
-  feature_columns_today: [
-    "$open", "$high", "$low", "$close", "$vwap", "$volume",
-  ]
-  feature_columns_yesterday: [
-    "$open_v1", "$high_v1", "$low_v1", "$close_v1", "$vwap_v1", "$volume_v1",
-  ]
-exchange:
-  limit_threshold: ['$close == 0', '$close == 0']
-  deal_price: ["If($close == 0, $vwap, $close)", "If($close == 0, $vwap, $close)"]
-  volume_threshold:
-    all: ["cum", "0.2 * DayCumsum($volume, '9:45', '14:44')"]
-    buy: ["current", "$close"]
-    sell: ["current", "$close"]
-strategies: 
-  30min: 
-    class: TWAPStrategy
-    module_path: qlib.contrib.strategy.rule_strategy
-    kwargs: {}
-  1day: 
-    class: SAOEIntStrategy
-    module_path: qlib.rl.order_execution.strategy
-    kwargs:
-      state_interpreter:
-        class: FullHistoryStateInterpreter
-        module_path: qlib.rl.order_execution.interpreter
-        kwargs:
-          max_step: 8
-          data_ticks: 240
-          data_dim: 6
-          processed_data_provider:
-            class: PickleProcessedDataProvider
-            module_path: qlib.rl.data.pickle_styled
-            kwargs:
-              data_dir: ./data/pickle_dataframe/feature
-      action_interpreter: 
-        class: CategoricalActionInterpreter
-        module_path: qlib.rl.order_execution.interpreter
-        kwargs: 
-          values: 14
-          max_step: 8
-      network: 
-          class: Recurrent
-          module_path: qlib.rl.order_execution.network
-          kwargs: {}
-      policy: 
-          class: PPO
-          module_path: qlib.rl.order_execution.policy
-          kwargs: 
-            lr: 1.0e-4
-            weight_file: ./checkpoints/latest.pth
-concurrency: 5

examples/rl/experiment_config/training/config.yml

@@ -1,59 +0,0 @@
-simulator:
-  time_per_step: 30
-  vol_limit: null
-env:
-  concurrency: 1
-  parallel_mode: dummy
-action_interpreter:
-  class: CategoricalActionInterpreter
-  kwargs:
-    values: 14
-    max_step: 8
-  module_path: qlib.rl.order_execution.interpreter
-state_interpreter:
-  class: FullHistoryStateInterpreter
-  kwargs:
-    data_dim: 6
-    data_ticks: 240
-    max_step: 8
-    processed_data_provider:
-      class: PickleProcessedDataProvider
-      module_path: qlib.rl.data.pickle_styled
-      kwargs:
-        data_dir: ./data/pickle_dataframe/feature
-  module_path: qlib.rl.order_execution.interpreter
-reward:
-  class: PAPenaltyReward
-  kwargs:
-    penalty: 100.0
-  module_path: qlib.rl.order_execution.reward
-data:
-  source:
-    order_dir: ./data/training_order_split
-    data_dir: ./data/pickle_dataframe/backtest
-    total_time: 240
-    default_start_time: 0
-    default_end_time: 240
-    proc_data_dim: 6
-  num_workers: 0
-  queue_size: 20
-network:
-  class: Recurrent
-  module_path: qlib.rl.order_execution.network
-policy:
-  class: PPO
-  kwargs:
-    lr: 0.0001
-  module_path: qlib.rl.order_execution.policy
-runtime:
-  seed: 42
-  use_cuda: false
-trainer:
-  max_epoch: 2
-  repeat_per_collect: 5
-  earlystop_patience: 2
-  episode_per_collect: 20
-  batch_size: 16
-  val_every_n_epoch: 1
-  checkpoint_path: ./checkpoints
-  checkpoint_every_n_iters: 1

examples/rl/scripts/collect_pickle_dataframe.py

@@ -1,21 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-import os
-import pickle
-import pandas as pd
-from tqdm import tqdm
-
-os.makedirs(os.path.join("data", "pickle_dataframe"), exist_ok=True)
-
-for tag in ("backtest", "feature"):
-    df = pickle.load(open(os.path.join("data", "pickle", f"{tag}.pkl"), "rb"))
-    df = pd.concat(list(df.values())).reset_index()
-    df["date"] = df["datetime"].dt.date.astype("datetime64")
-    instruments = sorted(set(df["instrument"]))
-
-    os.makedirs(os.path.join("data", "pickle_dataframe", tag), exist_ok=True)
-    for instrument in tqdm(instruments):
-        cur = df[df["instrument"] == instrument].sort_values(by=["datetime"])
-        cur = cur.set_index(["instrument", "datetime", "date"])
-        pickle.dump(cur, open(os.path.join("data", "pickle_dataframe", tag, f"{instrument}.pkl"), "wb"))

examples/rl/scripts/data_pipeline.sh

@@ -1,14 +0,0 @@
-# Generate `bin` format data
-set -e
-python ../../scripts/dump_bin.py dump_all --csv_path ./data/csv --qlib_dir ./data/bin --include_fields open,close,high,low,vwap,volume --symbol_field_name symbol --date_field_name date --freq 1min
-
-# Generate pickle format data
-python scripts/gen_pickle_data.py -c scripts/pickle_data_config.yml
-if [ -e stat/ ]; then
-    rm -r stat/
-fi
-python scripts/collect_pickle_dataframe.py
-
-# Sample orders
-python scripts/gen_training_orders.py
-python scripts/gen_backtest_orders.py

examples/rl/scripts/gen_backtest_orders.py

@@ -1,55 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-import argparse
-import os
-import pandas as pd
-import numpy as np
-import pickle
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--seed", type=int, default=20220926)
-parser.add_argument("--num_order", type=int, default=10)
-args = parser.parse_args()
-
-np.random.seed(args.seed)
-
-path = os.path.join("data", "pickle", "backtesttest.pkl")
-df = pickle.load(open(path, "rb")).reset_index()
-df["date"] = df["datetime"].dt.date.astype("datetime64")
-
-instruments = sorted(set(df["instrument"]))
-
-# TODO: The example is expected to be able to handle data containing missing values.
-# TODO: Currently, we just simply skip dates that contain missing data. We will add
-# TODO: this feature in the future.
-skip_dates = {}
-for instrument in instruments:
-    csv_df = pd.read_csv(os.path.join("data", "csv", f"{instrument}.csv"))
-    csv_df = csv_df[csv_df["close"].isna()]
-    dates = set([str(d).split(" ")[0] for d in csv_df["date"]])
-    skip_dates[instrument] = dates
-
-df_list = []
-for instrument in instruments:
-    print(instrument)
-
-    cur_df = df[df["instrument"] == instrument]
-
-    dates = sorted(set([str(d).split(" ")[0] for d in cur_df["date"]]))
-    dates = [date for date in dates if date not in skip_dates[instrument]]
-
-    n = args.num_order
-    df_list.append(
-        pd.DataFrame(
-            {
-                "date": sorted(np.random.choice(dates, size=n, replace=False)),
-                "instrument": [instrument] * n,
-                "amount": np.random.randint(low=3, high=11, size=n) * 100.0,
-                "order_type": np.random.randint(low=0, high=2, size=n),
-            }
-        ).set_index(["date", "instrument"]),
-    )
-
-total_df = pd.concat(df_list)
-total_df.to_csv("data/backtest_orders.csv")

examples/rl/scripts/gen_pickle_data.py

@@ -1,43 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-import yaml
-import argparse
-import os
-from copy import deepcopy
-
-from qlib.contrib.data.highfreq_provider import HighFreqProvider
-
-loader = yaml.FullLoader
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-c", "--config", type=str, default="config.yml")
-    parser.add_argument("-d", "--dest", type=str, default=".")
-    parser.add_argument("-s", "--split", type=str, choices=["none", "date", "stock", "both"], default="stock")
-    args = parser.parse_args()
-
-    conf = yaml.load(open(args.config), Loader=loader)
-
-    for k, v in conf.items():
-        if isinstance(v, dict) and "path" in v:
-            v["path"] = os.path.join(args.dest, v["path"])
-    provider = HighFreqProvider(**conf)
-
-    # Gen dataframe
-    if "feature_conf" in conf:
-        feature = provider._gen_dataframe(deepcopy(provider.feature_conf))
-    if "backtest_conf" in conf:
-        backtest = provider._gen_dataframe(deepcopy(provider.backtest_conf))
-
-    provider.feature_conf["path"] = os.path.splitext(provider.feature_conf["path"])[0] + "/"
-    provider.backtest_conf["path"] = os.path.splitext(provider.backtest_conf["path"])[0] + "/"
-    # Split by date
-    if args.split == "date" or args.split == "both":
-        provider._gen_day_dataset(deepcopy(provider.feature_conf), "feature")
-        provider._gen_day_dataset(deepcopy(provider.backtest_conf), "backtest")
-
-    # Split by stock
-    if args.split == "stock" or args.split == "both":
-        provider._gen_stock_dataset(deepcopy(provider.feature_conf), "feature")
-        provider._gen_stock_dataset(deepcopy(provider.backtest_conf), "backtest")

examples/rl/scripts/gen_training_orders.py

@@ -1,39 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-import argparse
-import os
-import pandas as pd
-import numpy as np
-import pickle
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--seed", type=int, default=20220926)
-parser.add_argument("--stock", type=str, default="AAPL")
-parser.add_argument("--train_size", type=int, default=10)
-parser.add_argument("--valid_size", type=int, default=2)
-parser.add_argument("--test_size", type=int, default=2)
-args = parser.parse_args()
-
-np.random.seed(args.seed)
-
-os.makedirs(os.path.join("data", "training_order_split"), exist_ok=True)
-
-for group, n in zip(("train", "valid", "test"), (args.train_size, args.valid_size, args.test_size)):
-    path = os.path.join("data", "pickle", f"backtest{group}.pkl")
-    df = pickle.load(open(path, "rb")).reset_index()
-    df["date"] = df["datetime"].dt.date.astype("datetime64")
-
-    dates = sorted(set([str(d).split(" ")[0] for d in df["date"]]))
-
-    data_df = pd.DataFrame(
-        {
-            "date": sorted(np.random.choice(dates, size=n, replace=False)),
-            "instrument": [args.stock] * n,
-            "amount": np.random.randint(low=3, high=11, size=n) * 100.0,
-            "order_type": [0] * n,
-        }
-    ).set_index(["date", "instrument"])
-
-    os.makedirs(os.path.join("data", "training_order_split", group), exist_ok=True)
-    pickle.dump(data_df, open(os.path.join("data", "training_order_split", group, f"{args.stock}.pkl"), "wb"))

examples/rl/scripts/pickle_data_config.yml

@@ -1,57 +0,0 @@
-# start & end time for training/validation/test datasets
-start_time: !!str &start 2020-01-01
-end_time: !!str &end 2020-07-31
-train_end_time: !!str &tend 2020-03-31
-valid_start_time: !!str &vstart 2020-04-01
-valid_end_time: !!str &vend 2020-05-31
-test_start_time: !!str &tstart 2020-06-01
-# the instrument set
-instruments: &ins all
-# qlib related configuration
-qlib_conf:
-    provider_uri: ./data/bin # path to generated qlib bin
-    redis_port: 233
-feature_conf:
-    path: ./data/pickle/feature.pkl # output path of feature
-    class: DatasetH
-    module_path: qlib.data.dataset
-    kwargs:
-        handler:
-            class: HighFreqGeneralHandler
-            module_path: qlib.contrib.data.highfreq_handler
-            kwargs:
-                start_time: *start
-                end_time: *end
-                fit_start_time: *start
-                fit_end_time: *tend
-                instruments: *ins
-                day_length: 240 # how many minutes in one trading day
-                infer_processors:
-                - class: HighFreqNorm
-                  module_path: qlib.contrib.data.highfreq_processor
-                  kwargs:
-                    feature_save_dir: ./stat/  #  output path of statistics of features (for feature normalization)
-                    norm_groups: 
-                        price: 10
-                        volume: 2
-        segments:
-            train: !!python/tuple [*start, *tend]
-            valid: !!python/tuple [*vstart, *vend]
-            test: !!python/tuple [*tstart, *end]
-backtest_conf:
-    path: ./data/pickle/backtest.pkl # output path of backtest
-    class: DatasetH
-    module_path: qlib.data.dataset
-    kwargs:
-        handler:
-            class: HighFreqGeneralBacktestHandler
-            module_path: qlib.contrib.data.highfreq_handler
-            kwargs:
-                start_time: *start
-                end_time: *end
-                instruments: *ins
-                day_length: 240
-        segments:
-            train: !!python/tuple [*start, *tend]
-            valid: !!python/tuple [*vstart, *vend]
-            test: !!python/tuple [*tstart, *end]

examples/run_all_model.py

@@ -117,10 +117,8 @@ def get_all_folders(models, exclude) -> dict:
 
 
 # function to get all the files under the model folder
-def get_all_files(folder_path, dataset, universe="") -> (str, str):
+def get_all_files(folder_path, dataset) -> (str, str):
-    if universe != "":
+    yaml_path = str(Path(f"{folder_path}") / f"*{dataset}*.yaml")
-        universe = f"_{universe}"
-    yaml_path = str(Path(f"{folder_path}") / f"*{dataset}{universe}.yaml")
     req_path = str(Path(f"{folder_path}") / f"*.txt")
     yaml_file = glob.glob(yaml_path)
     req_file = glob.glob(req_path)
@@ -226,7 +224,6 @@ class ModelRunner:
         times=1,
         models=None,
         dataset="Alpha360",
-        universe="",
         exclude=False,
         qlib_uri: str = "git+https://github.com/microsoft/qlib#egg=pyqlib",
         exp_folder_name: str = "run_all_model_records",
@@ -248,12 +245,9 @@ class ModelRunner:
             determines whether the model being used is excluded or included.
         dataset : str
             determines the dataset to be used for each model.
-        universe  : str
-            the stock universe of the dataset.
-            default "" indicates that
         qlib_uri : str
             the uri to install qlib with pip
-            it could be URI on the remote or local path (NOTE: the local path must be an absolute path)
+            it could be url on the we or local path (NOTE: the local path must be a absolute path)
         exp_folder_name: str
             the name of the experiment folder
         wait_before_rm_env : bool
@@ -265,15 +259,6 @@ class ModelRunner:
         -------
         Here are some use cases of the function in the bash:
 
-        The run_all_models  will decide which config to run based no `models` `dataset`  `universe`
-        Example 1):
-
-            models="lightgbm", dataset="Alpha158", universe="" will result in running the following config
-            examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
-
-            models="lightgbm", dataset="Alpha158", universe="csi500" will result in running the following config
-            examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_csi500.yaml
-
         .. code-block:: bash
 
             # Case 1 - run all models multiple times
@@ -294,9 +279,6 @@ class ModelRunner:
             # Case 6 - run other models except those are given as arguments for one time
             python run_all_model.py run --models=[mlp,tft,sfm] --exclude=True
 
-            # Case 7 - run lightgbm model on csi500.
-            python run_all_model.py run 3 lightgbm Alpha158 csi500
-
         """
         self._init_qlib(exp_folder_name)
 
@@ -308,7 +290,7 @@ class ModelRunner:
         for fn in folders:
             # get all files
             sys.stderr.write("Retrieving files...\n")
-            yaml_path, req_path = get_all_files(folders[fn], dataset, universe=universe)
+            yaml_path, req_path = get_all_files(folders[fn], dataset)
             if yaml_path is None:
                 sys.stderr.write(f"There is no {dataset}.yaml file in {folders[fn]}")
                 continue

examples/tutorial/detailed_workflow.ipynb

@@ -967,10 +967,10 @@
     "###################################\n",
     "port_analysis_config = {\n",
     "    \"executor\": {\n",
+    "            \"time_per_step\"\n",
     "        \"class\": \"SimulatorExecutor\",\n",
     "        \"module_path\": \"qlib.backtest.executor\",\n",
-    "        \"kwargs\": {\n",
+    "        \"kwargs\": {: \"day\",\n",
-    "            \"time_per_step\": \"day\",\n",
     "            \"generate_portfolio_metrics\": True,\n",
     "        },\n",
     "    },\n",

examples/workflow_by_code.ipynb

@@ -38,9 +38,6 @@
     "    # install qlib\n",
     "    ! pip install --upgrade numpy\n",
     "    ! pip install pyqlib\n",
-    "    if 'google.colab' in sys.modules:\n",
-    "        # The Google colab environment is a little outdated. We have to downgrade the pyyaml to make it compatible with other packages\n",
-    "        ! pip install pyyaml==5.4.1\n",
     "    # reload\n",
     "    site.main()\n",
     "\n",

examples/workflow_by_code.py

@@ -1,12 +1,6 @@
 #  Copyright (c) Microsoft Corporation.
 #  Licensed under the MIT License.
-"""
-Qlib provides two kinds of interfaces. 
-(1) Users could define the Quant research workflow by a simple configuration.
-(2) Qlib is designed in a modularized way and supports creating research workflow by code just like building blocks.
 
-The interface of (1) is `qrun XXX.yaml`.  The interface of (2) is script like this, which nearly does the same thing as `qrun XXX.yaml`
-"""
 import qlib
 from qlib.constant import REG_CN
 from qlib.utils import init_instance_by_config, flatten_dict

qlib/__init__.py

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 from pathlib import Path
 
-__version__ = "0.9.0"
+__version__ = "0.8.5.99"
 __version__bak = __version__  # This version is backup for QlibConfig.reset_qlib_version
 import os
 from typing import Union
@@ -34,7 +34,8 @@ def init(default_conf="client", **kwargs):
     from .config import C  # pylint: disable=C0415
     from .data.cache import H  # pylint: disable=C0415
 
-    logger = get_module_logger("Initialization")
+    # FIXME: this logger ignored the level in config
+    logger = get_module_logger("Initialization", level=logging.INFO)
 
     skip_if_reg = kwargs.pop("skip_if_reg", False)
     if skip_if_reg and C.registered:
@@ -47,7 +48,6 @@ def init(default_conf="client", **kwargs):
     if clear_mem_cache:
         H.clear()
     C.set(default_conf, **kwargs)
-    get_module_logger.setLevel(C.logging_level)
 
     # mount nfs
     for _freq, provider_uri in C.provider_uri.items():
@@ -94,7 +94,7 @@ def _mount_nfs_uri(provider_uri, mount_path, auto_mount: bool = False):
     else:
         # Judging system type
         sys_type = platform.system()
-        if "windows" in sys_type.lower():
+        if "win" in sys_type.lower():
             # system: window
             exec_result = os.popen(f"mount -o anon {provider_uri} {mount_path}")
             result = exec_result.read()
@@ -113,8 +113,6 @@ def _mount_nfs_uri(provider_uri, mount_path, auto_mount: bool = False):
             # system: linux/Unix/Mac
             # check mount
             _remote_uri = provider_uri[:-1] if provider_uri.endswith("/") else provider_uri
-            # `mount a /b/c` is different from `mount a /b/c/`. So we convert it into string to make sure handling it accurately
-            mount_path = str(mount_path)
             _mount_path = mount_path[:-1] if mount_path.endswith("/") else mount_path
             _check_level_num = 2
             _is_mount = False

qlib/backtest/__init__.py

@@ -2,12 +2,8 @@
 # Licensed under the MIT License.
 
 from __future__ import annotations
-
 import copy
-from pathlib import Path
+from typing import List, Tuple, Union, TYPE_CHECKING
-from typing import TYPE_CHECKING, Any, Generator, List, Optional, Tuple, Union
-
-import pandas as pd
 
 from .account import Account
 
@@ -15,14 +11,15 @@ if TYPE_CHECKING:
     from ..strategy.base import BaseStrategy
     from .executor import BaseExecutor
     from .decision import BaseTradeDecision
-
+from .position import Position
-from ..config import C
-from ..log import get_module_logger
-from ..utils import init_instance_by_config
-from .backtest import INDICATOR_METRIC, PORT_METRIC, backtest_loop, collect_data_loop
-from .decision import Order
 from .exchange import Exchange
+from .backtest import backtest_loop
+from .backtest import collect_data_loop
 from .utils import CommonInfrastructure
+from .decision import Order
+from ..utils import init_instance_by_config
+from ..log import get_module_logger
+from ..config import C
 
 # make import more user-friendly by adding `from qlib.backtest import STH`
 
@@ -31,35 +28,26 @@ logger = get_module_logger("backtest caller")
 
 
 def get_exchange(
-    exchange: Union[str, dict, object, Path] = None,
+    exchange=None,
-    freq: str = "day",
+    freq="day",
-    start_time: Union[pd.Timestamp, str] = None,
+    start_time=None,
-    end_time: Union[pd.Timestamp, str] = None,
+    end_time=None,
-    codes: Union[list, str] = "all",
+    codes="all",
-    subscribe_fields: list = [],
+    subscribe_fields=[],
-    open_cost: float = 0.0015,
+    open_cost=0.0015,
-    close_cost: float = 0.0025,
+    close_cost=0.0025,
-    min_cost: float = 5.0,
+    min_cost=5.0,
-    limit_threshold: Union[Tuple[str, str], float, None] = None,
+    limit_threshold=None,
-    deal_price: Union[str, Tuple[str, str], List[str]] = None,
+    deal_price: Union[str, Tuple[str], List[str]] = None,
-    **kwargs: Any,
+    **kwargs,
-) -> Exchange:
+):
     """get_exchange
 
     Parameters
     ----------
 
     # exchange related arguments
-    exchange: Exchange
+    exchange: Exchange().
-        It could be None or any types that are acceptable by `init_instance_by_config`.
-    freq: str
-        frequency of data.
-    start_time: Union[pd.Timestamp, str]
-        closed start time for backtest.
-    end_time: Union[pd.Timestamp, str]
-        closed end time for backtest.
-    codes: Union[list, str]
-        list stock_id list or a string of instruments (i.e. all, csi500, sse50)
     subscribe_fields: list
         subscribe fields.
     open_cost : float
@@ -69,10 +57,12 @@ def get_exchange(
     min_cost : float
         min transaction cost.  It is an absolute amount of cost instead of a ratio of your order's deal amount.
         e.g. You must pay at least 5 yuan of commission regardless of your order's deal amount.
-    deal_price: Union[str, Tuple[str, str], List[str]]
+    trade_unit : int
+        Included in kwargs.  Please refer to the docs of `__init__` of `Exchange`
+    deal_price: Union[str, Tuple[str], List[str]]
                 The `deal_price` supports following two types of input
                 - <deal_price> : str
-                - (<buy_price>, <sell_price>): Tuple[str, str] or List[str]
+                - (<buy_price>, <sell_price>): Tuple[str] or List[str]
 
                 <deal_price>, <buy_price> or <sell_price> := <price>
                 <price> := str
@@ -111,14 +101,10 @@ def get_exchange(
 
 
 def create_account_instance(
-    start_time: Union[pd.Timestamp, str],
+    start_time, end_time, benchmark: str, account: Union[float, int, dict], pos_type: str = "Position"
-    end_time: Union[pd.Timestamp, str],
-    benchmark: Optional[str],
-    account: Union[float, int, dict],
-    pos_type: str = "Position",
 ) -> Account:
     """
-    # TODO: is very strange pass benchmark_config in the account (maybe for report)
+    # TODO: is very strange pass benchmark_config in the account(maybe for report)
     # There should be a post-step to process the report.
 
     Parameters
@@ -146,42 +132,42 @@ def create_account_instance(
             key "cash" means initial cash.
             key "stock1" means the information of first stock with amount and price(optional).
             ...
-    pos_type: str
-        Postion type.
     """
     if isinstance(account, (int, float)):
-        init_cash = account
+        pos_kwargs = {"init_cash": account}
-        position_dict = {}
     elif isinstance(account, dict):
-        init_cash = account.pop("cash")
+        init_cash = account["cash"]
-        position_dict = account
+        del account["cash"]
+        pos_kwargs = {
+            "init_cash": init_cash,
+            "position_dict": account,
+        }
     else:
-        raise ValueError("account must be in (int, float, dict)")
+        raise ValueError("account must be in (int, float, Position)")
 
-    return Account(
+    kwargs = {
-        init_cash=init_cash,
+        "init_cash": account,
-        position_dict=position_dict,
+        "benchmark_config": {
-        pos_type=pos_type,
-        benchmark_config={}
-        if benchmark is None
-        else {
             "benchmark": benchmark,
             "start_time": start_time,
             "end_time": end_time,
         },
-    )
+        "pos_type": pos_type,
+    }
+    kwargs.update(pos_kwargs)
+    return Account(**kwargs)
 
 
 def get_strategy_executor(
-    start_time: Union[pd.Timestamp, str],
+    start_time,
-    end_time: Union[pd.Timestamp, str],
+    end_time,
-    strategy: Union[str, dict, object, Path],
+    strategy: BaseStrategy,
-    executor: Union[str, dict, object, Path],
+    executor: BaseExecutor,
-    benchmark: Optional[str] = "SH000300",
+    benchmark: str = "SH000300",
-    account: Union[float, int, dict] = 1e9,
+    account: Union[float, int, Position] = 1e9,
     exchange_kwargs: dict = {},
     pos_type: str = "Position",
-) -> Tuple[BaseStrategy, BaseExecutor]:
+):
 
     # NOTE:
     # - for avoiding recursive import
@@ -190,11 +176,7 @@ def get_strategy_executor(
     from .executor import BaseExecutor  # pylint: disable=C0415
 
     trade_account = create_account_instance(
-        start_time=start_time,
+        start_time=start_time, end_time=end_time, benchmark=benchmark, account=account, pos_type=pos_type
-        end_time=end_time,
-        benchmark=benchmark,
-        account=account,
-        pos_type=pos_type,
     )
 
     exchange_kwargs = copy.copy(exchange_kwargs)
@@ -214,36 +196,34 @@ def get_strategy_executor(
 
 
 def backtest(
-    start_time: Union[pd.Timestamp, str],
+    start_time,
-    end_time: Union[pd.Timestamp, str],
+    end_time,
-    strategy: Union[str, dict, object, Path],
+    strategy,
-    executor: Union[str, dict, object, Path],
+    executor,
-    benchmark: str = "SH000300",
+    benchmark="SH000300",
-    account: Union[float, int, dict] = 1e9,
+    account=1e9,
-    exchange_kwargs: dict = {},
+    exchange_kwargs={},
     pos_type: str = "Position",
-) -> Tuple[PORT_METRIC, INDICATOR_METRIC]:
+):
-    """initialize the strategy and executor, then backtest function for the interaction of the outermost strategy and
+    """initialize the strategy and executor, then backtest function for the interaction of the outermost strategy and executor in the nested decision execution
-    executor in the nested decision execution
 
     Parameters
     ----------
-    start_time : Union[pd.Timestamp, str]
+    start_time : pd.Timestamp|str
         closed start time for backtest
         **NOTE**: This will be applied to the outmost executor's calendar.
-    end_time : Union[pd.Timestamp, str]
+    end_time : pd.Timestamp|str
         closed end time for backtest
         **NOTE**: This will be applied to the outmost executor's calendar.
         E.g. Executor[day](Executor[1min]),   setting `end_time == 20XX0301` will include all the minutes on 20XX0301
-    strategy : Union[str, dict, object, Path]
+    strategy : Union[str, dict, BaseStrategy]
-        for initializing outermost portfolio strategy. Please refer to the docs of init_instance_by_config for more
+        for initializing outermost portfolio strategy. Please refer to the docs of init_instance_by_config for more information.
-        information.
+    executor : Union[str, dict, BaseExecutor]
-    executor : Union[str, dict, object, Path]
         for initializing the outermost executor.
     benchmark: str
         the benchmark for reporting.
     account : Union[float, int, Position]
-        information for describing how to create the account
+        information for describing how to creating the account
         For `float` or `int`:
             Using Account with only initial cash
         For `Position`:
@@ -255,9 +235,9 @@ def backtest(
 
     Returns
     -------
-    portfolio_dict: PORT_METRIC
+    portfolio_metrics_dict: Dict[PortfolioMetrics]
         it records the trading portfolio_metrics information
-    indicator_dict: INDICATOR_METRIC
+    indicator_dict: Dict[Indicator]
         it computes the trading indicator
         It is organized in a dict format
 
@@ -272,20 +252,21 @@ def backtest(
         exchange_kwargs,
         pos_type=pos_type,
     )
-    return backtest_loop(start_time, end_time, trade_strategy, trade_executor)
+    portfolio_metrics, indicator = backtest_loop(start_time, end_time, trade_strategy, trade_executor)
+    return portfolio_metrics, indicator
 
 
 def collect_data(
-    start_time: Union[pd.Timestamp, str],
+    start_time,
-    end_time: Union[pd.Timestamp, str],
+    end_time,
-    strategy: Union[str, dict, object, Path],
+    strategy,
-    executor: Union[str, dict, object, Path],
+    executor,
-    benchmark: str = "SH000300",
+    benchmark="SH000300",
-    account: Union[float, int, dict] = 1e9,
+    account=1e9,
-    exchange_kwargs: dict = {},
+    exchange_kwargs={},
     pos_type: str = "Position",
     return_value: dict = None,
-) -> Generator[object, None, None]:
+):
     """initialize the strategy and executor, then collect the trade decision data for rl training
 
     please refer to the docs of the backtest for the explanation of the parameters
@@ -310,7 +291,7 @@ def collect_data(
 
 def format_decisions(
     decisions: List[BaseTradeDecision],
-) -> Optional[Tuple[str, List[Tuple[BaseTradeDecision, Union[Tuple, None]]]]]:
+) -> Tuple[str, List[Tuple[BaseTradeDecision, Union[Tuple, None]]]]:
     """
     format the decisions collected by `qlib.backtest.collect_data`
     The decisions will be organized into a tree-like structure.
@@ -335,7 +316,7 @@ def format_decisions(
 
     cur_freq = decisions[0].strategy.trade_calendar.get_freq()
 
-    res: Tuple[str, list] = (cur_freq, [])
+    res = (cur_freq, [])
     last_dec_idx = 0
     for i, dec in enumerate(decisions[1:], 1):
         if dec.strategy.trade_calendar.get_freq() == cur_freq:
@@ -345,4 +326,4 @@ def format_decisions(
     return res
 
 
-__all__ = ["Order", "backtest", "get_strategy_executor"]
+__all__ = ["Order"]

qlib/backtest/account.py

@@ -1,19 +1,15 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 from __future__ import annotations
-
 import copy
-from typing import Dict, List, Optional, Tuple, cast
+from typing import Dict, List, Tuple
-
+from qlib.utils import init_instance_by_config
 import pandas as pd
 
-from qlib.utils import init_instance_by_config
+from .position import BasePosition
-
+from .report import PortfolioMetrics, Indicator
 from .decision import BaseTradeDecision, Order
 from .exchange import Exchange
-from .high_performance_ds import BaseOrderIndicator
-from .position import BasePosition
-from .report import Indicator, PortfolioMetrics
 
 """
 rtn & earning in the Account
@@ -38,42 +34,40 @@ class AccumulatedInfo:
     AccumulatedInfo should be shared across different levels
     """
 
-    def __init__(self) -> None:
+    def __init__(self):
         self.reset()
 
-    def reset(self) -> None:
+    def reset(self):
-        self.rtn: float = 0.0  # accumulated return, do not consider cost
+        self.rtn = 0  # accumulated return, do not consider cost
-        self.cost: float = 0.0  # accumulated cost
+        self.cost = 0  # accumulated cost
-        self.to: float = 0.0  # accumulated turnover
+        self.to = 0  # accumulated turnover
 
-    def add_return_value(self, value: float) -> None:
+    def add_return_value(self, value):
         self.rtn += value
 
-    def add_cost(self, value: float) -> None:
+    def add_cost(self, value):
         self.cost += value
 
-    def add_turnover(self, value: float) -> None:
+    def add_turnover(self, value):
         self.to += value
 
     @property
-    def get_return(self) -> float:
+    def get_return(self):
         return self.rtn
 
     @property
-    def get_cost(self) -> float:
+    def get_cost(self):
         return self.cost
 
     @property
-    def get_turnover(self) -> float:
+    def get_turnover(self):
         return self.to
 
 
 class Account:
     """
-    The correctness of the metrics of Account in nested execution depends on the shallow copy of `trade_account` in
+    The correctness of the metrics of Account in nested execution depends on the shallow copy of `trade_account` in qlib/backtest/executor.py:NestedExecutor
-    qlib/backtest/executor.py:NestedExecutor
+    Different level of executor has different Account object when calculating metrics. But the position object is shared cross all the Account object.
-    Different level of executor has different Account object when calculating metrics. But the position object is
-    shared cross all the Account object.
     """
 
     def __init__(
@@ -84,7 +78,7 @@ class Account:
         benchmark_config: dict = {},
         pos_type: str = "Position",
         port_metr_enabled: bool = True,
-    ) -> None:
+    ):
         """the trade account of backtest.
 
         Parameters
@@ -105,10 +99,10 @@ class Account:
 
         self._pos_type = pos_type
         self._port_metr_enabled = port_metr_enabled
-        self.benchmark_config: dict = {}  # avoid no attribute error
+        self.benchmark_config = None  # avoid no attribute error
         self.init_vars(init_cash, position_dict, freq, benchmark_config)
 
-    def init_vars(self, init_cash: float, position_dict: dict, freq: str, benchmark_config: dict) -> None:
+    def init_vars(self, init_cash, position_dict, freq: str, benchmark_config: dict):
         # 1) the following variables are shared by multiple layers
         # - you will see a shallow copy instead of deepcopy in the NestedExecutor;
         self.init_cash = init_cash
@@ -120,22 +114,22 @@ class Account:
                     "position_dict": position_dict,
                 },
                 "module_path": "qlib.backtest.position",
-            },
+            }
         )
         self.accum_info = AccumulatedInfo()
 
         # 2) following variables are not shared between layers
-        self.portfolio_metrics: Optional[PortfolioMetrics] = None
+        self.portfolio_metrics = None
-        self.hist_positions: Dict[pd.Timestamp, BasePosition] = {}
+        self.hist_positions = {}
         self.reset(freq=freq, benchmark_config=benchmark_config)
 
-    def is_port_metr_enabled(self) -> bool:
+    def is_port_metr_enabled(self):
         """
         Is portfolio-based metrics enabled.
         """
         return self._port_metr_enabled and not self.current_position.skip_update()
 
-    def reset_report(self, freq: str, benchmark_config: dict) -> None:
+    def reset_report(self, freq, benchmark_config):
         # portfolio related metrics
         if self.is_port_metr_enabled():
             # NOTE:
@@ -146,13 +140,13 @@ class Account:
             # fill stock value
             # The frequency of account may not align with the trading frequency.
             # This may result in obscure bugs when data quality is low.
-            if isinstance(self.benchmark_config, dict) and "start_time" in self.benchmark_config:
+            if isinstance(self.benchmark_config, dict) and self.benchmark_config.get("start_time") is not None:
                 self.current_position.fill_stock_value(self.benchmark_config["start_time"], self.freq)
 
         # trading related metrics(e.g. high-frequency trading)
         self.indicator = Indicator()
 
-    def reset(self, freq: str = None, benchmark_config: dict = None, port_metr_enabled: bool = None) -> None:
+    def reset(self, freq=None, benchmark_config=None, port_metr_enabled: bool = None):
         """reset freq and report of account
 
         Parameters
@@ -161,7 +155,6 @@ class Account:
             frequency of account & report, by default None
         benchmark_config : {}, optional
             benchmark config of report, by default None
-        port_metr_enabled: bool
         """
         if freq is not None:
             self.freq = freq
@@ -172,13 +165,13 @@ class Account:
 
         self.reset_report(self.freq, self.benchmark_config)
 
-    def get_hist_positions(self) -> Dict[pd.Timestamp, BasePosition]:
+    def get_hist_positions(self):
         return self.hist_positions
 
-    def get_cash(self) -> float:
+    def get_cash(self):
         return self.current_position.get_cash()
 
-    def _update_state_from_order(self, order: Order, trade_val: float, cost: float, trade_price: float) -> None:
+    def _update_state_from_order(self, order, trade_val, cost, trade_price):
         if self.is_port_metr_enabled():
             # update turnover
             self.accum_info.add_turnover(trade_val)
@@ -198,14 +191,13 @@ class Account:
                 profit = self.current_position.get_stock_price(order.stock_id) * trade_amount - trade_val
                 self.accum_info.add_return_value(profit)  # note here do not consider cost
 
-    def update_order(self, order: Order, trade_val: float, cost: float, trade_price: float) -> None:
+    def update_order(self, order, trade_val, cost, trade_price):
         if self.current_position.skip_update():
             # TODO: supporting polymorphism for account
             # updating order for infinite position is meaningless
             return
 
-        # if stock is sold out, no stock price information in Position, then we should update account first,
+        # if stock is sold out, no stock price information in Position, then we should update account first, then update current position
-        # then update current position
         # if stock is bought, there is no stock in current position, update current, then update account
         # The cost will be subtracted from the cash at last. So the trading logic can ignore the cost calculation
         if order.direction == Order.SELL:
@@ -220,40 +212,29 @@ class Account:
             self.current_position.update_order(order, trade_val, cost, trade_price)
             self._update_state_from_order(order, trade_val, cost, trade_price)
 
-    def update_current_position(
+    def update_current_position(self, trade_start_time, trade_end_time, trade_exchange):
-        self,
+        """update current to make rtn consistent with earning at the end of bar, and update holding bar count of stock"""
-        trade_start_time: pd.Timestamp,
-        trade_end_time: pd.Timestamp,
-        trade_exchange: Exchange,
-    ) -> None:
-        """
-        Update current to make rtn consistent with earning at the end of bar, and update holding bar count of stock
-        """
         # update price for stock in the position and the profit from changed_price
         # NOTE: updating position does not only serve portfolio metrics, it also serve the strategy
-        assert self.current_position is not None
-
         if not self.current_position.skip_update():
             stock_list = self.current_position.get_stock_list()
             for code in stock_list:
-                # if suspended, no new price to be updated, profit is 0
+                # if suspend, no new price to be updated, profit is 0
                 if trade_exchange.check_stock_suspended(code, trade_start_time, trade_end_time):
                     continue
-                bar_close = cast(float, trade_exchange.get_close(code, trade_start_time, trade_end_time))
+                bar_close = trade_exchange.get_close(code, trade_start_time, trade_end_time)
                 self.current_position.update_stock_price(stock_id=code, price=bar_close)
             # update holding day count
             # NOTE: updating bar_count does not only serve portfolio metrics, it also serve the strategy
             self.current_position.add_count_all(bar=self.freq)
 
-    def update_portfolio_metrics(self, trade_start_time: pd.Timestamp, trade_end_time: pd.Timestamp) -> None:
+    def update_portfolio_metrics(self, trade_start_time, trade_end_time):
         """update portfolio_metrics"""
         # calculate earning
         # account_value - last_account_value
         # for the first trade date, account_value - init_cash
         # self.portfolio_metrics.is_empty() to judge is_first_trade_date
         # get last_account_value, last_total_cost, last_total_turnover
-        assert self.portfolio_metrics is not None
-
         if self.portfolio_metrics.is_empty():
             last_account_value = self.init_cash
             last_total_cost = 0
@@ -262,16 +243,14 @@ class Account:
             last_account_value = self.portfolio_metrics.get_latest_account_value()
             last_total_cost = self.portfolio_metrics.get_latest_total_cost()
             last_total_turnover = self.portfolio_metrics.get_latest_total_turnover()
-
         # get now_account_value, now_stock_value, now_earning, now_cost, now_turnover
         now_account_value = self.current_position.calculate_value()
         now_stock_value = self.current_position.calculate_stock_value()
         now_earning = now_account_value - last_account_value
         now_cost = self.accum_info.get_cost - last_total_cost
         now_turnover = self.accum_info.get_turnover - last_total_turnover
-
         # update portfolio_metrics for today
-        # judge whether the trading is begin.
+        # judge whether the the trading is begin.
         # and don't add init account state into portfolio_metrics, due to we don't have excess return in those days.
         self.portfolio_metrics.update_portfolio_metrics_record(
             trade_start_time=trade_start_time,
@@ -288,7 +267,7 @@ class Account:
             stock_value=now_stock_value,
         )
 
-    def update_hist_positions(self, trade_start_time: pd.Timestamp) -> None:
+    def update_hist_positions(self, trade_start_time):
         """update history position"""
         now_account_value = self.current_position.calculate_value()
         # set now_account_value to position
@@ -304,11 +283,11 @@ class Account:
         trade_exchange: Exchange,
         atomic: bool,
         outer_trade_decision: BaseTradeDecision,
-        trade_info: list = [],
+        trade_info: list = None,
-        inner_order_indicators: List[BaseOrderIndicator] = [],
+        inner_order_indicators: List[Dict[str, pd.Series]] = None,
-        decision_list: List[Tuple[BaseTradeDecision, pd.Timestamp, pd.Timestamp]] = [],
+        decision_list: List[Tuple[BaseTradeDecision, pd.Timestamp, pd.Timestamp]] = None,
         indicator_config: dict = {},
-    ) -> None:
+    ):
         """update trade indicators and order indicators in each bar end"""
         # TODO: will skip empty decisions make it faster?  `outer_trade_decision.empty():`
 
@@ -340,11 +319,11 @@ class Account:
         trade_exchange: Exchange,
         atomic: bool,
         outer_trade_decision: BaseTradeDecision,
-        trade_info: list = [],
+        trade_info: list = None,
-        inner_order_indicators: List[BaseOrderIndicator] = [],
+        inner_order_indicators: List[Dict[str, pd.Series]] = None,
-        decision_list: List[Tuple[BaseTradeDecision, pd.Timestamp, pd.Timestamp]] = [],
+        decision_list: List[Tuple[BaseTradeDecision, pd.Timestamp, pd.Timestamp]] = None,
         indicator_config: dict = {},
-    ) -> None:
+    ):
         """update account at each trading bar step
 
         Parameters
@@ -359,8 +338,6 @@ class Account:
             whether the trading executor is atomic, which means there is no higher-frequency trading executor inside it
             - if atomic is True, calculate the indicators with trade_info
             - else, aggregate indicators with inner indicators
-        outer_trade_decision: BaseTradeDecision
-            external trade decision
         trade_info : List[(Order, float, float, float)], optional
             trading information, by default None
             - necessary if atomic is True
@@ -400,10 +377,9 @@ class Account:
             indicator_config=indicator_config,
         )
 
-    def get_portfolio_metrics(self) -> Tuple[pd.DataFrame, dict]:
+    def get_portfolio_metrics(self):
         """get the history portfolio_metrics and positions instance"""
         if self.is_port_metr_enabled():
-            assert self.portfolio_metrics is not None
             _portfolio_metrics = self.portfolio_metrics.generate_portfolio_metrics_dataframe()
             _positions = self.get_hist_positions()
             return _portfolio_metrics, _positions

qlib/backtest/backtest.py

@@ -2,72 +2,48 @@
 # Licensed under the MIT License.
 
 from __future__ import annotations
-
-from typing import Dict, TYPE_CHECKING, Generator, Optional, Tuple, Union, cast
-
-import pandas as pd
-
 from qlib.backtest.decision import BaseTradeDecision
-from qlib.backtest.report import Indicator
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from qlib.strategy.base import BaseStrategy
     from qlib.backtest.executor import BaseExecutor
-
+from ..utils.time import Freq
 from tqdm.auto import tqdm
 
-from ..utils.time import Freq
 
-
+def backtest_loop(start_time, end_time, trade_strategy: BaseStrategy, trade_executor: BaseExecutor):
-PORT_METRIC = Dict[str, Tuple[pd.DataFrame, dict]]
-INDICATOR_METRIC = Dict[str, Tuple[pd.DataFrame, Indicator]]
-
-
-def backtest_loop(
-    start_time: Union[pd.Timestamp, str],
-    end_time: Union[pd.Timestamp, str],
-    trade_strategy: BaseStrategy,
-    trade_executor: BaseExecutor,
-) -> Tuple[PORT_METRIC, INDICATOR_METRIC]:
     """backtest function for the interaction of the outermost strategy and executor in the nested decision execution
 
     please refer to the docs of `collect_data_loop`
 
     Returns
     -------
-    portfolio_dict: PORT_METRIC
+    portfolio_metrics: PortfolioMetrics
         it records the trading portfolio_metrics information
-    indicator_dict: INDICATOR_METRIC
+    indicator: Indicator
         it computes the trading indicator
     """
-    return_value: dict = {}
+    return_value = {}
     for _decision in collect_data_loop(start_time, end_time, trade_strategy, trade_executor, return_value):
         pass
-
+    return return_value.get("portfolio_metrics"), return_value.get("indicator")
-    portfolio_dict = cast(PORT_METRIC, return_value.get("portfolio_dict"))
-    indicator_dict = cast(INDICATOR_METRIC, return_value.get("indicator_dict"))
-
-    return portfolio_dict, indicator_dict
 
 
 def collect_data_loop(
-    start_time: Union[pd.Timestamp, str],
+    start_time, end_time, trade_strategy: BaseStrategy, trade_executor: BaseExecutor, return_value: dict = None
-    end_time: Union[pd.Timestamp, str],
+):
-    trade_strategy: BaseStrategy,
-    trade_executor: BaseExecutor,
-    return_value: dict = None,
-) -> Generator[BaseTradeDecision, Optional[BaseTradeDecision], None]:
     """Generator for collecting the trade decision data for rl training
 
     Parameters
     ----------
-    start_time : Union[pd.Timestamp, str]
+    start_time : pd.Timestamp|str
         closed start time for backtest
         **NOTE**: This will be applied to the outmost executor's calendar.
-    end_time : Union[pd.Timestamp, str]
+    end_time : pd.Timestamp|str
         closed end time for backtest
         **NOTE**: This will be applied to the outmost executor's calendar.
-        E.g. Executor[day](Executor[1min]), setting `end_time == 20XX0301` will include all the minutes on 20XX0301
+        E.g. Executor[day](Executor[1min]),   setting `end_time == 20XX0301` will include all the minutes on 20XX0301
     trade_strategy : BaseStrategy
         the outermost portfolio strategy
     trade_executor : BaseExecutor
@@ -88,23 +64,18 @@ def collect_data_loop(
         while not trade_executor.finished():
             _trade_decision: BaseTradeDecision = trade_strategy.generate_trade_decision(_execute_result)
             _execute_result = yield from trade_executor.collect_data(_trade_decision, level=0)
-            trade_strategy.post_exe_step(_execute_result)
             bar.update(1)
-        trade_strategy.post_upper_level_exe_step()
 
     if return_value is not None:
         all_executors = trade_executor.get_all_executors()
-
+        all_portfolio_metrics = {
-        portfolio_dict: PORT_METRIC = {}
+            "{}{}".format(*Freq.parse(_executor.time_per_step)): _executor.trade_account.get_portfolio_metrics()
-        indicator_dict: INDICATOR_METRIC = {}
+            for _executor in all_executors
-
+            if _executor.trade_account.is_port_metr_enabled()
-        for executor in all_executors:
+        }
-            key = "{}{}".format(*Freq.parse(executor.time_per_step))
+        all_indicators = {}
-            if executor.trade_account.is_port_metr_enabled():
+        for _executor in all_executors:
-                portfolio_dict[key] = executor.trade_account.get_portfolio_metrics()
+            key = "{}{}".format(*Freq.parse(_executor.time_per_step))
-
+            all_indicators[key] = _executor.trade_account.get_trade_indicator().generate_trade_indicators_dataframe()
-            indicator_df = executor.trade_account.get_trade_indicator().generate_trade_indicators_dataframe()
+            all_indicators[key + "_obj"] = _executor.trade_account.get_trade_indicator()
-            indicator_obj = executor.trade_account.get_trade_indicator()
+        return_value.update({"portfolio_metrics": all_portfolio_metrics, "indicator": all_indicators})
-            indicator_dict[key] = (indicator_df, indicator_obj)
-
-        return_value.update({"portfolio_dict": portfolio_dict, "indicator_dict": indicator_dict})

qlib/backtest/decision.py

@@ -2,33 +2,27 @@
 # Licensed under the MIT License.
 
 from __future__ import annotations
-
-from abc import abstractmethod
-from datetime import time
 from enum import IntEnum
+from qlib.data.data import Cal
+from qlib.utils.time import concat_date_time, epsilon_change
+from qlib.log import get_module_logger
+
+from typing import ClassVar, Optional, Union, List, Tuple
 
 # try to fix circular imports when enabling type hints
-from typing import TYPE_CHECKING, Any, ClassVar, Generic, List, Optional, Tuple, TypeVar, Union, cast
+from typing import TYPE_CHECKING
-
-from qlib.backtest.utils import TradeCalendarManager
-from qlib.data.data import Cal
-from qlib.log import get_module_logger
-from qlib.utils.time import concat_date_time, epsilon_change
 
 if TYPE_CHECKING:
     from qlib.strategy.base import BaseStrategy
     from qlib.backtest.exchange import Exchange
-
+from qlib.backtest.utils import TradeCalendarManager
-from dataclasses import dataclass
-
 import numpy as np
 import pandas as pd
-
+from dataclasses import dataclass
-DecisionType = TypeVar("DecisionType")
 
 
 class OrderDir(IntEnum):
-    # Order direction
+    # Order  direction
     SELL = 0
     BUY = 1
 
@@ -52,7 +46,7 @@ class Order:
     # - they are set by users and is time-invariant.
     stock_id: str
     amount: float  # `amount` is a non-negative and adjusted value
-    direction: OrderDir
+    direction: int
 
     # 2) time variant values:
     # - Users may want to set these values when using lower level APIs
@@ -67,8 +61,8 @@ class Order:
     # What the value should be about in all kinds of cases
     # - not tradable: the deal_amount == 0 , factor is None
     #    - the stock is suspended and the entire order fails. No cost for this order
-    # - dealt or partially dealt: deal_amount >= 0 and factor is not None
+    # - dealed or partially dealed: deal_amount >= 0 and factor is not None
-    deal_amount: float = 0.0  # `deal_amount` is a non-negative value
+    deal_amount: Optional[float] = None  # `deal_amount` is a non-negative value
     factor: Optional[float] = None
 
     # TODO:
@@ -80,10 +74,10 @@ class Order:
     SELL: ClassVar[OrderDir] = OrderDir.SELL
     BUY: ClassVar[OrderDir] = OrderDir.BUY
 
-    def __post_init__(self) -> None:
+    def __post_init__(self):
         if self.direction not in {Order.SELL, Order.BUY}:
             raise NotImplementedError("direction not supported, `Order.SELL` for sell, `Order.BUY` for buy")
-        self.deal_amount = 0.0
+        self.deal_amount = 0
         self.factor = None
 
     @property
@@ -105,7 +99,7 @@ class Order:
         return self.deal_amount * self.sign
 
     @property
-    def sign(self) -> int:
+    def sign(self) -> float:
         """
         return the sign of trading
         - `+1` indicates buying
@@ -118,12 +112,15 @@ class Order:
         if isinstance(direction, OrderDir):
             return direction
         elif isinstance(direction, (int, float, np.integer, np.floating)):
-            return Order.BUY if direction > 0 else Order.SELL
+            if direction > 0:
+                return Order.BUY
+            else:
+                return Order.SELL
         elif isinstance(direction, str):
-            dl = direction.lower().strip()
+            dl = direction.lower()
-            if dl == "sell":
+            if dl.strip() == "sell":
                 return OrderDir.SELL
-            elif dl == "buy":
+            elif dl.strip() == "buy":
                 return OrderDir.BUY
             else:
                 raise NotImplementedError(f"This type of input is not supported")
@@ -135,35 +132,20 @@ class Order:
         else:
             raise NotImplementedError(f"This type of input is not supported")
 
-    @property
-    def key_by_day(self) -> tuple:
-        """A hashable & unique key to identify this order, under the granularity in day."""
-        return self.stock_id, self.date, self.direction
-
-    @property
-    def key(self) -> tuple:
-        """A hashable & unique key to identify this order."""
-        return self.stock_id, self.start_time, self.end_time, self.direction
-
-    @property
-    def date(self) -> pd.Timestamp:
-        """Date of the order."""
-        return pd.Timestamp(self.start_time.replace(hour=0, minute=0, second=0))
-
 
 class OrderHelper:
     """
     Motivation
     - Make generating order easier
         - User may have no knowledge about the adjust-factor information about the system.
-        - It involves too much interaction with the exchange when generating orders.
+        - It involves to much interaction with the exchange when generating orders.
     """
 
-    def __init__(self, exchange: Exchange) -> None:
+    def __init__(self, exchange: Exchange):
         self.exchange = exchange
 
-    @staticmethod
     def create(
+        self,
         code: str,
         amount: float,
         direction: OrderDir,
@@ -193,18 +175,21 @@ class OrderHelper:
         Order:
             The created order
         """
+        if start_time is not None:
+            start_time = pd.Timestamp(start_time)
+        if end_time is not None:
+            end_time = pd.Timestamp(end_time)
         # NOTE: factor is a value belongs to the results section. User don't have to care about it when creating orders
         return Order(
             stock_id=code,
             amount=amount,
-            start_time=None if start_time is None else pd.Timestamp(start_time),
+            start_time=start_time,
-            end_time=None if end_time is None else pd.Timestamp(end_time),
+            end_time=end_time,
             direction=direction,
         )
 
 
 class TradeRange:
-    @abstractmethod
     def __call__(self, trade_calendar: TradeCalendarManager) -> Tuple[int, int]:
         """
         This method will be call with following way
@@ -231,7 +216,6 @@ class TradeRange:
         """
         raise NotImplementedError(f"Please implement the `__call__` method")
 
-    @abstractmethod
     def clip_time_range(self, start_time: pd.Timestamp, end_time: pd.Timestamp) -> Tuple[pd.Timestamp, pd.Timestamp]:
         """
         Parameters
@@ -250,58 +234,56 @@ class TradeRange:
 
 
 class IdxTradeRange(TradeRange):
-    def __init__(self, start_idx: int, end_idx: int) -> None:
+    def __init__(self, start_idx: int, end_idx: int):
         self._start_idx = start_idx
         self._end_idx = end_idx
 
     def __call__(self, trade_calendar: TradeCalendarManager = None) -> Tuple[int, int]:
         return self._start_idx, self._end_idx
 
-    def clip_time_range(self, start_time: pd.Timestamp, end_time: pd.Timestamp) -> Tuple[pd.Timestamp, pd.Timestamp]:
-        raise NotImplementedError
-
 
 class TradeRangeByTime(TradeRange):
     """This is a helper function for make decisions"""
 
-    def __init__(self, start_time: str | time, end_time: str | time) -> None:
+    def __init__(self, start_time: str, end_time: str):
         """
         This is a callable class.
 
         **NOTE**:
-        - It is designed for minute-bar for intra-day trading!!!!!
+        - It is designed for minute-bar for intraday trading!!!!!
         - Both start_time and end_time are **closed** in the range
 
         Parameters
         ----------
-        start_time : str | time
+        start_time : str
             e.g. "9:30"
-        end_time : str | time
+        end_time : str
             e.g. "14:30"
         """
-        self.start_time = pd.Timestamp(start_time).time() if isinstance(start_time, str) else start_time
+        self.start_time = pd.Timestamp(start_time).time()
-        self.end_time = pd.Timestamp(end_time).time() if isinstance(end_time, str) else end_time
+        self.end_time = pd.Timestamp(end_time).time()
         assert self.start_time < self.end_time
 
-    def __call__(self, trade_calendar: TradeCalendarManager) -> Tuple[int, int]:
+    def __call__(self, trade_calendar: TradeCalendarManager = None) -> Tuple[int, int]:
         if trade_calendar is None:
             raise NotImplementedError("trade_calendar is necessary for getting TradeRangeByTime.")
-
+        start = trade_calendar.start_time
-        start_date = trade_calendar.start_time.date()
+        val_start, val_end = concat_date_time(start.date(), self.start_time), concat_date_time(
-        val_start, val_end = concat_date_time(start_date, self.start_time), concat_date_time(start_date, self.end_time)
+            start.date(), self.end_time
+        )
         return trade_calendar.get_range_idx(val_start, val_end)
 
     def clip_time_range(self, start_time: pd.Timestamp, end_time: pd.Timestamp) -> Tuple[pd.Timestamp, pd.Timestamp]:
         start_date = start_time.date()
         val_start, val_end = concat_date_time(start_date, self.start_time), concat_date_time(start_date, self.end_time)
         # NOTE: `end_date` should not be used. Because the `end_date` is for slicing. It may be in the next day
-        # Assumption: start_time and end_time is for intra-day trading. So it is OK for only using start_date
+        # Assumption: start_time and end_time is for intraday trading. So it is OK for only using start_date
         return max(val_start, start_time), min(val_end, end_time)
 
 
-class BaseTradeDecision(Generic[DecisionType]):
+class BaseTradeDecision:
     """
-    Trade decisions are made by strategy and executed by executor
+    Trade decisions ara made by strategy and executed by exeuter
 
     Motivation:
         Here are several typical scenarios for `BaseTradeDecision`
@@ -315,7 +297,7 @@ class BaseTradeDecision(Generic[DecisionType]):
         2. Same as `case 1.3`
     """
 
-    def __init__(self, strategy: BaseStrategy, trade_range: Union[Tuple[int, int], TradeRange] = None) -> None:
+    def __init__(self, strategy: BaseStrategy, trade_range: Union[Tuple[int, int], TradeRange] = None):
         """
         Parameters
         ----------
@@ -334,21 +316,20 @@ class BaseTradeDecision(Generic[DecisionType]):
         """
         self.strategy = strategy
         self.start_time, self.end_time = strategy.trade_calendar.get_step_time()
-        # upper strategy has no knowledge about the sub executor before `_init_sub_trading`
+        self.total_step = None  # upper strategy has no knowledge about the sub executor before `_init_sub_trading`
-        self.total_step: Optional[int] = None
+        if isinstance(trade_range, Tuple):
-        if isinstance(trade_range, tuple):
             # for Tuple[int, int]
             trade_range = IdxTradeRange(*trade_range)
-        self.trade_range: Optional[TradeRange] = trade_range
+        self.trade_range: TradeRange = trade_range
 
-    def get_decision(self) -> List[DecisionType]:
+    def get_decision(self) -> List[object]:
         """
         get the **concrete decision**  (e.g. execution orders)
         This will be called by the inner strategy
 
         Returns
         -------
-        List[DecisionType:
+        List[object]:
             The decision result. Typically it is some orders
             Example:
                 []:
@@ -358,7 +339,7 @@ class BaseTradeDecision(Generic[DecisionType]):
         """
         raise NotImplementedError(f"This type of input is not supported")
 
-    def update(self, trade_calendar: TradeCalendarManager) -> Optional[BaseTradeDecision]:
+    def update(self, trade_calendar: TradeCalendarManager) -> Union["BaseTradeDecision", None]:
         """
         Be called at the **start** of each step.
 
@@ -373,8 +354,10 @@ class BaseTradeDecision(Generic[DecisionType]):
 
         Returns
         -------
+        None:
+            No update, use previous decision(or unavailable)
         BaseTradeDecision:
-            New update, use new decision. If no updates, return None (use previous decision (or unavailable))
+            New update, use new decision
         """
         # purpose 1)
         self.total_step = trade_calendar.get_trade_len()
@@ -382,13 +365,13 @@ class BaseTradeDecision(Generic[DecisionType]):
         # purpose 2)
         return self.strategy.update_trade_decision(self, trade_calendar)
 
-    def _get_range_limit(self, **kwargs: Any) -> Tuple[int, int]:
+    def _get_range_limit(self, **kwargs) -> Tuple[int, int]:
         if self.trade_range is not None:
-            return self.trade_range(trade_calendar=cast(TradeCalendarManager, kwargs.get("inner_calendar")))
+            return self.trade_range(trade_calendar=kwargs.get("inner_calendar"))
         else:
             raise NotImplementedError("The decision didn't provide an index range")
 
-    def get_range_limit(self, **kwargs: Any) -> Tuple[int, int]:
+    def get_range_limit(self, **kwargs) -> Tuple[int, int]:
         """
         return the expected step range for limiting the decision execution time
         Both left and right are **closed**
@@ -429,22 +412,21 @@ class BaseTradeDecision(Generic[DecisionType]):
         """
         try:
             _start_idx, _end_idx = self._get_range_limit(**kwargs)
-        except NotImplementedError as e:
+        except NotImplementedError:
             if "default_value" in kwargs:
                 return kwargs["default_value"]
             else:
                 # Default to get full index
-                raise NotImplementedError(f"The decision didn't provide an index range") from e
+                raise NotImplementedError(f"The decision didn't provide an index range") from NotImplementedError
 
         # clip index
         if getattr(self, "total_step", None) is not None:
             # if `self.update` is called.
             # Then the _start_idx, _end_idx should be clipped
-            assert self.total_step is not None
             if _start_idx < 0 or _end_idx >= self.total_step:
                 logger = get_module_logger("decision")
                 logger.warning(
-                    f"[{_start_idx},{_end_idx}] go beyond the total_step({self.total_step}), it will be clipped.",
+                    f"[{_start_idx},{_end_idx}] go beyoud the total_step({self.total_step}), it will be clipped"
                 )
                 _start_idx, _end_idx = max(0, _start_idx), min(self.total_step - 1, _end_idx)
         return _start_idx, _end_idx
@@ -462,7 +444,7 @@ class BaseTradeDecision(Generic[DecisionType]):
         Parameters
         ----------
         rtype: str
-            - "full": return the full limitation of the decision in the day
+            - "full": return the full limitation of the deicsion in the day
             - "step": return the limitation of current step
 
         raise_error: bool
@@ -515,10 +497,11 @@ class BaseTradeDecision(Generic[DecisionType]):
                 return True
         return True
 
-    def mod_inner_decision(self, inner_trade_decision: BaseTradeDecision) -> None:
+    def mod_inner_decision(self, inner_trade_decision: BaseTradeDecision):
         """
+
         This method will be called on the inner_trade_decision after it is generated.
-        `inner_trade_decision` will be changed **inplace**.
+        `inner_trade_decision` will be changed **inplaced**.
 
         Motivation of the `mod_inner_decision`
         - Leave a hook for outer decision to affect the decision generated by the inner strategy
@@ -536,61 +519,29 @@ class BaseTradeDecision(Generic[DecisionType]):
             inner_trade_decision.trade_range = self.trade_range
 
 
-class EmptyTradeDecision(BaseTradeDecision[object]):
+class EmptyTradeDecision(BaseTradeDecision):
-    def get_decision(self) -> List[object]:
-        return []
-
     def empty(self) -> bool:
         return True
 
 
-class TradeDecisionWO(BaseTradeDecision[Order]):
+class TradeDecisionWO(BaseTradeDecision):
     """
     Trade Decision (W)ith (O)rder.
     Besides, the time_range is also included.
     """
 
-    def __init__(
+    def __init__(self, order_list: List[Order], strategy: BaseStrategy, trade_range: Tuple[int, int] = None):
-        self,
-        order_list: List[Order],
-        strategy: BaseStrategy,
-        trade_range: Union[Tuple[int, int], TradeRange] = None,
-    ) -> None:
         super().__init__(strategy, trade_range=trade_range)
-        self.order_list = cast(List[Order], order_list)
+        self.order_list = order_list
         start, end = strategy.trade_calendar.get_step_time()
         for o in order_list:
-            assert isinstance(o, Order)
             if o.start_time is None:
                 o.start_time = start
             if o.end_time is None:
                 o.end_time = end
 
-    def get_decision(self) -> List[Order]:
+    def get_decision(self) -> List[object]:
         return self.order_list
 
     def __repr__(self) -> str:
-        return (
+        return f"class: {self.__class__.__name__}; strategy: {self.strategy}; trade_range: {self.trade_range}; order_list[{len(self.order_list)}]"
-            f"class: {self.__class__.__name__}; "
-            f"strategy: {self.strategy}; "
-            f"trade_range: {self.trade_range}; "
-            f"order_list[{len(self.order_list)}]"
-        )
-
-
-class TradeDecisionWithDetails(TradeDecisionWO):
-    """
-    Decision with detail information.
-    Detail information is used to generate execution reports.
-    """
-
-    def __init__(
-        self,
-        order_list: List[Order],
-        strategy: BaseStrategy,
-        trade_range: Optional[Tuple[int, int]] = None,
-        details: Optional[Any] = None,
-    ) -> None:
-        super().__init__(order_list, strategy, trade_range)
-
-        self.details = details

qlib/backtest/exchange.py

@@ -1,58 +1,45 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 from __future__ import annotations
-
 from collections import defaultdict
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union, cast
+from typing import TYPE_CHECKING
-
+from typing import List, Tuple, Union
-from ..utils.index_data import IndexData
 
 if TYPE_CHECKING:
     from .account import Account
 
+from qlib.backtest.position import BasePosition, Position
 import random
-
 import numpy as np
 import pandas as pd
 
-from qlib.backtest.position import BasePosition
+from ..data.data import D
-
 from ..config import C
 from ..constant import REG_CN
-from ..data.data import D
 from ..log import get_module_logger
 from .decision import Order, OrderDir, OrderHelper
 from .high_performance_ds import BaseQuote, NumpyQuote
 
 
 class Exchange:
-    # `quote_df` is a pd.DataFrame class that contains basic information for backtesting
-    # After some processing, the data will later be maintained by `quote_cls` object for faster data retrieving.
-    # Some conventions for `quote_df`
-    # - $close is for calculating the total value at end of each day.
-    #   - if $close is None, the stock on that day is regarded as suspended.
-    # - $factor is for rounding to the trading unit;
-    #   - if any $factor is missing when $close exists, trading unit rounding will be disabled
-    quote_df: pd.DataFrame
-
     def __init__(
         self,
-        freq: str = "day",
+        freq="day",
-        start_time: Union[pd.Timestamp, str] = None,
+        start_time=None,
-        end_time: Union[pd.Timestamp, str] = None,
+        end_time=None,
-        codes: Union[list, str] = "all",
+        codes="all",
-        deal_price: Union[str, Tuple[str, str], List[str]] = None,
+        deal_price: Union[str, Tuple[str], List[str]] = None,
-        subscribe_fields: list = [],
+        subscribe_fields=[],
         limit_threshold: Union[Tuple[str, str], float, None] = None,
-        volume_threshold: Union[tuple, dict] = None,
+        volume_threshold=None,
-        open_cost: float = 0.0015,
+        open_cost=0.0015,
-        close_cost: float = 0.0025,
+        close_cost=0.0025,
-        min_cost: float = 5.0,
+        min_cost=5,
-        impact_cost: float = 0.0,
+        impact_cost=0.0,
-        extra_quote: pd.DataFrame = None,
+        extra_quote=None,
-        quote_cls: Type[BaseQuote] = NumpyQuote,
+        quote_cls=NumpyQuote,
-        **kwargs: Any,
+        **kwargs,
-    ) -> None:
+    ):
         """__init__
         :param freq:             frequency of data
         :param start_time:       closed start time for backtest
@@ -85,12 +72,11 @@ class Exchange:
                                  ]
                                 1) ("cum" or "current", limit_str) denotes a single volume limit.
                                     - limit_str is qlib data expression which is allowed to define your own Operator.
-                                    Please refer to qlib/contrib/ops/high_freq.py, here are any custom operator for
+                                    Please refer to qlib/contrib/ops/high_freq.py, here are any custom operator for high frequency,
-                                    high frequency, such as DayCumsum. !!!NOTE: if you want you use the custom
+                                    such as DayCumsum. !!!NOTE: if you want you use the custom operator, you need to
-                                    operator, you need to register it in qlib_init.
+                                    register it in qlib_init.
-                                    - "cum" means that this is a cumulative value over time, such as cumulative market
+                                    - "cum" means that this is a cumulative value over time, such as cumulative market volume.
-                                    volume. So when it is used as a volume limit, it is necessary to subtract the dealt
+                                    So when it is used as a volume limit, it is necessary to subtract the dealt amount.
-                                    amount.
                                     - "current" means that this is a real-time value and will not accumulate over time,
                                     so it can be directly used as a capacity limit.
                                     e.g. ("cum", "0.2 * DayCumsum($volume, '9:45', '14:45')"), ("current", "$bidV1")
@@ -98,7 +84,7 @@ class Exchange:
                                 "buy" means the volume limits of buying. "sell" means the volume limits of selling.
                                 Different volume limits will be aggregated with min(). If volume_threshold is only
                                 ("cum" or "current", limit_str) instead of a dict, the volume limits are for
-                                both by default. In other words, it is same as {"all": ("cum" or "current", limit_str)}.
+                                both by deault. In other words, it is same as {"all": ("cum" or "current", limit_str)}.
                                 3) e.g. "volume_threshold": {
                                             "all": ("cum", "0.2 * DayCumsum($volume, '9:45', '14:45')"),
                                             "buy": ("current", "$askV1"),
@@ -118,14 +104,13 @@ class Exchange:
                                             Necessary fields:
                                                 $close is for calculating the total value at end of each day.
                                             Optional fields:
-                                                $volume is only necessary when we limit the trade amount or calculate
+                                                $volume is only necessary when we limit the trade amount or calculate PA(vwap) indicator
-                                                PA(vwap) indicator
                                                 $vwap is only necessary when we use the $vwap price as the deal price
                                                 $factor is for rounding to the trading unit
-                                                limit_sell will be set to False by default (False indicates we can sell
+                                                limit_sell will be set to False by default(False indicates we can sell this
-                                                this target on this day).
+                                                target on this day).
-                                                limit_buy will be set to False by default (False indicates we can buy
+                                                limit_buy will be set to False by default(False indicates we can buy this
-                                                this target on this day).
+                                                target on this day).
                                     index: MultipleIndex(instrument, pd.Datetime)
         """
         self.freq = freq
@@ -141,7 +126,7 @@ class Exchange:
         if deal_price is None:
             deal_price = C.deal_price
 
-        # we have some verbose information here. So logging is enabled
+        # we have some verbose information here. So logging is enable
         self.logger = get_module_logger("online operator")
 
         # TODO: the quote, trade_dates, codes are not necessary.
@@ -150,7 +135,7 @@ class Exchange:
         if limit_threshold is None:
             if C.region == REG_CN:
                 self.logger.warning(f"limit_threshold not set. The stocks hit the limit may be bought/sold")
-        elif self.limit_type == self.LT_FLT and abs(cast(float, limit_threshold)) > 0.1:
+        elif self.limit_type == self.LT_FLT and abs(limit_threshold) > 0.1:
             if C.region == REG_CN:
                 self.logger.warning(f"limit_threshold may not be set to a reasonable value")
 
@@ -159,7 +144,7 @@ class Exchange:
                 deal_price = "$" + deal_price
             self.buy_price = self.sell_price = deal_price
         elif isinstance(deal_price, (tuple, list)):
-            self.buy_price, self.sell_price = cast(Tuple[str, str], deal_price)
+            self.buy_price, self.sell_price = deal_price
         else:
             raise NotImplementedError(f"This type of input is not supported")
 
@@ -168,7 +153,6 @@ class Exchange:
         self.codes = codes
         # Necessary fields
         # $close is for calculating the total value at end of each day.
-        # - if $close is None, the stock on that day is regarded as suspended.
         # $factor is for rounding to the trading unit
         # $change is for calculating the limit of the stock
 
@@ -177,10 +161,10 @@ class Exchange:
 
         necessary_fields = {self.buy_price, self.sell_price, "$close", "$change", "$factor", "$volume"}
         if self.limit_type == self.LT_TP_EXP:
-            assert isinstance(limit_threshold, tuple)
             for exp in limit_threshold:
                 necessary_fields.add(exp)
-        all_fields = list(necessary_fields | set(vol_lt_fields) | set(subscribe_fields))
+        all_fields = necessary_fields | vol_lt_fields
+        all_fields = list(all_fields | set(subscribe_fields))
 
         self.all_fields = all_fields
 
@@ -198,28 +182,23 @@ class Exchange:
         self.quote_cls = quote_cls
         self.quote: BaseQuote = self.quote_cls(self.quote_df, freq)
 
-    def get_quote_from_qlib(self) -> None:
+    def get_quote_from_qlib(self):
         # get stock data from qlib
         if len(self.codes) == 0:
             self.codes = D.instruments()
         self.quote_df = D.features(
-            self.codes,
+            self.codes, self.all_fields, self.start_time, self.end_time, freq=self.freq, disk_cache=True
-            self.all_fields,
+        ).dropna(subset=["$close"])
-            self.start_time,
-            self.end_time,
-            freq=self.freq,
-            disk_cache=True,
-        )
         self.quote_df.columns = self.all_fields
 
         # check buy_price data and sell_price data
-        for attr in ("buy_price", "sell_price"):
+        for attr in "buy_price", "sell_price":
             pstr = getattr(self, attr)  # price string
             if self.quote_df[pstr].isna().any():
                 self.logger.warning("{} field data contains nan.".format(pstr))
 
         # update trade_w_adj_price
-        if (self.quote_df["$factor"].isna() & ~self.quote_df["$close"].isna()).any():
+        if self.quote_df["$factor"].isna().any():
             # The 'factor.day.bin' file not exists, and `factor` field contains `nan`
             # Use adjusted price
             self.trade_w_adj_price = True
@@ -255,13 +234,13 @@ class Exchange:
             assert set(self.extra_quote.columns) == set(self.quote_df.columns) - {"$change"}
             self.quote_df = pd.concat([self.quote_df, self.extra_quote], sort=False, axis=0)
 
-    LT_TP_EXP = "(exp)"  # Tuple[str, str]:  the limitation is calculated by a Qlib expression.
+    LT_TP_EXP = "(exp)"  # Tuple[str, str]
-    LT_FLT = "float"  # float:  the trading limitation is based on `abs($change) < limit_threshold`
+    LT_FLT = "float"  # float
-    LT_NONE = "none"  # none:  there is no trading limitation
+    LT_NONE = "none"  # none
 
-    def _get_limit_type(self, limit_threshold: Union[tuple, float, None]) -> str:
+    def _get_limit_type(self, limit_threshold):
         """get limit type"""
-        if isinstance(limit_threshold, tuple):
+        if isinstance(limit_threshold, Tuple):
             return self.LT_TP_EXP
         elif isinstance(limit_threshold, float):
             return self.LT_FLT
@@ -270,31 +249,23 @@ class Exchange:
         else:
             raise NotImplementedError(f"This type of `limit_threshold` is not supported")
 
-    def _update_limit(self, limit_threshold: Union[Tuple, float, None]) -> None:
+    def _update_limit(self, limit_threshold):
-        # $close may contain NaN, the nan indicates that the stock is not tradable at that timestamp
-        suspended = self.quote_df["$close"].isna()
         # check limit_threshold
         limit_type = self._get_limit_type(limit_threshold)
         if limit_type == self.LT_NONE:
-            self.quote_df["limit_buy"] = suspended
+            self.quote_df["limit_buy"] = False
-            self.quote_df["limit_sell"] = suspended
+            self.quote_df["limit_sell"] = False
         elif limit_type == self.LT_TP_EXP:
             # set limit
-            limit_threshold = cast(tuple, limit_threshold)
+            self.quote_df["limit_buy"] = self.quote_df[limit_threshold[0]]
-            # astype bool is necessary, because quote_df is an expression and could be float
+            self.quote_df["limit_sell"] = self.quote_df[limit_threshold[1]]
-            self.quote_df["limit_buy"] = self.quote_df[limit_threshold[0]].astype("bool") | suspended
-            self.quote_df["limit_sell"] = self.quote_df[limit_threshold[1]].astype("bool") | suspended
         elif limit_type == self.LT_FLT:
-            limit_threshold = cast(float, limit_threshold)
+            self.quote_df["limit_buy"] = self.quote_df["$change"].ge(limit_threshold)
-            self.quote_df["limit_buy"] = self.quote_df["$change"].ge(limit_threshold) | suspended
+            self.quote_df["limit_sell"] = self.quote_df["$change"].le(-limit_threshold)  # pylint: disable=E1130
-            self.quote_df["limit_sell"] = (
-                self.quote_df["$change"].le(-limit_threshold) | suspended
-            )  # pylint: disable=E1130
 
-    @staticmethod
+    def _get_vol_limit(self, volume_threshold):
-    def _get_vol_limit(volume_threshold: Union[tuple, dict, None]) -> Tuple[Optional[list], Optional[list], set]:
         """
-        preprocess the volume limit.
+        preproccess the volume limit.
         get the fields need to get from qlib.
         get the volume limit list of buying and selling which is composed of all limits.
         Parameters
@@ -324,7 +295,8 @@ class Exchange:
             volume_threshold = {"all": volume_threshold}
 
         assert isinstance(volume_threshold, dict)
-        for key, vol_limit in volume_threshold.items():
+        for key in volume_threshold:
+            vol_limit = volume_threshold[key]
             assert isinstance(vol_limit, tuple)
             fields.add(vol_limit[1])
 
@@ -335,96 +307,60 @@ class Exchange:
 
         return buy_vol_limit, sell_vol_limit, fields
 
-    def check_stock_limit(
+    def check_stock_limit(self, stock_id, start_time, end_time, direction=None):
-        self,
-        stock_id: str,
-        start_time: pd.Timestamp,
-        end_time: pd.Timestamp,
-        direction: int = None,
-    ) -> bool:
         """
         Parameters
         ----------
-        stock_id : str
-        start_time: pd.Timestamp
-        end_time: pd.Timestamp
         direction : int, optional
             trade direction, by default None
             - if direction is None, check if tradable for buying and selling.
             - if direction == Order.BUY, check the if tradable for buying
             - if direction == Order.SELL, check the sell limit for selling.
-
-        Returns
-        -------
-        True: the trading of the stock is limited (maybe hit the highest/lowest price), hence the stock is not tradable
-        False: the trading of the stock is not limited, hence the stock may be tradable
         """
-        # NOTE:
-        # **all** is used when checking limitation.
-        # For example, the stock trading is limited in a day if every minute is limited in a day if every minute is limited.
         if direction is None:
-            # The trading limitation is related to the trading direction
-            # if the direction is not provided, then any limitation from buy or sell will result in trading limitation
             buy_limit = self.quote.get_data(stock_id, start_time, end_time, field="limit_buy", method="all")
             sell_limit = self.quote.get_data(stock_id, start_time, end_time, field="limit_sell", method="all")
-            return bool(buy_limit or sell_limit)
+            return buy_limit or sell_limit
         elif direction == Order.BUY:
-            return cast(bool, self.quote.get_data(stock_id, start_time, end_time, field="limit_buy", method="all"))
+            return self.quote.get_data(stock_id, start_time, end_time, field="limit_buy", method="all")
         elif direction == Order.SELL:
-            return cast(bool, self.quote.get_data(stock_id, start_time, end_time, field="limit_sell", method="all"))
+            return self.quote.get_data(stock_id, start_time, end_time, field="limit_sell", method="all")
         else:
             raise ValueError(f"direction {direction} is not supported!")
 
-    def check_stock_suspended(
+    def check_stock_suspended(self, stock_id, start_time, end_time):
-        self,
-        stock_id: str,
-        start_time: pd.Timestamp,
-        end_time: pd.Timestamp,
-    ) -> bool:
-        """if stock is suspended(hence not tradable), True will be returned"""
         # is suspended
         if stock_id in self.quote.get_all_stock():
-            # suspended stocks are represented by None $close stock
+            return self.quote.get_data(stock_id, start_time, end_time, "$close") is None
-            # The $close may contain NaN,
-            close = self.quote.get_data(stock_id, start_time, end_time, "$close")
-            if close is None:
-                # if no close record exists
-                return True
-            elif isinstance(close, IndexData):
-                # **any** non-NaN $close represents trading opportunity may exist
-                #  if all returned is nan, then the stock is suspended
-                return cast(bool, cast(IndexData, close).isna().all())
-            else:
-                # it is single value, make sure is not None
-                return np.isnan(close)
         else:
-            # if the stock is not in the stock list, then it is not tradable and regarded as suspended
             return True
 
-    def is_stock_tradable(
+    def is_stock_tradable(self, stock_id, start_time, end_time, direction=None):
-        self,
-        stock_id: str,
-        start_time: pd.Timestamp,
-        end_time: pd.Timestamp,
-        direction: int = None,
-    ) -> bool:
         # check if stock can be traded
-        return not (
+        # same as check in check_order
-            self.check_stock_suspended(stock_id, start_time, end_time)
+        if self.check_stock_suspended(stock_id, start_time, end_time) or self.check_stock_limit(
-            or self.check_stock_limit(stock_id, start_time, end_time, direction)
+            stock_id, start_time, end_time, direction
-        )
+        ):
+            return False
+        else:
+            return True
 
-    def check_order(self, order: Order) -> bool:
+    def check_order(self, order):
         # check limit and suspended
-        return self.is_stock_tradable(order.stock_id, order.start_time, order.end_time, order.direction)
+        if self.check_stock_suspended(order.stock_id, order.start_time, order.end_time) or self.check_stock_limit(
+            order.stock_id, order.start_time, order.end_time, order.direction
+        ):
+            return False
+        else:
+            return True
 
     def deal_order(
         self,
-        order: Order,
+        order,
         trade_account: Account = None,
         position: BasePosition = None,
-        dealt_order_amount: Dict[str, float] = defaultdict(float),
+        dealt_order_amount: defaultdict = defaultdict(float),
-    ) -> Tuple[float, float, float]:
+    ):
         """
         Deal order when the actual transaction
         the results section in `Order` will be changed.
@@ -435,9 +371,9 @@ class Exchange:
         :return: trade_val, trade_cost, trade_price
         """
         # check order first.
-        if not self.check_order(order):
+        if self.check_order(order) is False:
             order.deal_amount = 0.0
-            # using np.nan instead of None to make it more convenient to show the value in format string
+            # using np.nan instead of None to make it more convenient to should the value in format string
             self.logger.debug(f"Order failed due to trading limitation: {order}")
             return 0.0, 0.0, np.nan
 
@@ -446,9 +382,7 @@ class Exchange:
 
         # NOTE: order will be changed in this function
         trade_price, trade_val, trade_cost = self._calc_trade_info_by_order(
-            order,
+            order, trade_account.current_position if trade_account else position, dealt_order_amount
-            trade_account.current_position if trade_account else position,
-            dealt_order_amount,
         )
         if trade_val > 1e-5:
             # If the order can only be deal 0 value. Nothing to be updated
@@ -462,50 +396,23 @@ class Exchange:
 
         return trade_val, trade_cost, trade_price
 
-    def get_quote_info(
+    def get_quote_info(self, stock_id, start_time, end_time, method="ts_data_last"):
-        self,
+        return self.quote.get_data(stock_id, start_time, end_time, method=method)
-        stock_id: str,
-        start_time: pd.Timestamp,
-        end_time: pd.Timestamp,
-        field: str,
-        method: str = "ts_data_last",
-    ) -> Union[None, int, float, bool, IndexData]:
-        return self.quote.get_data(stock_id, start_time, end_time, field=field, method=method)
 
-    def get_close(
+    def get_close(self, stock_id, start_time, end_time, method="ts_data_last"):
-        self,
-        stock_id: str,
-        start_time: pd.Timestamp,
-        end_time: pd.Timestamp,
-        method: str = "ts_data_last",
-    ) -> Union[None, int, float, bool, IndexData]:
         return self.quote.get_data(stock_id, start_time, end_time, field="$close", method=method)
 
-    def get_volume(
+    def get_volume(self, stock_id, start_time, end_time, method="sum"):
-        self,
-        stock_id: str,
-        start_time: pd.Timestamp,
-        end_time: pd.Timestamp,
-        method: Optional[str] = "sum",
-    ) -> Union[None, int, float, bool, IndexData]:
         """get the total deal volume of stock with `stock_id` between the time interval [start_time, end_time)"""
         return self.quote.get_data(stock_id, start_time, end_time, field="$volume", method=method)
 
-    def get_deal_price(
+    def get_deal_price(self, stock_id, start_time, end_time, direction: OrderDir, method="ts_data_last"):
-        self,
-        stock_id: str,
-        start_time: pd.Timestamp,
-        end_time: pd.Timestamp,
-        direction: OrderDir,
-        method: Optional[str] = "ts_data_last",
-    ) -> Union[None, int, float, bool, IndexData]:
         if direction == OrderDir.SELL:
             pstr = self.sell_price
         elif direction == OrderDir.BUY:
             pstr = self.buy_price
         else:
             raise NotImplementedError(f"This type of input is not supported")
-
         deal_price = self.quote.get_data(stock_id, start_time, end_time, field=pstr, method=method)
         if method is not None and (deal_price is None or np.isnan(deal_price) or deal_price <= 1e-08):
             self.logger.warning(f"(stock_id:{stock_id}, trade_time:{(start_time, end_time)}, {pstr}): {deal_price}!!!")
@@ -513,16 +420,11 @@ class Exchange:
             deal_price = self.get_close(stock_id, start_time, end_time, method)
         return deal_price
 
-    def get_factor(
+    def get_factor(self, stock_id, start_time, end_time) -> Union[float, None]:
-        self,
-        stock_id: str,
-        start_time: pd.Timestamp,
-        end_time: pd.Timestamp,
-    ) -> Optional[float]:
         """
         Returns
         -------
-        Optional[float]:
+        Union[float, None]:
             `None`: if the stock is suspended `None` may be returned
             `float`: return factor if the factor exists
         """
@@ -532,16 +434,11 @@ class Exchange:
         return self.quote.get_data(stock_id, start_time, end_time, field="$factor", method="ts_data_last")
 
     def generate_amount_position_from_weight_position(
-        self,
+        self, weight_position, cash, start_time, end_time, direction=OrderDir.BUY
-        weight_position: dict,
+    ):
-        cash: float,
-        start_time: pd.Timestamp,
-        end_time: pd.Timestamp,
-        direction: OrderDir = OrderDir.BUY,
-    ) -> dict:
         """
-        Generates the target position according to the weight and the cash.
+        The generate the target position according to the weight and the cash.
-        NOTE: All the cash will be assigned to the tradable stock.
+        NOTE: All the cash will assigned to the tadable stock.
         Parameter:
         weight_position : dict {stock_id : weight}; allocate cash by weight_position
             among then, weight must be in this range: 0 < weight < 1
@@ -554,14 +451,15 @@ class Exchange:
 
         # calculate the total weight of tradable value
         tradable_weight = 0.0
-        for stock_id, wp in weight_position.items():
+        for stock_id in weight_position:
             if self.is_stock_tradable(stock_id=stock_id, start_time=start_time, end_time=end_time):
                 # weight_position must be greater than 0 and less than 1
-                if wp < 0 or wp > 1:
+                if weight_position[stock_id] < 0 or weight_position[stock_id] > 1:
                     raise ValueError(
-                        "weight_position is {}, " "weight_position is not in the range of (0, 1).".format(wp),
+                        "weight_position is {}, "
+                        "weight_position is not in the range of (0, 1).".format(weight_position[stock_id])
                     )
-                tradable_weight += wp
+                tradable_weight += weight_position[stock_id]
 
         if tradable_weight - 1.0 >= 1e-5:
             raise ValueError("tradable_weight is {}, can not greater than 1.".format(tradable_weight))
@@ -569,24 +467,19 @@ class Exchange:
         amount_dict = {}
         for stock_id in weight_position:
             if weight_position[stock_id] > 0.0 and self.is_stock_tradable(
-                stock_id=stock_id,
+                stock_id=stock_id, start_time=start_time, end_time=end_time
-                start_time=start_time,
-                end_time=end_time,
             ):
                 amount_dict[stock_id] = (
                     cash
                     * weight_position[stock_id]
                     / tradable_weight
                     // self.get_deal_price(
-                        stock_id=stock_id,
+                        stock_id=stock_id, start_time=start_time, end_time=end_time, direction=direction
-                        start_time=start_time,
-                        end_time=end_time,
-                        direction=direction,
                     )
                 )
         return amount_dict
 
-    def get_real_deal_amount(self, current_amount: float, target_amount: float, factor: float = None) -> float:
+    def get_real_deal_amount(self, current_amount, target_amount, factor):
         """
         Calculate the real adjust deal amount when considering the trading unit
         :param current_amount:
@@ -608,13 +501,7 @@ class Exchange:
                 deal_amount = self.round_amount_by_trade_unit(deal_amount, factor)
                 return -deal_amount
 
-    def generate_order_for_target_amount_position(
+    def generate_order_for_target_amount_position(self, target_position, current_position, start_time, end_time):
-        self,
-        target_position: dict,
-        current_position: dict,
-        start_time: pd.Timestamp,
-        end_time: pd.Timestamp,
-    ) -> List[Order]:
         """
         Note: some future information is used in this function
         Parameter:
@@ -630,8 +517,7 @@ class Exchange:
         # three parts: kept stock_id, dropped stock_id, new stock_id
         # handle kept stock_id
 
-        # because the order of the set is not fixed, the trading order of the stock is different, so that the backtest
+        # because the order of the set is not fixed, the trading order of the stock is different, so that the backtest results of the same parameter are different;
-        # results of the same parameter are different;
         # so here we sort stock_id, and then randomly shuffle the order of stock_id
         # because the same random seed is used, the final stock_id order is fixed
         sorted_ids = sorted(set(list(current_position.keys()) + list(target_position.keys())))
@@ -639,7 +525,7 @@ class Exchange:
         random.shuffle(sorted_ids)
         for stock_id in sorted_ids:
 
-            # Do not generate order for the non-tradable stocks
+            # Do not generate order for the nontradable stocks
             if not self.is_stock_tradable(stock_id=stock_id, start_time=start_time, end_time=end_time):
                 continue
 
@@ -660,7 +546,7 @@ class Exchange:
                         start_time=start_time,
                         end_time=end_time,
                         factor=factor,
-                    ),
+                    )
                 )
             else:
                 # sell stock
@@ -672,19 +558,14 @@ class Exchange:
                         start_time=start_time,
                         end_time=end_time,
                         factor=factor,
-                    ),
+                    )
                 )
         # return order_list : buy + sell
         return sell_order_list + buy_order_list
 
     def calculate_amount_position_value(
-        self,
+        self, amount_dict, start_time, end_time, only_tradable=False, direction=OrderDir.SELL
-        amount_dict: dict,
+    ):
-        start_time: pd.Timestamp,
-        end_time: pd.Timestamp,
-        only_tradable: bool = False,
-        direction: OrderDir = OrderDir.SELL,
-    ) -> float:
         """Parameter
         position : Position()
         amount_dict : {stock_id : amount}
@@ -695,44 +576,30 @@ class Exchange:
         """
         value = 0
         for stock_id in amount_dict:
-            if not only_tradable or (
+            if (
-                not self.check_stock_suspended(stock_id=stock_id, start_time=start_time, end_time=end_time)
+                only_tradable is True
-                and not self.check_stock_limit(stock_id=stock_id, start_time=start_time, end_time=end_time)
+                and self.check_stock_suspended(stock_id=stock_id, start_time=start_time, end_time=end_time) is False
+                and self.check_stock_limit(stock_id=stock_id, start_time=start_time, end_time=end_time) is False
+                or only_tradable is False
             ):
                 value += (
                     self.get_deal_price(
-                        stock_id=stock_id,
+                        stock_id=stock_id, start_time=start_time, end_time=end_time, direction=direction
-                        start_time=start_time,
-                        end_time=end_time,
-                        direction=direction,
                     )
                     * amount_dict[stock_id]
                 )
         return value
 
-    def _get_factor_or_raise_error(
+    def _get_factor_or_raise_error(self, factor: float = None, stock_id: str = None, start_time=None, end_time=None):
-        self,
-        factor: float = None,
-        stock_id: str = None,
-        start_time: pd.Timestamp = None,
-        end_time: pd.Timestamp = None,
-    ) -> float:
         """Please refer to the docs of get_amount_of_trade_unit"""
         if factor is None:
             if stock_id is not None and start_time is not None and end_time is not None:
                 factor = self.get_factor(stock_id=stock_id, start_time=start_time, end_time=end_time)
             else:
                 raise ValueError(f"`factor` and (`stock_id`, `start_time`, `end_time`) can't both be None")
-        assert factor is not None
         return factor
 
-    def get_amount_of_trade_unit(
+    def get_amount_of_trade_unit(self, factor: float = None, stock_id: str = None, start_time=None, end_time=None):
-        self,
-        factor: float = None,
-        stock_id: str = None,
-        start_time: pd.Timestamp = None,
-        end_time: pd.Timestamp = None,
-    ) -> Optional[float]:
         """
         get the trade unit of amount based on **factor**
         the factor can be given directly or calculated in given time range and stock id.
@@ -750,23 +617,15 @@ class Exchange:
         """
         if not self.trade_w_adj_price and self.trade_unit is not None:
             factor = self._get_factor_or_raise_error(
-                factor=factor,
+                factor=factor, stock_id=stock_id, start_time=start_time, end_time=end_time
-                stock_id=stock_id,
-                start_time=start_time,
-                end_time=end_time,
             )
             return self.trade_unit / factor
         else:
             return None
 
     def round_amount_by_trade_unit(
-        self,
+        self, deal_amount, factor: float = None, stock_id: str = None, start_time=None, end_time=None
-        deal_amount: float,
+    ):
-        factor: float = None,
-        stock_id: str = None,
-        start_time: pd.Timestamp = None,
-        end_time: pd.Timestamp = None,
-    ) -> float:
         """Parameter
         Please refer to the docs of get_amount_of_trade_unit
         deal_amount : float, adjusted amount
@@ -776,15 +635,12 @@ class Exchange:
         if not self.trade_w_adj_price and self.trade_unit is not None:
             # the minimal amount is 1. Add 0.1 for solving precision problem.
             factor = self._get_factor_or_raise_error(
-                factor=factor,
+                factor=factor, stock_id=stock_id, start_time=start_time, end_time=end_time
-                stock_id=stock_id,
-                start_time=start_time,
-                end_time=end_time,
             )
             return (deal_amount * factor + 0.1) // self.trade_unit * self.trade_unit / factor
         return deal_amount
 
-    def _clip_amount_by_volume(self, order: Order, dealt_order_amount: dict) -> Optional[float]:
+    def _clip_amount_by_volume(self, order: Order, dealt_order_amount: dict) -> int:
         """parse the capacity limit string and return the actual amount of orders that can be executed.
         NOTE:
             this function will change the order.deal_amount **inplace**
@@ -796,12 +652,15 @@ class Exchange:
         dealt_order_amount : dict
             :param dealt_order_amount: the dealt order amount dict with the format of {stock_id: float}
         """
-        vol_limit = self.buy_vol_limit if order.direction == Order.BUY else self.sell_vol_limit
+        if order.direction == Order.BUY:
+            vol_limit = self.buy_vol_limit
+        elif order.direction == Order.SELL:
+            vol_limit = self.sell_vol_limit
 
         if vol_limit is None:
             return order.deal_amount
 
-        vol_limit_num: List[float] = []
+        vol_limit_num = []
         for limit in vol_limit:
             assert isinstance(limit, tuple)
             if limit[0] == "current":
@@ -812,7 +671,7 @@ class Exchange:
                     field=limit[1],
                     method="sum",
                 )
-                vol_limit_num.append(cast(float, limit_value))
+                vol_limit_num.append(limit_value)
             elif limit[0] == "cum":
                 limit_value = self.quote.get_data(
                     order.stock_id,
@@ -830,14 +689,12 @@ class Exchange:
         if vol_limit_min < orig_deal_amount:
             self.logger.debug(f"Order clipped due to volume limitation: {order}, {list(zip(vol_limit_num, vol_limit))}")
 
-        return None
+    def _get_buy_amount_by_cash_limit(self, trade_price, cash, cost_ratio):
-
-    def _get_buy_amount_by_cash_limit(self, trade_price: float, cash: float, cost_ratio: float) -> float:
         """return the real order amount after cash limit for buying.
         Parameters
         ----------
         trade_price : float
-        cash : float
+        position : cash
         cost_ratio : float
 
         Return
@@ -845,7 +702,7 @@ class Exchange:
         float
             the real order amount after cash limit for buying.
         """
-        max_trade_amount = 0.0
+        max_trade_amount = 0
         if cash >= self.min_cost:
             # critical_price means the stock transaction price when the service fee is equal to min_cost.
             critical_price = self.min_cost / cost_ratio + self.min_cost
@@ -857,12 +714,7 @@ class Exchange:
                 max_trade_amount = (cash - self.min_cost) / trade_price
         return max_trade_amount
 
-    def _calc_trade_info_by_order(
+    def _calc_trade_info_by_order(self, order, position: Position, dealt_order_amount):
-        self,
-        order: Order,
-        position: Optional[BasePosition],
-        dealt_order_amount: dict,
-    ) -> Tuple[float, float, float]:
         """
         Calculation of trade info
         **NOTE**: Order will be changed in this function
@@ -871,11 +723,8 @@ class Exchange:
         :param dealt_order_amount: the dealt order amount dict with the format of {stock_id: float}
         :return: trade_price, trade_val, trade_cost
         """
-        trade_price = cast(
+        trade_price = self.get_deal_price(order.stock_id, order.start_time, order.end_time, direction=order.direction)
-            float,
+        total_trade_val = self.get_volume(order.stock_id, order.start_time, order.end_time) * trade_price
-            self.get_deal_price(order.stock_id, order.start_time, order.end_time, direction=order.direction),
-        )
-        total_trade_val = cast(float, self.get_volume(order.stock_id, order.start_time, order.end_time)) * trade_price
         order.factor = self.get_factor(order.stock_id, order.start_time, order.end_time)
         order.deal_amount = order.amount  # set to full amount and clip it step by step
         # Clipping amount first
@@ -904,8 +753,7 @@ class Exchange:
                 if not np.isclose(order.deal_amount, current_amount):
                     # when not selling last stock. rounding is necessary
                     order.deal_amount = self.round_amount_by_trade_unit(
-                        min(current_amount, order.deal_amount),
+                        min(current_amount, order.deal_amount), order.factor
-                        order.factor,
                     )
 
                 # in case of negative value of cash
@@ -930,8 +778,7 @@ class Exchange:
                     # The money is not enough
                     max_buy_amount = self._get_buy_amount_by_cash_limit(trade_price, cash, cost_ratio)
                     order.deal_amount = self.round_amount_by_trade_unit(
-                        min(max_buy_amount, order.deal_amount),
+                        min(max_buy_amount, order.deal_amount), order.factor
-                        order.factor,
                     )
                     self.logger.debug(f"Order clipped due to cash limitation: {order}")
                 else:
@@ -942,7 +789,7 @@ class Exchange:
                 order.deal_amount = self.round_amount_by_trade_unit(order.deal_amount, order.factor)
 
         else:
-            raise NotImplementedError("order direction {} error".format(order.direction))
+            raise NotImplementedError("order type {} error".format(order.type))
 
         trade_val = order.deal_amount * trade_price
         trade_cost = max(trade_val * cost_ratio, self.min_cost)

qlib/backtest/executor.py

@@ -1,22 +1,19 @@
-from __future__ import annotations
-
-import copy
 from abc import abstractmethod
-from collections import defaultdict
+import copy
-from types import GeneratorType
-from typing import Any, Dict, Generator, List, Tuple, Union, cast
-
-import pandas as pd
-
-from qlib.backtest.account import Account
 from qlib.backtest.position import BasePosition
 from qlib.log import get_module_logger
+from types import GeneratorType
+from qlib.backtest.account import Account
+import pandas as pd
+from typing import List, Tuple, Union
+from collections import defaultdict
 
-from ..strategy.base import BaseStrategy
+from .decision import Order, BaseTradeDecision
-from ..utils import init_instance_by_config
-from .decision import BaseTradeDecision, Order
 from .exchange import Exchange
-from .utils import CommonInfrastructure, LevelInfrastructure, TradeCalendarManager, get_start_end_idx
+from .utils import TradeCalendarManager, CommonInfrastructure, LevelInfrastructure, get_start_end_idx
+
+from ..utils import init_instance_by_config
+from ..strategy.base import BaseStrategy
 
 
 class BaseExecutor:
@@ -33,9 +30,9 @@ class BaseExecutor:
         track_data: bool = False,
         trade_exchange: Exchange = None,
         common_infra: CommonInfrastructure = None,
-        settle_type: str = BasePosition.ST_NO,
+        settle_type=BasePosition.ST_NO,
-        **kwargs: Any,
+        **kwargs,
-    ) -> None:
+    ):
         """
         Parameters
         ----------
@@ -56,21 +53,15 @@ class BaseExecutor:
                 - 'base_price': the based price than which the trading price is advanced, Optional, default by 'twap'
                     - If 'base_price' is 'twap', the based price is the time weighted average price
                     - If 'base_price' is 'vwap', the based price is the volume weighted average price
-                - 'weight_method': weighted method when calculating total trading pa by different orders' pa in each
+                - 'weight_method': weighted method when calculating total trading pa by different orders' pa in each step, optional, default by 'mean'
-                    step, optional, default by 'mean'
                     - If 'weight_method' is 'mean', calculating mean value of different orders' pa
-                    - If 'weight_method' is 'amount_weighted', calculating amount weighted average value of different
+                    - If 'weight_method' is 'amount_weighted', calculating amount weighted average value of different orders' pa
-                        orders' pa
+                    - If 'weight_method' is 'value_weighted', calculating value weighted average value of different orders' pa
-                    - If 'weight_method' is 'value_weighted', calculating value weighted average value of different
-                        orders' pa
             - 'ffr_config': config for calculating fulfill rate(ffr), optional
-                - 'weight_method': weighted method when calculating total trading ffr by different orders' ffr in each
+                - 'weight_method': weighted method when calculating total trading ffr by different orders' ffr in each step, optional, default by 'mean'
-                    step, optional, default by 'mean'
                     - If 'weight_method' is 'mean', calculating mean value of different orders' ffr
-                    - If 'weight_method' is 'amount_weighted', calculating amount weighted average value of different
+                    - If 'weight_method' is 'amount_weighted', calculating amount weighted average value of different orders' ffr
-                        orders' ffr
+                    - If 'weight_method' is 'value_weighted', calculating value weighted average value of different orders' ffr
-                    - If 'weight_method' is 'value_weighted', calculating value weighted average value of different
-                        orders' ffr
             Example:
                 {
                     'show_indicator': True,
@@ -88,8 +79,7 @@ class BaseExecutor:
             whether to print trading info, by default False
         track_data : bool, optional
             whether to generate trade_decision, will be used when training rl agent
-            - If `self.track_data` is true, when making data for training, the input `trade_decision` of `execute` will
+            - If `self.track_data` is true, when making data for training, the input `trade_decision` of `execute` will be generated by `collect_data`
-                be generated by `collect_data`
             - Else,  `trade_decision` will not be generated
 
         trade_exchange : Exchange
@@ -114,17 +104,17 @@ class BaseExecutor:
         self.track_data = track_data
         self._trade_exchange = trade_exchange
         self.level_infra = LevelInfrastructure()
-        self.level_infra.reset_infra(common_infra=common_infra, executor=self)
+        self.level_infra.reset_infra(common_infra=common_infra)
         self._settle_type = settle_type
         self.reset(start_time=start_time, end_time=end_time, common_infra=common_infra)
         if common_infra is None:
             get_module_logger("BaseExecutor").warning(f"`common_infra` is not set for {self}")
 
         # record deal order amount in one day
-        self.dealt_order_amount: Dict[str, float] = defaultdict(float)
+        self.dealt_order_amount = defaultdict(float)
         self.deal_day = None
 
-    def reset_common_infra(self, common_infra: CommonInfrastructure, copy_trade_account: bool = False) -> None:
+    def reset_common_infra(self, common_infra, copy_trade_account=False):
         """
         reset infrastructure for trading
             - reset trade_account
@@ -134,18 +124,15 @@ class BaseExecutor:
         else:
             self.common_infra.update(common_infra)
 
-        self.level_infra.reset_infra(common_infra=self.common_infra)
-
         if common_infra.has("trade_account"):
-            # NOTE: there is a trick in the code.
+            if copy_trade_account:
-            # shallow copy is used instead of deepcopy.
+                # NOTE: there is a trick in the code.
-            # 1. So positions are shared
+                # shallow copy is used instead of deepcopy.
-            # 2. Others are not shared, so each level has it own metrics (portfolio and trading metrics)
+                # 1. So positions are shared
-            self.trade_account: Account = (
+                # 2. Others are not shared, so each level has it own metrics (portfolio and trading metrics)
-                copy.copy(common_infra.get("trade_account"))
+                self.trade_account: Account = copy.copy(common_infra.get("trade_account"))
-                if copy_trade_account
+            else:
-                else common_infra.get("trade_account")
+                self.trade_account = common_infra.get("trade_account")
-            )
             self.trade_account.reset(freq=self.time_per_step, port_metr_enabled=self.generate_portfolio_metrics)
 
     @property
@@ -161,7 +148,7 @@ class BaseExecutor:
         """
         return self.level_infra.get("trade_calendar")
 
-    def reset(self, common_infra: CommonInfrastructure = None, **kwargs: Any) -> None:
+    def reset(self, common_infra: CommonInfrastructure = None, **kwargs):
         """
         - reset `start_time` and `end_time`, used in trade calendar
         - reset `common_infra`, used to reset `trade_account`, `trade_exchange`, .etc
@@ -174,13 +161,13 @@ class BaseExecutor:
         if common_infra is not None:
             self.reset_common_infra(common_infra)
 
-    def get_level_infra(self) -> LevelInfrastructure:
+    def get_level_infra(self):
         return self.level_infra
 
-    def finished(self) -> bool:
+    def finished(self):
         return self.trade_calendar.finished()
 
-    def execute(self, trade_decision: BaseTradeDecision, level: int = 0) -> List[object]:
+    def execute(self, trade_decision: BaseTradeDecision, level: int = 0):
         """execute the trade decision and return the executed result
 
         NOTE: this function is never used directly in the framework. Should we delete it?
@@ -197,17 +184,14 @@ class BaseExecutor:
         execute_result : List[object]
             the executed result for trade decision
         """
-        return_value: dict = {}
+        return_value = {}
         for _decision in self.collect_data(trade_decision, return_value=return_value, level=level):
             pass
-        return cast(list, return_value.get("execute_result"))
+        return return_value.get("execute_result")
 
+    @classmethod
     @abstractmethod
-    def _collect_data(
+    def _collect_data(cls, trade_decision: BaseTradeDecision, level: int = 0) -> Tuple[List[object], dict]:
-        self,
-        trade_decision: BaseTradeDecision,
-        level: int = 0,
-    ) -> Union[Generator[Any, Any, Tuple[List[object], dict]], Tuple[List[object], dict]]:
         """
         Please refer to the doc of collect_data
         The only difference between `_collect_data` and `collect_data` is that some common steps are moved into
@@ -225,11 +209,8 @@ class BaseExecutor:
         """
 
     def collect_data(
-        self,
+        self, trade_decision: BaseTradeDecision, return_value: dict = None, level: int = 0
-        trade_decision: BaseTradeDecision,
+    ) -> List[object]:
-        return_value: dict = None,
-        level: int = 0,
-    ) -> Generator[Any, Any, List[object]]:
         """Generator for collecting the trade decision data for rl training
 
         his function will make a step forward
@@ -258,7 +239,6 @@ class BaseExecutor:
         object
             trade decision
         """
-
         if self.track_data:
             yield trade_decision
 
@@ -273,9 +253,7 @@ class BaseExecutor:
         obj = self._collect_data(trade_decision=trade_decision, level=level)
 
         if isinstance(obj, GeneratorType):
-            yield_res = yield from obj
+            res, kwargs = yield from obj
-            assert isinstance(yield_res, tuple) and len(yield_res) == 2
-            res, kwargs = yield_res
         else:
             # Some concrete executor don't have inner decisions
             res, kwargs = obj
@@ -299,10 +277,9 @@ class BaseExecutor:
 
         if return_value is not None:
             return_value.update({"execute_result": res})
-
         return res
 
-    def get_all_executors(self) -> List[BaseExecutor]:
+    def get_all_executors(self):
         """get all executors"""
         return [self]
 
@@ -310,8 +287,7 @@ class BaseExecutor:
 class NestedExecutor(BaseExecutor):
     """
     Nested Executor with inner strategy and executor
-    - At each time `execute` is called, it will call the inner strategy and executor to execute the `trade_decision`
+    - At each time `execute` is called, it will call the inner strategy and executor to execute the `trade_decision` in a higher frequency env.
-        in a higher frequency env.
     """
 
     def __init__(
@@ -328,8 +304,8 @@ class NestedExecutor(BaseExecutor):
         skip_empty_decision: bool = True,
         align_range_limit: bool = True,
         common_infra: CommonInfrastructure = None,
-        **kwargs: Any,
+        **kwargs,
-    ) -> None:
+    ):
         """
         Parameters
         ----------
@@ -347,14 +323,10 @@ class NestedExecutor(BaseExecutor):
             It is only for nested executor, because range_limit is given by outer strategy
         """
         self.inner_executor: BaseExecutor = init_instance_by_config(
-            inner_executor,
+            inner_executor, common_infra=common_infra, accept_types=BaseExecutor
-            common_infra=common_infra,
-            accept_types=BaseExecutor,
         )
         self.inner_strategy: BaseStrategy = init_instance_by_config(
-            inner_strategy,
+            inner_strategy, common_infra=common_infra, accept_types=BaseStrategy
-            common_infra=common_infra,
-            accept_types=BaseStrategy,
         )
 
         self._skip_empty_decision = skip_empty_decision
@@ -372,10 +344,10 @@ class NestedExecutor(BaseExecutor):
             **kwargs,
         )
 
-    def reset_common_infra(self, common_infra: CommonInfrastructure, copy_trade_account: bool = False) -> None:
+    def reset_common_infra(self, common_infra, copy_trade_account=False):
         """
         reset infrastructure for trading
-            - reset inner_strategy and inner_executor common infra
+            - reset inner_strategyand inner_executor common infra
         """
         # NOTE: please refer to the docs of BaseExecutor.reset_common_infra for the meaning of `copy_trade_account`
 
@@ -386,7 +358,7 @@ class NestedExecutor(BaseExecutor):
         self.inner_executor.reset_common_infra(common_infra, copy_trade_account=True)
         self.inner_strategy.reset_common_infra(common_infra)
 
-    def _init_sub_trading(self, trade_decision: BaseTradeDecision) -> None:
+    def _init_sub_trading(self, trade_decision):
         trade_start_time, trade_end_time = self.trade_calendar.get_step_time()
         self.inner_executor.reset(start_time=trade_start_time, end_time=trade_end_time)
         sub_level_infra = self.inner_executor.get_level_infra()
@@ -396,18 +368,14 @@ class NestedExecutor(BaseExecutor):
     def _update_trade_decision(self, trade_decision: BaseTradeDecision) -> BaseTradeDecision:
         # outer strategy have chance to update decision each iterator
         updated_trade_decision = trade_decision.update(self.inner_executor.trade_calendar)
-        if updated_trade_decision is not None:  # TODO: always is None for now?
+        if updated_trade_decision is not None:
             trade_decision = updated_trade_decision
             # NEW UPDATE
             # create a hook for inner strategy to update outer decision
-            trade_decision = self.inner_strategy.alter_outer_trade_decision(trade_decision)
+            self.inner_strategy.alter_outer_trade_decision(trade_decision)
         return trade_decision
 
-    def _collect_data(
+    def _collect_data(self, trade_decision: BaseTradeDecision, level: int = 0):
-        self,
-        trade_decision: BaseTradeDecision,
-        level: int = 0,
-    ) -> Generator[Any, Any, Tuple[List[object], dict]]:
         execute_result = []
         inner_order_indicators = []
         decision_list = []
@@ -422,8 +390,8 @@ class NestedExecutor(BaseExecutor):
 
             if trade_decision.empty() and self._skip_empty_decision:
                 # give one chance for outer strategy to update the strategy
-                # - For updating some information in the sub executor (the strategy have no knowledge of the inner
+                # - For updating some information in the sub executor(the strategy have no knowledge of the inner
-                #   executor when generating the decision)
+                # executor when generating the decision)
                 break
 
             sub_cal: TradeCalendarManager = self.inner_executor.trade_calendar
@@ -437,19 +405,15 @@ class NestedExecutor(BaseExecutor):
 
                 # NOTE: !!!!!
                 # the two lines below is for a special case in RL
-                # To solve the conflicts below
+                # To solve the confliction below
-                # - Normally, user will create a strategy and embed it into Qlib's executor and simulator interaction
+                # - Normally, user will create a strategy and embed it into Qlib's executor and simulator interaction loop
-                #   loop For a _nested qlib example_, (Qlib Strategy) <=> (Qlib Executor[(inner Qlib Strategy) <=>
+                #   For a _nested qlib example_, (Qlib Strategy) <=> (Qlib Executor[(inner Qlib Strategy) <=> (inner Qlib Executor)])
-                #   (inner Qlib Executor)])
                 # - However, RL-based framework has it's own script to run the loop
                 #   For an _RL learning example_, (RL Policy) <=> (RL Env[(inner Qlib Executor)])
-                # To make it possible to run  _nested qlib example_ and _RL learning example_ together, the solution
+                # To make it possible to run  _nested qlib example_ and _RL learning example_ together, the solution below is proposed
-                # below is proposed
+                # - The entry script follow the example of  _RL learning example_ to be compatible with all kinds of RL Framework
-                # - The entry script follow the example of  _RL learning example_ to be compatible with all kinds of
-                #   RL Framework
                 # - Each step of (RL Env) will make (inner Qlib Executor) one step forward
-                #     - (inner Qlib Strategy) is a proxy strategy, it will give the program control right to (RL Env)
+                #     - (inner Qlib Strategy) is a proxy strategy, it will give the program control right to (RL Env) by `yield from` and wait for the action from the policy
-                #       by `yield from` and wait for the action from the policy
                 # So the two lines below is the implementation of yielding control rights
                 if isinstance(res, GeneratorType):
                     res = yield from res
@@ -463,26 +427,21 @@ class NestedExecutor(BaseExecutor):
 
                 # NOTE: Trade Calendar will step forward in the follow line
                 _inner_execute_result = yield from self.inner_executor.collect_data(
-                    trade_decision=_inner_trade_decision,
+                    trade_decision=_inner_trade_decision, level=level + 1
-                    level=level + 1,
                 )
-                assert isinstance(_inner_execute_result, list)
                 self.post_inner_exe_step(_inner_execute_result)
                 execute_result.extend(_inner_execute_result)
 
                 inner_order_indicators.append(
-                    self.inner_executor.trade_account.get_trade_indicator().get_order_indicator(raw=True),
+                    self.inner_executor.trade_account.get_trade_indicator().get_order_indicator(raw=True)
                 )
             else:
                 # do nothing and just step forward
                 sub_cal.step()
 
-        # Let inner strategy know that the outer level execution is done.
-        self.inner_strategy.post_upper_level_exe_step()
-
         return execute_result, {"inner_order_indicators": inner_order_indicators, "decision_list": decision_list}
 
-    def post_inner_exe_step(self, inner_exe_res: List[object]) -> None:
+    def post_inner_exe_step(self, inner_exe_res):
         """
         A hook for doing sth after each step of inner strategy
 
@@ -491,25 +450,12 @@ class NestedExecutor(BaseExecutor):
         inner_exe_res :
             the execution result of inner task
         """
-        self.inner_strategy.post_exe_step(inner_exe_res)
 
-    def get_all_executors(self) -> List[BaseExecutor]:
+    def get_all_executors(self):
         """get all executors, including self and inner_executor.get_all_executors()"""
         return [self, *self.inner_executor.get_all_executors()]
 
 
-def _retrieve_orders_from_decision(trade_decision: BaseTradeDecision) -> List[Order]:
-    """
-    IDE-friendly helper function.
-    """
-    decisions = trade_decision.get_decision()
-    orders: List[Order] = []
-    for decision in decisions:
-        assert isinstance(decision, Order)
-        orders.append(decision)
-    return orders
-
-
 class SimulatorExecutor(BaseExecutor):
     """Executor that simulate the true market"""
 
@@ -518,10 +464,10 @@ class SimulatorExecutor(BaseExecutor):
 
     # available trade_types
     TT_SERIAL = "serial"
-    # The orders will be executed serially in a sequence
+    ## The orders will be executed serially in a sequence
     # In each trading step, it is possible that users sell instruments first and use the money to buy new instruments
     TT_PARAL = "parallel"
-    # The orders will be executed in parallel
+    ## The orders will be executed parallelly
     # In each trading step, if users try to sell instruments first and buy new instruments with money, failure will
     # occur
 
@@ -536,8 +482,8 @@ class SimulatorExecutor(BaseExecutor):
         track_data: bool = False,
         common_infra: CommonInfrastructure = None,
         trade_type: str = TT_SERIAL,
-        **kwargs: Any,
+        **kwargs,
-    ) -> None:
+    ):
         """
         Parameters
         ----------
@@ -571,7 +517,7 @@ class SimulatorExecutor(BaseExecutor):
         List[Order]:
             get a list orders according to `self.trade_type`
         """
-        orders = _retrieve_orders_from_decision(trade_decision)
+        orders = trade_decision.get_decision()
 
         if self.trade_type == self.TT_SERIAL:
             # Orders will be traded in a parallel way
@@ -579,26 +525,29 @@ class SimulatorExecutor(BaseExecutor):
         elif self.trade_type == self.TT_PARAL:
             # NOTE: !!!!!!!
             # Assumption: there will not be orders in different trading direction in a single step of a strategy !!!!
-            # The parallel trading failure will be caused only by the conflicts of money
+            # The parallel trading failure will be caused only by the confliction of money
-            # Therefore, make the buying go first will make sure the conflicts happen.
+            # Therefore, make the buying go first will make sure the confliction happen.
             # It equals to parallel trading after sorting the order by direction
             order_it = sorted(orders, key=lambda order: -order.direction)
         else:
             raise NotImplementedError(f"This type of input is not supported")
         return order_it
 
-    def _collect_data(self, trade_decision: BaseTradeDecision, level: int = 0) -> Tuple[List[object], dict]:
+    def _update_dealt_order_amount(self, order):
+        """update date and dealt order amount in the day."""
+
+        now_deal_day = self.trade_calendar.get_step_time()[0].floor(freq="D")
+        if self.deal_day is None or now_deal_day > self.deal_day:
+            self.dealt_order_amount = defaultdict(float)
+            self.deal_day = now_deal_day
+        self.dealt_order_amount[order.stock_id] += order.deal_amount
+
+    def _collect_data(self, trade_decision: BaseTradeDecision, level: int = 0):
+
         trade_start_time, _ = self.trade_calendar.get_step_time()
-        execute_result: list = []
+        execute_result = []
 
         for order in self._get_order_iterator(trade_decision):
-            # Each time we move into a new date, clear `self.dealt_order_amount` since it only maintains intraday
-            # information.
-            now_deal_day = self.trade_calendar.get_step_time()[0].floor(freq="D")
-            if self.deal_day is None or now_deal_day > self.deal_day:
-                self.dealt_order_amount = defaultdict(float)
-                self.deal_day = now_deal_day
-
             # execute the order.
             # NOTE: The trade_account will be changed in this function
             trade_val, trade_cost, trade_price = self.trade_exchange.deal_order(
@@ -607,13 +556,10 @@ class SimulatorExecutor(BaseExecutor):
                 dealt_order_amount=self.dealt_order_amount,
             )
             execute_result.append((order, trade_val, trade_cost, trade_price))
-
+            self._update_dealt_order_amount(order)
-            self.dealt_order_amount[order.stock_id] += order.deal_amount
-
             if self.verbose:
                 print(
-                    "[I {:%Y-%m-%d %H:%M:%S}]: {} {}, price {:.2f}, amount {}, deal_amount {}, factor {}, "
+                    "[I {:%Y-%m-%d %H:%M:%S}]: {} {}, price {:.2f}, amount {}, deal_amount {}, factor {}, value {:.2f}, cash {:.2f}.".format(
-                    "value {:.2f}, cash {:.2f}.".format(
                         trade_start_time,
                         "sell" if order.direction == Order.SELL else "buy",
                         order.stock_id,
@@ -623,6 +569,6 @@ class SimulatorExecutor(BaseExecutor):
                         order.factor,
                         trade_val,
                         self.trade_account.get_cash(),
-                    ),
+                    )
                 )
         return execute_result, {"trade_info": execute_result}

qlib/backtest/high_performance_ds.py

@@ -1,27 +1,24 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-from __future__ import annotations
+from functools import lru_cache
+import logging
+from typing import List, Text, Union, Callable, Iterable, Dict
+from collections import OrderedDict
 
 import inspect
-import logging
-from collections import OrderedDict
-from functools import lru_cache
-from typing import Any, Callable, Dict, Iterable, List, Optional, Text, Union, cast
-
-import numpy as np
 import pandas as pd
+import numpy as np
 
-import qlib.utils.index_data as idd
-
-from ..log import get_module_logger
 from ..utils.index_data import IndexData, SingleData
 from ..utils.resam import resam_ts_data, ts_data_last
-from ..utils.time import Freq, is_single_value
+from ..log import get_module_logger
+from ..utils.time import is_single_value, Freq
+import qlib.utils.index_data as idd
 
 
 class BaseQuote:
-    def __init__(self, quote_df: pd.DataFrame, freq: str) -> None:
+    def __init__(self, quote_df: pd.DataFrame, freq):
         self.logger = get_module_logger("online operator", level=logging.INFO)
 
     def get_all_stock(self) -> Iterable:
@@ -41,7 +38,7 @@ class BaseQuote:
         start_time: Union[pd.Timestamp, str],
         end_time: Union[pd.Timestamp, str],
         field: Union[str],
-        method: Optional[str] = None,
+        method: Union[str, None] = None,
     ) -> Union[None, int, float, bool, IndexData]:
         """get the specific field of stock data during start time and end_time,
            and apply method to the data.
@@ -101,7 +98,7 @@ class BaseQuote:
 
 
 class PandasQuote(BaseQuote):
-    def __init__(self, quote_df: pd.DataFrame, freq: str) -> None:
+    def __init__(self, quote_df: pd.DataFrame, freq):
         super().__init__(quote_df=quote_df, freq=freq)
         quote_dict = {}
         for stock_id, stock_val in quote_df.groupby(level="instrument"):
@@ -126,7 +123,7 @@ class PandasQuote(BaseQuote):
 
 
 class NumpyQuote(BaseQuote):
-    def __init__(self, quote_df: pd.DataFrame, freq: str, region: str = "cn") -> None:
+    def __init__(self, quote_df: pd.DataFrame, freq, region="cn"):
         """NumpyQuote
 
         Parameters
@@ -180,8 +177,7 @@ class NumpyQuote(BaseQuote):
                 data = self._agg_data(data, method)
             return data
 
-    @staticmethod
+    def _agg_data(self, data: IndexData, method):
-    def _agg_data(data: IndexData, method: str) -> Union[IndexData, np.ndarray, None]:
         """Agg data by specific method."""
         # FIXME: why not call the method of data directly?
         if method == "sum":
@@ -227,31 +223,31 @@ class BaseSingleMetric:
         """
         raise NotImplementedError(f"Please implement the `__init__` method")
 
-    def __add__(self, other: Union[BaseSingleMetric, int, float]) -> BaseSingleMetric:
+    def __add__(self, other: Union["BaseSingleMetric", int, float]) -> "BaseSingleMetric":
         raise NotImplementedError(f"Please implement the `__add__` method")
 
-    def __radd__(self, other: Union[BaseSingleMetric, int, float]) -> BaseSingleMetric:
+    def __radd__(self, other: Union["BaseSingleMetric", int, float]) -> "BaseSingleMetric":
         return self + other
 
-    def __sub__(self, other: Union[BaseSingleMetric, int, float]) -> BaseSingleMetric:
+    def __sub__(self, other: Union["BaseSingleMetric", int, float]) -> "BaseSingleMetric":
         raise NotImplementedError(f"Please implement the `__sub__` method")
 
-    def __rsub__(self, other: Union[BaseSingleMetric, int, float]) -> BaseSingleMetric:
+    def __rsub__(self, other: Union["BaseSingleMetric", int, float]) -> "BaseSingleMetric":
         raise NotImplementedError(f"Please implement the `__rsub__` method")
 
-    def __mul__(self, other: Union[BaseSingleMetric, int, float]) -> BaseSingleMetric:
+    def __mul__(self, other: Union["BaseSingleMetric", int, float]) -> "BaseSingleMetric":
         raise NotImplementedError(f"Please implement the `__mul__` method")
 
-    def __truediv__(self, other: Union[BaseSingleMetric, int, float]) -> BaseSingleMetric:
+    def __truediv__(self, other: Union["BaseSingleMetric", int, float]) -> "BaseSingleMetric":
         raise NotImplementedError(f"Please implement the `__truediv__` method")
 
-    def __eq__(self, other: object) -> BaseSingleMetric:
+    def __eq__(self, other: Union["BaseSingleMetric", int, float]) -> "BaseSingleMetric":
         raise NotImplementedError(f"Please implement the `__eq__` method")
 
-    def __gt__(self, other: Union[BaseSingleMetric, int, float]) -> BaseSingleMetric:
+    def __gt__(self, other: Union["BaseSingleMetric", int, float]) -> "BaseSingleMetric":
         raise NotImplementedError(f"Please implement the `__gt__` method")
 
-    def __lt__(self, other: Union[BaseSingleMetric, int, float]) -> BaseSingleMetric:
+    def __lt__(self, other: Union["BaseSingleMetric", int, float]) -> "BaseSingleMetric":
         raise NotImplementedError(f"Please implement the `__lt__` method")
 
     def __len__(self) -> int:
@@ -268,7 +264,7 @@ class BaseSingleMetric:
 
         raise NotImplementedError(f"Please implement the `count` method")
 
-    def abs(self) -> BaseSingleMetric:
+    def abs(self) -> "BaseSingleMetric":
         raise NotImplementedError(f"Please implement the `abs` method")
 
     @property
@@ -277,18 +273,18 @@ class BaseSingleMetric:
 
         raise NotImplementedError(f"Please implement the `empty` method")
 
-    def add(self, other: BaseSingleMetric, fill_value: float = None) -> BaseSingleMetric:
+    def add(self, other: "BaseSingleMetric", fill_value: float = None) -> "BaseSingleMetric":
         """Replace np.NaN with fill_value in two metrics and add them."""
 
         raise NotImplementedError(f"Please implement the `add` method")
 
-    def replace(self, replace_dict: dict) -> BaseSingleMetric:
+    def replace(self, replace_dict: dict) -> "BaseSingleMetric":
         """Replace the value of metric according to replace_dict."""
 
         raise NotImplementedError(f"Please implement the `replace` method")
 
-    def apply(self, func: Callable) -> BaseSingleMetric:
+    def apply(self, func: dict) -> "BaseSingleMetric":
-        """Replace the value of metric with func (metric).
+        """Replace the value of metric with func(metric).
         Currently, the func is only qlib/backtest/order/Order.parse_dir.
         """
 
@@ -307,11 +303,11 @@ class BaseOrderIndicator:
         to inherit the BaseSingleMetric.
     """
 
-    def __init__(self):
+    def __init__(self, data):
-        self.data = {}  # will be created in the subclass
+        self.data = data
         self.logger = get_module_logger("online operator")
 
-    def assign(self, col: str, metric: Union[dict, pd.Series]) -> None:
+    def assign(self, col: str, metric: Union[dict, pd.Series]):
         """assign one metric.
 
         Parameters
@@ -331,7 +327,7 @@ class BaseOrderIndicator:
 
         raise NotImplementedError(f"Please implement the 'assign' method")
 
-    def transfer(self, func: Callable, new_col: str = None) -> Optional[BaseSingleMetric]:
+    def transfer(self, func: Callable, new_col: str = None) -> Union[None, BaseSingleMetric]:
         """compute new metric with existing metrics.
 
         Parameters
@@ -355,7 +351,6 @@ class BaseOrderIndicator:
         tmp_metric = func(**func_kwargs)
         if new_col is not None:
             self.data[new_col] = tmp_metric
-            return None
         else:
             return tmp_metric
 
@@ -376,7 +371,7 @@ class BaseOrderIndicator:
 
         raise NotImplementedError(f"Please implement the 'get_metric_series' method")
 
-    def get_index_data(self, metric: str) -> SingleData:
+    def get_index_data(self, metric) -> SingleData:
         """get one metric with the format of SingleData
 
         Parameters
@@ -393,12 +388,7 @@ class BaseOrderIndicator:
         raise NotImplementedError(f"Please implement the 'get_index_data' method")
 
     @staticmethod
-    def sum_all_indicators(
+    def sum_all_indicators(order_indicator, indicators: list, metrics: Union[str, List[str]], fill_value: float = None):
-        order_indicator: BaseOrderIndicator,
-        indicators: List[BaseOrderIndicator],
-        metrics: Union[str, List[str]],
-        fill_value: float = 0,
-    ) -> None:
         """sum indicators with the same metrics.
         and assign to the order_indicator(BaseOrderIndicator).
         NOTE: indicators could be a empty list when orders in lower level all fail.
@@ -536,17 +526,16 @@ class PandasSingleMetric(SingleMetric):
     def index(self):
         return list(self.metric.index)
 
-    def add(self, other: BaseSingleMetric, fill_value: float = None) -> PandasSingleMetric:
+    def add(self, other, fill_value=None):
-        other = cast(PandasSingleMetric, other)
         return self.__class__(self.metric.add(other.metric, fill_value=fill_value))
 
-    def replace(self, replace_dict: dict) -> PandasSingleMetric:
+    def replace(self, replace_dict: dict):
         return self.__class__(self.metric.replace(replace_dict))
 
-    def apply(self, func: Callable) -> PandasSingleMetric:
+    def apply(self, func: Callable):
         return self.__class__(self.metric.apply(func))
 
-    def reindex(self, index: Any, fill_value: float) -> PandasSingleMetric:
+    def reindex(self, index, fill_value):
         return self.__class__(self.metric.reindex(index, fill_value=fill_value))
 
     def __repr__(self):
@@ -560,14 +549,13 @@ class PandasOrderIndicator(BaseOrderIndicator):
     Str is the name of metric.
     """
 
-    def __init__(self) -> None:
+    def __init__(self):
-        super(PandasOrderIndicator, self).__init__()
         self.data: Dict[str, PandasSingleMetric] = OrderedDict()
 
-    def assign(self, col: str, metric: Union[dict, pd.Series]) -> None:
+    def assign(self, col: str, metric: Union[dict, pd.Series]):
         self.data[col] = PandasSingleMetric(metric)
 
-    def get_index_data(self, metric: str) -> SingleData:
+    def get_index_data(self, metric):
         if metric in self.data:
             return idd.SingleData(self.data[metric].metric)
         else:
@@ -583,12 +571,7 @@ class PandasOrderIndicator(BaseOrderIndicator):
         return {k: v.metric for k, v in self.data.items()}
 
     @staticmethod
-    def sum_all_indicators(
+    def sum_all_indicators(order_indicator, indicators: list, metrics: Union[str, List[str]], fill_value=0):
-        order_indicator: BaseOrderIndicator,
-        indicators: List[BaseOrderIndicator],
-        metrics: Union[str, List[str]],
-        fill_value: float = 0,
-    ) -> None:
         if isinstance(metrics, str):
             metrics = [metrics]
         for metric in metrics:
@@ -608,14 +591,13 @@ class NumpyOrderIndicator(BaseOrderIndicator):
     Str is the name of metric.
     """
 
-    def __init__(self) -> None:
+    def __init__(self):
-        super(NumpyOrderIndicator, self).__init__()
         self.data: Dict[str, SingleData] = OrderedDict()
 
-    def assign(self, col: str, metric: dict) -> None:
+    def assign(self, col: str, metric: dict):
         self.data[col] = idd.SingleData(metric)
 
-    def get_index_data(self, metric: str) -> SingleData:
+    def get_index_data(self, metric):
         if metric in self.data:
             return self.data[metric]
         else:
@@ -631,27 +613,21 @@ class NumpyOrderIndicator(BaseOrderIndicator):
         return tmp_metric_dict
 
     @staticmethod
-    def sum_all_indicators(
+    def sum_all_indicators(order_indicator, indicators: list, metrics: Union[str, List[str]], fill_value=0):
-        order_indicator: BaseOrderIndicator,
-        indicators: List[BaseOrderIndicator],
-        metrics: Union[str, List[str]],
-        fill_value: float = 0,
-    ) -> None:
         # get all index(stock_id)
-        stock_set: set = set()
+        stocks = set()
         for indicator in indicators:
             # set(np.ndarray.tolist()) is faster than set(np.ndarray)
-            stock_set = stock_set | set(indicator.data[metrics[0]].index.tolist())
+            stocks = stocks | set(indicator.data[metrics[0]].index.tolist())
-        stocks = sorted(list(stock_set))
+        stocks = list(stocks)
+        stocks.sort()
 
         # add metric by index
         if isinstance(metrics, str):
             metrics = [metrics]
         for metric in metrics:
             order_indicator.data[metric] = idd.sum_by_index(
-                [indicator.data[metric] for indicator in indicators],
+                [indicator.data[metric] for indicator in indicators], stocks, fill_value
-                stocks,
-                fill_value,
             )
 
     def __repr__(self):

qlib/backtest/position.py

@@ -2,28 +2,24 @@
 # Licensed under the MIT License.
 
 
-from datetime import timedelta
+from typing import Dict, List, Union
-from typing import Any, Dict, List, Union
 
-import numpy as np
 import pandas as pd
+from datetime import timedelta
+import numpy as np
 
-from ..data.data import D
 from .decision import Order
+from ..data.data import D
 
 
 class BasePosition:
     """
-    The Position wants to maintain the position like a dictionary
+    The Position want to maintain the position like a dictionary
     Please refer to the `Position` class for the position
     """
 
-    def __init__(self, *args: Any, cash: float = 0.0, **kwargs: Any) -> None:
+    def __init__(self, *args, cash=0.0, **kwargs):
         self._settle_type = self.ST_NO
-        self.position: dict = {}
-
-    def fill_stock_value(self, start_time: Union[str, pd.Timestamp], freq: str, last_days: int = 30) -> None:
-        pass
 
     def skip_update(self) -> bool:
         """
@@ -53,7 +49,7 @@ class BasePosition:
         """
         raise NotImplementedError(f"Please implement the `check_stock` method")
 
-    def update_order(self, order: Order, trade_val: float, cost: float, trade_price: float) -> None:
+    def update_order(self, order: Order, trade_val: float, cost: float, trade_price: float):
         """
         Parameters
         ----------
@@ -68,7 +64,7 @@ class BasePosition:
         """
         raise NotImplementedError(f"Please implement the `update_order` method")
 
-    def update_stock_price(self, stock_id: str, price: float) -> None:
+    def update_stock_price(self, stock_id, price: float):
         """
         Updating the latest price of the order
         The useful when clearing balance at each bar end
@@ -93,16 +89,13 @@ class BasePosition:
         """
         raise NotImplementedError(f"Please implement the `calculate_stock_value` method")
 
-    def calculate_value(self) -> float:
+    def get_stock_list(self) -> List:
-        raise NotImplementedError(f"Please implement the `calculate_value` method")
-
-    def get_stock_list(self) -> List[str]:
         """
         Get the list of stocks in the position.
         """
         raise NotImplementedError(f"Please implement the `get_stock_list` method")
 
-    def get_stock_price(self, code: str) -> float:
+    def get_stock_price(self, code) -> float:
         """
         get the latest price of the stock
 
@@ -113,7 +106,7 @@ class BasePosition:
         """
         raise NotImplementedError(f"Please implement the `get_stock_price` method")
 
-    def get_stock_amount(self, code: str) -> float:
+    def get_stock_amount(self, code) -> float:
         """
         get the amount of the stock
 
@@ -131,20 +124,18 @@ class BasePosition:
 
     def get_cash(self, include_settle: bool = False) -> float:
         """
-        Parameters
-        ----------
-        include_settle:
-            will the unsettled(delayed) cash included
-            Default: not include those unavailable cash
 
         Returns
         -------
         float:
             the available(tradable) cash in position
+        include_settle:
+            will the unsettled(delayed) cash included
+            Default: not include those unavailable cash
         """
         raise NotImplementedError(f"Please implement the `get_cash` method")
 
-    def get_stock_amount_dict(self) -> dict:
+    def get_stock_amount_dict(self) -> Dict:
         """
         generate stock amount dict {stock_id : amount of stock}
 
@@ -155,7 +146,7 @@ class BasePosition:
         """
         raise NotImplementedError(f"Please implement the `get_stock_amount_dict` method")
 
-    def get_stock_weight_dict(self, only_stock: bool = False) -> dict:
+    def get_stock_weight_dict(self, only_stock: bool = False) -> Dict:
         """
         generate stock weight dict {stock_id : value weight of stock in the position}
         it is meaningful in the beginning or the end of each trade step
@@ -174,7 +165,7 @@ class BasePosition:
         """
         raise NotImplementedError(f"Please implement the `get_stock_weight_dict` method")
 
-    def add_count_all(self, bar: str) -> None:
+    def add_count_all(self, bar):
         """
         Will be called at the end of each bar on each level
 
@@ -185,19 +176,24 @@ class BasePosition:
         """
         raise NotImplementedError(f"Please implement the `add_count_all` method")
 
-    def update_weight_all(self) -> None:
+    def update_weight_all(self):
         """
         Updating the position weight;
 
         # TODO: this function is a little weird. The weight data in the position is in a wrong state after dealing order
         # and before updating weight.
+
+        Parameters
+        ----------
+        bar :
+            The level to be updated
         """
         raise NotImplementedError(f"Please implement the `add_count_all` method")
 
     ST_CASH = "cash"
-    ST_NO = "None"  # String is more typehint friendly than None
+    ST_NO = None
 
-    def settle_start(self, settle_type: str) -> None:
+    def settle_start(self, settle_type: str):
         """
         settlement start
         It will act like start and commit a transaction
@@ -214,16 +210,21 @@ class BasePosition:
         """
         raise NotImplementedError(f"Please implement the `settle_conf` method")
 
-    def settle_commit(self) -> None:
+    def settle_commit(self):
         """
         settlement commit
+
+        Parameters
+        ----------
+        settle_type : str
+            please refer to the documents of Executor
         """
         raise NotImplementedError(f"Please implement the `settle_commit` method")
 
-    def __str__(self) -> str:
+    def __str__(self):
         return self.__dict__.__str__()
 
-    def __repr__(self) -> str:
+    def __repr__(self):
         return self.__dict__.__repr__()
 
 
@@ -241,11 +242,13 @@ class Position(BasePosition):
     }
     """
 
-    def __init__(self, cash: float = 0, position_dict: Dict[str, Union[Dict[str, float], float]] = {}) -> None:
+    def __init__(self, cash: float = 0, position_dict: Dict[str, Dict[str, float]] = {}):
         """Init position by cash and position_dict.
 
         Parameters
         ----------
+        start_time :
+            the start time of backtest. It's for filling the initial value of stocks.
         cash : float, optional
             initial cash in account, by default 0
         position_dict : Dict[
@@ -265,9 +268,9 @@ class Position(BasePosition):
         # Otherwise the initial value
         self.init_cash = cash
         self.position = position_dict.copy()
-        for stock, value in self.position.items():
+        for stock in self.position:
-            if isinstance(value, int):
+            if isinstance(self.position[stock], int):
-                self.position[stock] = {"amount": value}
+                self.position[stock] = {"amount": self.position[stock]}
         self.position["cash"] = cash
 
         # If the stock price information is missing, the account value will not be calculated temporarily
@@ -276,23 +279,21 @@ class Position(BasePosition):
         except KeyError:
             pass
 
-    def fill_stock_value(self, start_time: Union[str, pd.Timestamp], freq: str, last_days: int = 30) -> None:
+    def fill_stock_value(self, start_time: Union[str, pd.Timestamp], freq: str, last_days: int = 30):
         """fill the stock value by the close price of latest last_days from qlib.
 
         Parameters
         ----------
         start_time :
             the start time of backtest.
-        freq : str
-            Frequency
         last_days : int, optional
             the days to get the latest close price, by default 30.
         """
         stock_list = []
-        for stock, value in self.position.items():
+        for stock in self.position:
-            if not isinstance(value, dict):
+            if not isinstance(self.position[stock], dict):
                 continue
-            if value.get("price", None) is None:
+            if ("price" not in self.position[stock]) or (self.position[stock]["price"] is None):
                 stock_list.append(stock)
 
         if len(stock_list) == 0:
@@ -303,12 +304,7 @@ class Position(BasePosition):
         price_end_time = start_time
         price_start_time = start_time - timedelta(days=last_days)
         price_df = D.features(
-            stock_list,
+            stock_list, ["$close"], price_start_time, price_end_time, freq=freq, disk_cache=True
-            ["$close"],
-            price_start_time,
-            price_end_time,
-            freq=freq,
-            disk_cache=True,
         ).dropna()
         price_dict = price_df.groupby(["instrument"]).tail(1).reset_index(level=1, drop=True)["$close"].to_dict()
 
@@ -320,7 +316,7 @@ class Position(BasePosition):
             self.position[stock]["price"] = price_dict[stock]
         self.position["now_account_value"] = self.calculate_value()
 
-    def _init_stock(self, stock_id: str, amount: float, price: float = None) -> None:
+    def _init_stock(self, stock_id, amount, price=None):
         """
         initialization the stock in current position
 
@@ -338,7 +334,7 @@ class Position(BasePosition):
         self.position[stock_id]["price"] = price
         self.position[stock_id]["weight"] = 0  # update the weight in the end of the trade date
 
-    def _buy_stock(self, stock_id: str, trade_val: float, cost: float, trade_price: float) -> None:
+    def _buy_stock(self, stock_id, trade_val, cost, trade_price):
         trade_amount = trade_val / trade_price
         if stock_id not in self.position:
             self._init_stock(stock_id=stock_id, amount=trade_amount, price=trade_price)
@@ -348,16 +344,15 @@ class Position(BasePosition):
 
         self.position["cash"] -= trade_val + cost
 
-    def _sell_stock(self, stock_id: str, trade_val: float, cost: float, trade_price: float) -> None:
+    def _sell_stock(self, stock_id, trade_val, cost, trade_price):
         trade_amount = trade_val / trade_price
         if stock_id not in self.position:
             raise KeyError("{} not in current position".format(stock_id))
         else:
             if np.isclose(self.position[stock_id]["amount"], trade_amount):
                 # Selling all the stocks
-                # we use np.isclose instead of abs(<the final amount>) <= 1e-5  because `np.isclose` consider both
+                # we use np.isclose instead of abs(<the final amount>) <= 1e-5  because `np.isclose` consider both ralative amount and absolute amount
-                # relative amount and absolute amount
+                # Using  abs(<the final amount>) <= 1e-5 will result in error when the amount is large
-                # Using abs(<the final amount>) <= 1e-5 will result in error when the amount is large
                 self._del_stock(stock_id)
             else:
                 # decrease the amount of stock
@@ -366,10 +361,8 @@ class Position(BasePosition):
                 if self.position[stock_id]["amount"] < -1e-5:
                     raise ValueError(
                         "only have {} {}, require {}".format(
-                            self.position[stock_id]["amount"] + trade_amount,
+                            self.position[stock_id]["amount"] + trade_amount, stock_id, trade_amount
-                            stock_id,
+                        )
-                            trade_amount,
-                        ),
                     )
 
         new_cash = trade_val - cost
@@ -380,13 +373,13 @@ class Position(BasePosition):
         else:
             raise NotImplementedError(f"This type of input is not supported")
 
-    def _del_stock(self, stock_id: str) -> None:
+    def _del_stock(self, stock_id):
         del self.position[stock_id]
 
-    def check_stock(self, stock_id: str) -> bool:
+    def check_stock(self, stock_id):
         return stock_id in self.position
 
-    def update_order(self, order: Order, trade_val: float, cost: float, trade_price: float) -> None:
+    def update_order(self, order, trade_val, cost, trade_price):
         # handle order, order is a order class, defined in exchange.py
         if order.direction == Order.BUY:
             # BUY
@@ -397,54 +390,54 @@ class Position(BasePosition):
         else:
             raise NotImplementedError("do not support order direction {}".format(order.direction))
 
-    def update_stock_price(self, stock_id: str, price: float) -> None:
+    def update_stock_price(self, stock_id, price):
         self.position[stock_id]["price"] = price
 
-    def update_stock_count(self, stock_id: str, bar: str, count: float) -> None:  # TODO: check type of `bar`
+    def update_stock_count(self, stock_id, bar, count):
         self.position[stock_id][f"count_{bar}"] = count
 
-    def update_stock_weight(self, stock_id: str, weight: float) -> None:
+    def update_stock_weight(self, stock_id, weight):
         self.position[stock_id]["weight"] = weight
 
-    def calculate_stock_value(self) -> float:
+    def calculate_stock_value(self):
         stock_list = self.get_stock_list()
         value = 0
         for stock_id in stock_list:
             value += self.position[stock_id]["amount"] * self.position[stock_id]["price"]
         return value
 
-    def calculate_value(self) -> float:
+    def calculate_value(self):
         value = self.calculate_stock_value()
         value += self.position["cash"] + self.position.get("cash_delay", 0.0)
         return value
 
-    def get_stock_list(self) -> List[str]:
+    def get_stock_list(self):
         stock_list = list(set(self.position.keys()) - {"cash", "now_account_value", "cash_delay"})
         return stock_list
 
-    def get_stock_price(self, code: str) -> float:
+    def get_stock_price(self, code):
         return self.position[code]["price"]
 
-    def get_stock_amount(self, code: str) -> float:
+    def get_stock_amount(self, code):
         return self.position[code]["amount"] if code in self.position else 0
 
-    def get_stock_count(self, code: str, bar: str) -> float:
+    def get_stock_count(self, code, bar):
         """the days the account has been hold, it may be used in some special strategies"""
         if f"count_{bar}" in self.position[code]:
             return self.position[code][f"count_{bar}"]
         else:
             return 0
 
-    def get_stock_weight(self, code: str) -> float:
+    def get_stock_weight(self, code):
         return self.position[code]["weight"]
 
-    def get_cash(self, include_settle: bool = False) -> float:
+    def get_cash(self, include_settle=False):
         cash = self.position["cash"]
         if include_settle:
             cash += self.position.get("cash_delay", 0.0)
         return cash
 
-    def get_stock_amount_dict(self) -> dict:
+    def get_stock_amount_dict(self):
         """generate stock amount dict {stock_id : amount of stock}"""
         d = {}
         stock_list = self.get_stock_list()
@@ -452,7 +445,7 @@ class Position(BasePosition):
             d[stock_code] = self.get_stock_amount(code=stock_code)
         return d
 
-    def get_stock_weight_dict(self, only_stock: bool = False) -> dict:
+    def get_stock_weight_dict(self, only_stock=False):
         """get_stock_weight_dict
         generate stock weight dict {stock_id : value weight of stock in the position}
         it is meaningful in the beginning or the end of each trade date
@@ -470,7 +463,7 @@ class Position(BasePosition):
             d[stock_code] = self.position[stock_code]["amount"] * self.position[stock_code]["price"] / position_value
         return d
 
-    def add_count_all(self, bar: str) -> None:
+    def add_count_all(self, bar):
         stock_list = self.get_stock_list()
         for code in stock_list:
             if f"count_{bar}" in self.position[code]:
@@ -478,18 +471,18 @@ class Position(BasePosition):
             else:
                 self.position[code][f"count_{bar}"] = 1
 
-    def update_weight_all(self) -> None:
+    def update_weight_all(self):
         weight_dict = self.get_stock_weight_dict()
         for stock_code, weight in weight_dict.items():
             self.update_stock_weight(stock_code, weight)
 
-    def settle_start(self, settle_type: str) -> None:
+    def settle_start(self, settle_type):
         assert self._settle_type == self.ST_NO, "Currently, settlement can't be nested!!!!!"
         self._settle_type = settle_type
         if settle_type == self.ST_CASH:
             self.position["cash_delay"] = 0.0
 
-    def settle_commit(self) -> None:
+    def settle_commit(self):
         if self._settle_type != self.ST_NO:
             if self._settle_type == self.ST_CASH:
                 self.position["cash"] += self.position["cash_delay"]
@@ -514,10 +507,10 @@ class InfPosition(BasePosition):
         # InfPosition always have any stocks
         return True
 
-    def update_order(self, order: Order, trade_val: float, cost: float, trade_price: float) -> None:
+    def update_order(self, order: Order, trade_val: float, cost: float, trade_price: float):
         pass
 
-    def update_stock_price(self, stock_id: str, price: float) -> None:
+    def update_stock_price(self, stock_id, price: float):
         pass
 
     def calculate_stock_value(self) -> float:
@@ -529,36 +522,33 @@ class InfPosition(BasePosition):
         """
         return np.inf
 
-    def calculate_value(self) -> float:
+    def get_stock_list(self) -> List:
-        raise NotImplementedError(f"InfPosition doesn't support calculating value")
-
-    def get_stock_list(self) -> List[str]:
         raise NotImplementedError(f"InfPosition doesn't support stock list position")
 
-    def get_stock_price(self, code: str) -> float:
+    def get_stock_price(self, code) -> float:
         """the price of the inf position is meaningless"""
         return np.nan
 
-    def get_stock_amount(self, code: str) -> float:
+    def get_stock_amount(self, code) -> float:
         return np.inf
 
-    def get_cash(self, include_settle: bool = False) -> float:
+    def get_cash(self, include_settle=False) -> float:
         return np.inf
 
-    def get_stock_amount_dict(self) -> dict:
+    def get_stock_amount_dict(self) -> Dict:
         raise NotImplementedError(f"InfPosition doesn't support get_stock_amount_dict")
 
-    def get_stock_weight_dict(self, only_stock: bool = False) -> dict:
+    def get_stock_weight_dict(self, only_stock: bool = False) -> Dict:
         raise NotImplementedError(f"InfPosition doesn't support get_stock_weight_dict")
 
-    def add_count_all(self, bar: str) -> None:
+    def add_count_all(self, bar):
         raise NotImplementedError(f"InfPosition doesn't support add_count_all")
 
-    def update_weight_all(self) -> None:
+    def update_weight_all(self):
         raise NotImplementedError(f"InfPosition doesn't support update_weight_all")
 
-    def settle_start(self, settle_type: str) -> None:
+    def settle_start(self, settle_type: str):
         pass
 
-    def settle_commit(self) -> None:
+    def settle_commit(self):
         pass

qlib/backtest/profit_attribution.py

@@ -4,15 +4,13 @@
 This module is not well maintained.
 """
 
-import datetime
-from pathlib import Path
-
 import numpy as np
 import pandas as pd
-
-from ..config import C
-from ..data import D
 from .position import Position
+from ..data import D
+from ..config import C
+import datetime
+from pathlib import Path
 
 
 def get_benchmark_weight(
@@ -216,9 +214,7 @@ def get_stock_group(stock_group_field_df, bench_stock_weight_df, group_method, g
         for idx, row in (~bench_stock_weight_df.isna()).iterrows():
             bench_values = stock_group_field_df.loc[idx, row[row].index]
             new_stock_group_df.loc[idx] = get_daily_bin_group(
-                bench_values,
+                bench_values, stock_group_field_df.loc[idx], group_n=group_n
-                stock_group_field_df.loc[idx],
-                group_n=group_n,
             )
         return new_stock_group_df
 
@@ -319,7 +315,7 @@ def brinson_pa(
                 # The excess profit from the interaction of assets allocation and stocks selection
                 "RIN": Q4 - Q3 - Q2 + Q1,
                 "RTotal": Q4 - Q1,  # The totoal excess profit
-            },
+            }
         ),
         {
             "port_group_ret": port_group_ret_df,

qlib/backtest/report.py

@@ -2,20 +2,19 @@
 # Licensed under the MIT License.
 
 
-import pathlib
 from collections import OrderedDict
-from typing import Any, Dict, List, Optional, Text, Tuple, Type, Union, cast
+import pathlib
+from typing import Dict, List, Tuple, Union
 
 import numpy as np
 import pandas as pd
 
-import qlib.utils.index_data as idd
-from qlib.backtest.decision import BaseTradeDecision, Order, OrderDir
 from qlib.backtest.exchange import Exchange
-
+from qlib.backtest.decision import BaseTradeDecision, Order, OrderDir
+from .high_performance_ds import BaseOrderIndicator, NumpyOrderIndicator, SingleMetric
 from ..tests.config import CSI300_BENCH
 from ..utils.resam import get_higher_eq_freq_feature, resam_ts_data
-from .high_performance_ds import BaseOrderIndicator, BaseSingleMetric, NumpyOrderIndicator
+import qlib.utils.index_data as idd
 
 
 class PortfolioMetrics:
@@ -38,7 +37,7 @@ class PortfolioMetrics:
         update report
     """
 
-    def __init__(self, freq: str = "day", benchmark_config: dict = {}) -> None:
+    def __init__(self, freq: str = "day", benchmark_config: dict = {}):
         """
         Parameters
         ----------
@@ -49,17 +48,13 @@ class PortfolioMetrics:
             - benchmark : Union[str, list, pd.Series]
                 - If `benchmark` is pd.Series, `index` is trading date; the value T is the change from T-1 to T.
                     example:
-                        print(
+                        print(D.features(D.instruments('csi500'), ['$close/Ref($close, 1)-1'])['$close/Ref($close, 1)-1'].head())
-                            D.features(D.instruments('csi500'),
-                            ['$close/Ref($close, 1)-1'])['$close/Ref($close, 1)-1'].head()
-                        )
                             2017-01-04    0.011693
                             2017-01-05    0.000721
                             2017-01-06   -0.004322
                             2017-01-09    0.006874
                             2017-01-10   -0.003350
-                - If `benchmark` is list, will use the daily average change of the stock pool in the list as the
+                - If `benchmark` is list, will use the daily average change of the stock pool in the list as the 'bench'.
-                    'bench'.
                 - If `benchmark` is str, will use the daily change as the 'bench'.
                 benchmark code, default is SH000300 CSI300
             - start_time : Union[str, pd.Timestamp], optional
@@ -74,26 +69,25 @@ class PortfolioMetrics:
         self.init_vars()
         self.init_bench(freq=freq, benchmark_config=benchmark_config)
 
-    def init_vars(self) -> None:
+    def init_vars(self):
-        self.accounts: dict = OrderedDict()  # account position value for each trade time
+        self.accounts = OrderedDict()  # account position value for each trade time
-        self.returns: dict = OrderedDict()  # daily return rate for each trade time
+        self.returns = OrderedDict()  # daily return rate for each trade time
-        self.total_turnovers: dict = OrderedDict()  # total turnover for each trade time
+        self.total_turnovers = OrderedDict()  # total turnover for each trade time
-        self.turnovers: dict = OrderedDict()  # turnover for each trade time
+        self.turnovers = OrderedDict()  # turnover for each trade time
-        self.total_costs: dict = OrderedDict()  # total trade cost for each trade time
+        self.total_costs = OrderedDict()  # total trade cost for each trade time
-        self.costs: dict = OrderedDict()  # trade cost rate for each trade time
+        self.costs = OrderedDict()  # trade cost rate for each trade time
-        self.values: dict = OrderedDict()  # value for each trade time
+        self.values = OrderedDict()  # value for each trade time
-        self.cashes: dict = OrderedDict()
+        self.cashes = OrderedDict()
-        self.benches: dict = OrderedDict()
+        self.benches = OrderedDict()
-        self.latest_pm_time: Optional[pd.TimeStamp] = None
+        self.latest_pm_time = None  # pd.TimeStamp
 
-    def init_bench(self, freq: str = None, benchmark_config: dict = None) -> None:
+    def init_bench(self, freq=None, benchmark_config=None):
         if freq is not None:
             self.freq = freq
         self.benchmark_config = benchmark_config
         self.bench = self._cal_benchmark(self.benchmark_config, self.freq)
 
-    @staticmethod
+    def _cal_benchmark(self, benchmark_config, freq):
-    def _cal_benchmark(benchmark_config: Optional[dict], freq: str) -> Optional[pd.Series]:
         if benchmark_config is None:
             return None
         benchmark = benchmark_config.get("benchmark", CSI300_BENCH)
@@ -115,12 +109,7 @@ class PortfolioMetrics:
                 raise ValueError(f"The benchmark {_codes} does not exist. Please provide the right benchmark")
             return _temp_result.groupby(level="datetime")[_temp_result.columns.tolist()[0]].mean().fillna(0)
 
-    def _sample_benchmark(
+    def _sample_benchmark(self, bench, trade_start_time, trade_end_time):
-        self,
-        bench: pd.Series,
-        trade_start_time: Union[str, pd.Timestamp],
-        trade_end_time: Union[str, pd.Timestamp],
-    ) -> Optional[float]:
         if self.bench is None:
             return None
 
@@ -130,35 +119,35 @@ class PortfolioMetrics:
         _ret = resam_ts_data(bench, trade_start_time, trade_end_time, method=cal_change)
         return 0.0 if _ret is None else _ret - 1
 
-    def is_empty(self) -> bool:
+    def is_empty(self):
         return len(self.accounts) == 0
 
-    def get_latest_date(self) -> pd.Timestamp:
+    def get_latest_date(self):
         return self.latest_pm_time
 
-    def get_latest_account_value(self) -> float:
+    def get_latest_account_value(self):
         return self.accounts[self.latest_pm_time]
 
-    def get_latest_total_cost(self) -> Any:
+    def get_latest_total_cost(self):
         return self.total_costs[self.latest_pm_time]
 
-    def get_latest_total_turnover(self) -> Any:
+    def get_latest_total_turnover(self):
         return self.total_turnovers[self.latest_pm_time]
 
     def update_portfolio_metrics_record(
         self,
-        trade_start_time: Union[str, pd.Timestamp] = None,
+        trade_start_time=None,
-        trade_end_time: Union[str, pd.Timestamp] = None,
+        trade_end_time=None,
-        account_value: float = None,
+        account_value=None,
-        cash: float = None,
+        cash=None,
-        return_rate: float = None,
+        return_rate=None,
-        total_turnover: float = None,
+        total_turnover=None,
-        turnover_rate: float = None,
+        turnover_rate=None,
-        total_cost: float = None,
+        total_cost=None,
-        cost_rate: float = None,
+        cost_rate=None,
-        stock_value: float = None,
+        stock_value=None,
-        bench_value: float = None,
+        bench_value=None,
-    ) -> None:
+    ):
         # check data
         if None in [
             trade_start_time,
@@ -172,8 +161,7 @@ class PortfolioMetrics:
             stock_value,
         ]:
             raise ValueError(
-                "None in [trade_start_time, account_value, cash, return_rate, total_turnover, turnover_rate, "
+                "None in [trade_start_time, account_value, cash, return_rate, total_turnover, turnover_rate, total_cost, cost_rate, stock_value]"
-                "total_cost, cost_rate, stock_value]",
             )
 
         if trade_end_time is None and bench_value is None:
@@ -195,7 +183,7 @@ class PortfolioMetrics:
         self.latest_pm_time = trade_start_time
         # finish pm update in each step
 
-    def generate_portfolio_metrics_dataframe(self) -> pd.DataFrame:
+    def generate_portfolio_metrics_dataframe(self):
         pm = pd.DataFrame()
         pm["account"] = pd.Series(self.accounts)
         pm["return"] = pd.Series(self.returns)
@@ -209,18 +197,19 @@ class PortfolioMetrics:
         pm.index.name = "datetime"
         return pm
 
-    def save_portfolio_metrics(self, path: str) -> None:
+    def save_portfolio_metrics(self, path):
         r = self.generate_portfolio_metrics_dataframe()
         r.to_csv(path)
 
-    def load_portfolio_metrics(self, path: str) -> None:
+    def load_portfolio_metrics(self, path):
         """load pm from a file
         should have format like
         columns = ['account', 'return', 'total_turnover', 'turnover', 'cost', 'total_cost', 'value', 'cash', 'bench']
             :param
                 path: str/ pathlib.Path()
         """
-        with pathlib.Path(path).open("rb") as f:
+        path = pathlib.Path(path)
+        with path.open("rb") as f:
             r = pd.read_csv(f, index_col=0)
         r.index = pd.DatetimeIndex(r.index)
 
@@ -270,30 +259,30 @@ class Indicator:
 
     """
 
-    def __init__(self, order_indicator_cls: Type[BaseOrderIndicator] = NumpyOrderIndicator) -> None:
+    def __init__(self, order_indicator_cls=NumpyOrderIndicator):
         self.order_indicator_cls = order_indicator_cls
 
         # order indicator is metrics for a single order for a specific step
-        self.order_indicator_his: dict = OrderedDict()
+        self.order_indicator_his = OrderedDict()
         self.order_indicator: BaseOrderIndicator = self.order_indicator_cls()
 
         # trade indicator is metrics for all orders for a specific step
-        self.trade_indicator_his: dict = OrderedDict()
+        self.trade_indicator_his = OrderedDict()
-        self.trade_indicator: Dict[str, Optional[BaseSingleMetric]] = OrderedDict()
+        self.trade_indicator: Dict[str, float] = OrderedDict()
 
         self._trade_calendar = None
 
     # def reset(self, trade_calendar: TradeCalendarManager):
-    def reset(self) -> None:
+    def reset(self):
-        self.order_indicator = self.order_indicator_cls()
+        self.order_indicator: BaseOrderIndicator = self.order_indicator_cls()
         self.trade_indicator = OrderedDict()
         # self._trade_calendar = trade_calendar
 
-    def record(self, trade_start_time: Union[str, pd.Timestamp]) -> None:
+    def record(self, trade_start_time):
         self.order_indicator_his[trade_start_time] = self.get_order_indicator()
         self.trade_indicator_his[trade_start_time] = self.get_trade_indicator()
 
-    def _update_order_trade_info(self, trade_info: List[Tuple[Order, float, float, float]]) -> None:
+    def _update_order_trade_info(self, trade_info: list):
         amount = dict()
         deal_amount = dict()
         trade_price = dict()
@@ -322,7 +311,7 @@ class Indicator:
         self.order_indicator.assign("trade_dir", trade_dir)
         self.order_indicator.assign("pa", pa)
 
-    def _update_order_fulfill_rate(self) -> None:
+    def _update_order_fulfill_rate(self):
         def func(deal_amount, amount):
             # deal_amount is np.NaN or None when there is no inner decision. So full fill rate is 0.
             tmp_deal_amount = deal_amount.reindex(amount.index, 0)
@@ -331,11 +320,11 @@ class Indicator:
 
         self.order_indicator.transfer(func, "ffr")
 
-    def update_order_indicators(self, trade_info: List[Tuple[Order, float, float, float]]) -> None:
+    def update_order_indicators(self, trade_info: list):
         self._update_order_trade_info(trade_info=trade_info)
         self._update_order_fulfill_rate()
 
-    def _agg_order_trade_info(self, inner_order_indicators: List[BaseOrderIndicator]) -> None:
+    def _agg_order_trade_info(self, inner_order_indicators: List[Dict[str, pd.Series]]):
         # calculate total trade amount with each inner order indicator.
         def trade_amount_func(deal_amount, trade_price):
             return deal_amount * trade_price
@@ -346,10 +335,7 @@ class Indicator:
         # sum inner order indicators with same metric.
         all_metric = ["inner_amount", "deal_amount", "trade_price", "trade_value", "trade_cost", "trade_dir"]
         self.order_indicator_cls.sum_all_indicators(
-            self.order_indicator,
+            self.order_indicator, inner_order_indicators, all_metric, fill_value=0
-            inner_order_indicators,
-            all_metric,
-            fill_value=0,
         )
 
         def func(trade_price, deal_amount):
@@ -364,9 +350,9 @@ class Indicator:
 
         self.order_indicator.transfer(func_apply, "trade_dir")
 
-    def _update_trade_amount(self, outer_trade_decision: BaseTradeDecision) -> None:
+    def _update_trade_amount(self, outer_trade_decision: BaseTradeDecision):
         # NOTE: these indicator is designed for order execution, so the
-        decision: List[Order] = cast(List[Order], outer_trade_decision.get_decision())
+        decision: List[Order] = outer_trade_decision.get_decision()
         if len(decision) == 0:
             self.order_indicator.assign("amount", {})
         else:
@@ -381,7 +367,7 @@ class Indicator:
         decision: BaseTradeDecision,
         trade_exchange: Exchange,
         pa_config: dict = {},
-    ) -> Tuple[Optional[float], Optional[float]]:
+    ):
         """
         Get the base volume and price information
         All the base price values are rooted from this function
@@ -392,17 +378,12 @@ class Indicator:
 
         if decision.trade_range is not None:
             trade_start_time, trade_end_time = decision.trade_range.clip_time_range(
-                start_time=trade_start_time,
+                start_time=trade_start_time, end_time=trade_end_time
-                end_time=trade_end_time,
             )
 
         if price == "deal_price":
             price_s = trade_exchange.get_deal_price(
-                inst,
+                inst, trade_start_time, trade_end_time, direction=direction, method=None
-                trade_start_time,
-                trade_end_time,
-                direction=direction,
-                method=None,
             )
         else:
             raise NotImplementedError(f"This type of input is not supported")
@@ -421,35 +402,31 @@ class Indicator:
         # NOTE: there are some zeros in the trading price. These cases are known meaningless
         # for aligning the previous logic, remove it.
         # remove zero and negative values.
-        assert isinstance(price_s, idd.SingleData)
+        price_s = price_s.loc[(price_s > 1e-08).data.astype(np.bool)]
-        price_s = price_s.loc[(price_s > 1e-08).data.astype(bool)]
         # NOTE ~(price_s < 1e-08) is different from price_s >= 1e-8
         #   ~(np.NaN < 1e-8) -> ~(False)  -> True
 
-        assert isinstance(price_s, idd.SingleData)
         if agg == "vwap":
             volume_s = trade_exchange.get_volume(inst, trade_start_time, trade_end_time, method=None)
             if isinstance(volume_s, (int, float, np.number)):
                 volume_s = idd.SingleData(volume_s, [trade_start_time])
-            assert isinstance(volume_s, idd.SingleData)
             volume_s = volume_s.reindex(price_s.index)
         elif agg == "twap":
             volume_s = idd.SingleData(1, price_s.index)
         else:
             raise NotImplementedError(f"This type of input is not supported")
 
-        assert isinstance(volume_s, idd.SingleData)
         base_volume = volume_s.sum()
         base_price = (price_s * volume_s).sum() / base_volume
         return base_price, base_volume
 
     def _agg_base_price(
         self,
-        inner_order_indicators: List[BaseOrderIndicator],
+        inner_order_indicators: List[Dict[str, Union[SingleMetric, idd.SingleData]]],
         decision_list: List[Tuple[BaseTradeDecision, pd.Timestamp, pd.Timestamp]],
         trade_exchange: Exchange,
         pa_config: dict = {},
-    ) -> None:
+    ):
         """
         # NOTE:!!!!
         # Strong assumption!!!!!!
@@ -457,7 +434,7 @@ class Indicator:
 
         Parameters
         ----------
-        inner_order_indicators : List[BaseOrderIndicator]
+        inner_order_indicators : List[Dict[str, pd.Series]]
             the indicators of account of inner executor
         decision_list: List[Tuple[BaseTradeDecision, pd.Timestamp, pd.Timestamp]],
             a list of decisions according to inner_order_indicators
@@ -502,17 +479,14 @@ class Indicator:
                 bv_new = idd.SingleData(bv_new)
                 bp_all.append(bp_new)
                 bv_all.append(bv_new)
-            bp_all_multi_data = idd.concat(bp_all, axis=1)
+            bp_all = idd.concat(bp_all, axis=1)
-            bv_all_multi_data = idd.concat(bv_all, axis=1)
+            bv_all = idd.concat(bv_all, axis=1)
 
-            base_volume = bv_all_multi_data.sum(axis=1)
+            base_volume = bv_all.sum(axis=1)
             self.order_indicator.assign("base_volume", base_volume.to_dict())
-            self.order_indicator.assign(
+            self.order_indicator.assign("base_price", ((bp_all * bv_all).sum(axis=1) / base_volume).to_dict())
-                "base_price",
-                ((bp_all_multi_data * bv_all_multi_data).sum(axis=1) / base_volume).to_dict(),
-            )
 
-    def _agg_order_price_advantage(self) -> None:
+    def _agg_order_price_advantage(self):
         def if_empty_func(trade_price):
             return trade_price.empty
 
@@ -529,12 +503,12 @@ class Indicator:
 
     def agg_order_indicators(
         self,
-        inner_order_indicators: List[BaseOrderIndicator],
+        inner_order_indicators: List[Dict[str, pd.Series]],
         decision_list: List[Tuple[BaseTradeDecision, pd.Timestamp, pd.Timestamp]],
         outer_trade_decision: BaseTradeDecision,
         trade_exchange: Exchange,
-        indicator_config: dict = {},
+        indicator_config={},
-    ) -> None:
+    ):
         self._agg_order_trade_info(inner_order_indicators)
         self._update_trade_amount(outer_trade_decision)
         self._update_order_fulfill_rate()
@@ -542,66 +516,71 @@ class Indicator:
         self._agg_base_price(inner_order_indicators, decision_list, trade_exchange, pa_config=pa_config)  # TODO
         self._agg_order_price_advantage()
 
-    def _cal_trade_fulfill_rate(self, method: str = "mean") -> Optional[BaseSingleMetric]:
+    def _cal_trade_fulfill_rate(self, method="mean"):
         if method == "mean":
-            return self.order_indicator.transfer(
+
-                lambda ffr: ffr.mean(),
+            def func(ffr):
-            )
+                return ffr.mean()
+
         elif method == "amount_weighted":
-            return self.order_indicator.transfer(
+
-                lambda ffr, deal_amount: (ffr * deal_amount.abs()).sum() / (deal_amount.abs().sum()),
+            def func(ffr, deal_amount):
-            )
+                return (ffr * deal_amount.abs()).sum() / (deal_amount.abs().sum())
+
         elif method == "value_weighted":
-            return self.order_indicator.transfer(
+
-                lambda ffr, trade_value: (ffr * trade_value.abs()).sum() / (trade_value.abs().sum()),
+            def func(ffr, trade_value):
-            )
+                return (ffr * trade_value.abs()).sum() / (trade_value.abs().sum())
+
         else:
             raise ValueError(f"method {method} is not supported!")
+        return self.order_indicator.transfer(func)
 
-    def _cal_trade_price_advantage(self, method: str = "mean") -> Optional[BaseSingleMetric]:
+    def _cal_trade_price_advantage(self, method="mean"):
         if method == "mean":
-            return self.order_indicator.transfer(lambda pa: pa.mean())
+
+            def func(pa):
+                return pa.mean()
+
         elif method == "amount_weighted":
-            return self.order_indicator.transfer(
+
-                lambda pa, deal_amount: (pa * deal_amount.abs()).sum() / (deal_amount.abs().sum()),
+            def func(pa, deal_amount):
-            )
+                return (pa * deal_amount.abs()).sum() / (deal_amount.abs().sum())
+
         elif method == "value_weighted":
-            return self.order_indicator.transfer(
+
-                lambda pa, trade_value: (pa * trade_value.abs()).sum() / (trade_value.abs().sum()),
+            def func(pa, trade_value):
-            )
+                return (pa * trade_value.abs()).sum() / (trade_value.abs().sum())
+
         else:
             raise ValueError(f"method {method} is not supported!")
+        return self.order_indicator.transfer(func)
 
-    def _cal_trade_positive_rate(self) -> Optional[BaseSingleMetric]:
+    def _cal_trade_positive_rate(self):
         def func(pa):
             return (pa > 0).sum() / pa.count()
 
         return self.order_indicator.transfer(func)
 
-    def _cal_deal_amount(self) -> Optional[BaseSingleMetric]:
+    def _cal_deal_amount(self):
         def func(deal_amount):
             return deal_amount.abs().sum()
 
         return self.order_indicator.transfer(func)
 
-    def _cal_trade_value(self) -> Optional[BaseSingleMetric]:
+    def _cal_trade_value(self):
         def func(trade_value):
             return trade_value.abs().sum()
 
         return self.order_indicator.transfer(func)
 
-    def _cal_trade_order_count(self) -> Optional[BaseSingleMetric]:
+    def _cal_trade_order_count(self):
         def func(amount):
             return amount.count()
 
         return self.order_indicator.transfer(func)
 
-    def cal_trade_indicators(
+    def cal_trade_indicators(self, trade_start_time, freq, indicator_config={}):
-        self,
-        trade_start_time: Union[str, pd.Timestamp],
-        freq: str,
-        indicator_config: dict = {},
-    ) -> None:
         show_indicator = indicator_config.get("show_indicator", False)
         ffr_config = indicator_config.get("ffr_config", {})
         pa_config = indicator_config.get("pa_config", {})
@@ -619,22 +598,18 @@ class Indicator:
         self.trade_indicator["count"] = order_count
         if show_indicator:
             print(
-                "[Indicator({}) {}]: FFR: {}, PA: {}, POS: {}".format(
+                "[Indicator({}) {:%Y-%m-%d %H:%M:%S}]: FFR: {}, PA: {}, POS: {}".format(
-                    freq,
+                    freq, trade_start_time, fulfill_rate, price_advantage, positive_rate
-                    trade_start_time
+                )
-                    if isinstance(trade_start_time, str)
-                    else trade_start_time.strftime("%Y-%m-%d %H:%M:%S"),
-                    fulfill_rate,
-                    price_advantage,
-                    positive_rate,
-                ),
             )
 
-    def get_order_indicator(self, raw: bool = True) -> Union[BaseOrderIndicator, Dict[Text, pd.Series]]:
+    def get_order_indicator(self, raw: bool = True):
-        return self.order_indicator if raw else self.order_indicator.to_series()
+        if raw:
+            return self.order_indicator
+        return self.order_indicator.to_series()
 
-    def get_trade_indicator(self) -> Dict[str, Optional[BaseSingleMetric]]:
+    def get_trade_indicator(self):
         return self.trade_indicator
 
-    def generate_trade_indicators_dataframe(self) -> pd.DataFrame:
+    def generate_trade_indicators_dataframe(self):
         return pd.DataFrame.from_dict(self.trade_indicator_his, orient="index")

qlib/backtest/signal.py

@@ -1,16 +1,13 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
-import abc
-from typing import Dict, List, Text, Tuple, Union
-
-import pandas as pd
-
 from qlib.utils import init_instance_by_config
-
+from typing import Dict, List, Text, Tuple, Union
+from ..model.base import BaseModel
 from ..data.dataset import Dataset
 from ..data.dataset.utils import convert_index_format
-from ..model.base import BaseModel
 from ..utils.resam import resam_ts_data
+import pandas as pd
+import abc
 
 
 class Signal(metaclass=abc.ABCMeta):
@@ -22,7 +19,7 @@ class Signal(metaclass=abc.ABCMeta):
     """
 
     @abc.abstractmethod
-    def get_signal(self, start_time: pd.Timestamp, end_time: pd.Timestamp) -> Union[pd.Series, pd.DataFrame, None]:
+    def get_signal(self, start_time, end_time) -> Union[pd.Series, pd.DataFrame, None]:
         """
         get the signal at the end of the decision step(from `start_time` to `end_time`)
 
@@ -39,14 +36,13 @@ class SignalWCache(Signal):
     SignalWCache will store the prepared signal as a attribute and give the according signal based on input query
     """
 
-    def __init__(self, signal: Union[pd.Series, pd.DataFrame]) -> None:
+    def __init__(self, signal: Union[pd.Series, pd.DataFrame]):
         """
 
         Parameters
         ----------
         signal : Union[pd.Series, pd.DataFrame]
-            The expected format of the signal is like the data below (the order of index is not important and can be
+            The expected format of the signal is like the data below (the order of index is not important and can be automatically adjusted)
-            automatically adjusted)
 
                 instrument datetime
                 SH600000   2008-01-02  0.079704
@@ -57,8 +53,8 @@ class SignalWCache(Signal):
         """
         self.signal_cache = convert_index_format(signal, level="datetime")
 
-    def get_signal(self, start_time: pd.Timestamp, end_time: pd.Timestamp) -> Union[pd.Series, pd.DataFrame]:
+    def get_signal(self, start_time, end_time) -> Union[pd.Series, pd.DataFrame]:
-        # the frequency of the signal may not align with the decision frequency of strategy
+        # the frequency of the signal may not algin with the decision frequency of strategy
         # so resampling from the data is necessary
         # the latest signal leverage more recent data and therefore is used in trading.
         signal = resam_ts_data(self.signal_cache, start_time=start_time, end_time=end_time, method="last")
@@ -66,7 +62,7 @@ class SignalWCache(Signal):
 
 
 class ModelSignal(SignalWCache):
-    def __init__(self, model: BaseModel, dataset: Dataset) -> None:
+    def __init__(self, model: BaseModel, dataset: Dataset):
         self.model = model
         self.dataset = dataset
         pred_scores = self.model.predict(dataset)
@@ -74,7 +70,7 @@ class ModelSignal(SignalWCache):
             pred_scores = pred_scores.iloc[:, 0]
         super().__init__(pred_scores)
 
-    def _update_model(self) -> None:
+    def _update_model(self):
         """
         When using online data, update model in each bar as the following steps:
             - update dataset with online data, the dataset should support online update
@@ -86,7 +82,7 @@ class ModelSignal(SignalWCache):
 
 
 def create_signal_from(
-    obj: Union[Signal, Tuple[BaseModel, Dataset], List, Dict, Text, pd.Series, pd.DataFrame],
+    obj: Union[Signal, Tuple[BaseModel, Dataset], List, Dict, Text, pd.Series, pd.DataFrame]
 ) -> Signal:
     """
     create signal from diverse information

qlib/backtest/utils.py

@@ -2,20 +2,15 @@
 # Licensed under the MIT License.
 
 from __future__ import annotations
-
+import bisect
-from abc import abstractmethod
-from typing import Any, Set, Tuple, TYPE_CHECKING, Union
-
-import numpy as np
-
 from qlib.utils.time import epsilon_change
+from typing import TYPE_CHECKING, Tuple, Union
 
 if TYPE_CHECKING:
     from qlib.backtest.decision import BaseTradeDecision
 
-import warnings
-
 import pandas as pd
+import warnings
 
 from ..data.data import Cal
 
@@ -31,8 +26,8 @@ class TradeCalendarManager:
         freq: str,
         start_time: Union[str, pd.Timestamp] = None,
         end_time: Union[str, pd.Timestamp] = None,
-        level_infra: LevelInfrastructure = None,
+        level_infra: "LevelInfrastructure" = None,
-    ) -> None:
+    ):
         """
         Parameters
         ----------
@@ -48,26 +43,19 @@ class TradeCalendarManager:
         self.level_infra = level_infra
         self.reset(freq=freq, start_time=start_time, end_time=end_time)
 
-    def reset(
+    def reset(self, freq, start_time, end_time):
-        self,
-        freq: str,
-        start_time: Union[str, pd.Timestamp] = None,
-        end_time: Union[str, pd.Timestamp] = None,
-    ) -> None:
         """
         Please refer to the docs of `__init__`
 
         Reset the trade calendar
         - self.trade_len : The total count for trading step
-        - self.trade_step : The number of trading step finished, self.trade_step can be
+        - self.trade_step : The number of trading step finished, self.trade_step can be [0, 1, 2, ..., self.trade_len - 1]
-            [0, 1, 2, ..., self.trade_len - 1]
         """
         self.freq = freq
         self.start_time = pd.Timestamp(start_time) if start_time else None
         self.end_time = pd.Timestamp(end_time) if end_time else None
 
         _calendar = Cal.calendar(freq=freq, future=True)
-        assert isinstance(_calendar, np.ndarray)
         self._calendar = _calendar
         _, _, _start_index, _end_index = Cal.locate_index(start_time, end_time, freq=freq, future=True)
         self.start_index = _start_index
@@ -75,7 +63,7 @@ class TradeCalendarManager:
         self.trade_len = _end_index - _start_index + 1
         self.trade_step = 0
 
-    def finished(self) -> bool:
+    def finished(self):
         """
         Check if the trading finished
         - Should check before calling strategy.generate_decisions and executor.execute
@@ -84,32 +72,29 @@ class TradeCalendarManager:
         """
         return self.trade_step >= self.trade_len
 
-    def step(self) -> None:
+    def step(self):
         if self.finished():
             raise RuntimeError(f"The calendar is finished, please reset it if you want to call it!")
-        self.trade_step += 1
+        self.trade_step = self.trade_step + 1
 
-    def get_freq(self) -> str:
+    def get_freq(self):
         return self.freq
 
-    def get_trade_len(self) -> int:
+    def get_trade_len(self):
         """get the total step length"""
         return self.trade_len
 
-    def get_trade_step(self) -> int:
+    def get_trade_step(self):
         return self.trade_step
 
-    def get_step_time(self, trade_step: int = None, shift: int = 0) -> Tuple[pd.Timestamp, pd.Timestamp]:
+    def get_step_time(self, trade_step=None, shift=0):
         """
         Get the left and right endpoints of the trade_step'th trading interval
 
         About the endpoints:
-            - Qlib uses the closed interval in time-series data selection, which has the same performance as
+            - Qlib uses the closed interval in time-series data selection, which has the same performance as pandas.Series.loc
-            pandas.Series.loc
+            # - The returned right endpoints should minus 1 seconds because of the closed interval representation in Qlib.
-            # - The returned right endpoints should minus 1 seconds because of the closed interval representation in
+            # Note: Qlib supports up to minutely decision execution, so 1 seconds is less than any trading time interval.
-            #   Qlib.
-            # Note: Qlib supports up to minutely decision execution, so 1 seconds is less than any trading time
-            #   interval.
 
         Parameters
         ----------
@@ -120,14 +105,15 @@ class TradeCalendarManager:
 
         Returns
         -------
-        Tuple[pd.Timestamp, pd.Timestamp]
+        Tuple[pd.Timestamp, pd.Timestap]
             - If shift == 0, return the trading time range
             - If shift > 0, return the trading time range of the earlier shift bars
             - If shift < 0, return the trading time range of the later shift bar
         """
         if trade_step is None:
             trade_step = self.get_trade_step()
-        calendar_index = self.start_index + trade_step - shift
+        trade_step = trade_step - shift
+        calendar_index = self.start_index + trade_step
         return self._calendar[calendar_index], epsilon_change(self._calendar[calendar_index + 1])
 
     def get_data_cal_range(self, rtype: str = "full") -> Tuple[int, int]:
@@ -140,7 +126,7 @@ class TradeCalendarManager:
         Parameters
         ----------
         rtype: str
-            - "full": return the full limitation of the decision in the day
+            - "full": return the full limitation of the deicsion in the day
             - "step": return the limitation of current step
 
         Returns
@@ -148,8 +134,6 @@ class TradeCalendarManager:
         Tuple[int, int]:
         """
         # potential performance issue
-        assert self.level_infra is not None
-
         day_start = pd.Timestamp(self.start_time.date())
         day_end = epsilon_change(day_start + pd.Timedelta(days=1))
         freq = self.level_infra.get("common_infra").get("trade_exchange").freq
@@ -164,7 +148,7 @@ class TradeCalendarManager:
 
         return start_idx - day_start_idx, end_index - day_start_idx
 
-    def get_all_time(self) -> Tuple[pd.Timestamp, pd.Timestamp]:
+    def get_all_time(self):
         """Get the start_time and end_time for trading"""
         return self.start_time, self.end_time
 
@@ -183,33 +167,30 @@ class TradeCalendarManager:
         Tuple[int, int]:
             the index of the range.  **the left and right are closed**
         """
-        left = int(np.searchsorted(self._calendar, start_time, side="right") - 1)
+        left, right = (
-        right = int(np.searchsorted(self._calendar, end_time, side="right") - 1)
+            bisect.bisect_right(self._calendar, start_time) - 1,
+            bisect.bisect_right(self._calendar, end_time) - 1,
+        )
         left -= self.start_index
         right -= self.start_index
 
-        def clip(idx: int) -> int:
+        def clip(idx):
             return min(max(0, idx), self.trade_len - 1)
 
         return clip(left), clip(right)
 
     def __repr__(self) -> str:
-        return (
+        return f"class: {self.__class__.__name__}; {self.start_time}[{self.start_index}]~{self.end_time}[{self.end_index}]: [{self.trade_step}/{self.trade_len}]"
-            f"class: {self.__class__.__name__}; "
-            f"{self.start_time}[{self.start_index}]~{self.end_time}[{self.end_index}]: "
-            f"[{self.trade_step}/{self.trade_len}]"
-        )
 
 
 class BaseInfrastructure:
-    def __init__(self, **kwargs: Any) -> None:
+    def __init__(self, **kwargs):
         self.reset_infra(**kwargs)
 
-    @abstractmethod
+    def get_support_infra(self):
-    def get_support_infra(self) -> Set[str]:
         raise NotImplementedError("`get_support_infra` is not implemented!")
 
-    def reset_infra(self, **kwargs: Any) -> None:
+    def reset_infra(self, **kwargs):
         support_infra = self.get_support_infra()
         for k, v in kwargs.items():
             if k in support_infra:
@@ -217,58 +198,53 @@ class BaseInfrastructure:
             else:
                 warnings.warn(f"{k} is ignored in `reset_infra`!")
 
-    def get(self, infra_name: str) -> Any:
+    def get(self, infra_name):
         if hasattr(self, infra_name):
             return getattr(self, infra_name)
         else:
             warnings.warn(f"infra {infra_name} is not found!")
 
-    def has(self, infra_name: str) -> bool:
+    def has(self, infra_name):
         return infra_name in self.get_support_infra() and hasattr(self, infra_name)
 
-    def update(self, other: BaseInfrastructure) -> None:
+    def update(self, other):
         support_infra = other.get_support_infra()
         infra_dict = {_infra: getattr(other, _infra) for _infra in support_infra if hasattr(other, _infra)}
         self.reset_infra(**infra_dict)
 
 
 class CommonInfrastructure(BaseInfrastructure):
-    def get_support_infra(self) -> Set[str]:
+    def get_support_infra(self):
-        return {"trade_account", "trade_exchange"}
+        return ["trade_account", "trade_exchange"]
 
 
 class LevelInfrastructure(BaseInfrastructure):
     """level infrastructure is created by executor, and then shared to strategies on the same level"""
 
-    def get_support_infra(self) -> Set[str]:
+    def get_support_infra(self):
         """
         Descriptions about the infrastructure
 
         sub_level_infra:
         - **NOTE**: this will only work after _init_sub_trading !!!
         """
-        return {"trade_calendar", "sub_level_infra", "common_infra", "executor"}
+        return ["trade_calendar", "sub_level_infra", "common_infra"]
 
-    def reset_cal(
+    def reset_cal(self, freq, start_time, end_time):
-        self,
-        freq: str,
-        start_time: Union[str, pd.Timestamp, None],
-        end_time: Union[str, pd.Timestamp, None],
-    ) -> None:
         """reset trade calendar manager"""
         if self.has("trade_calendar"):
             self.get("trade_calendar").reset(freq, start_time=start_time, end_time=end_time)
         else:
             self.reset_infra(
-                trade_calendar=TradeCalendarManager(freq, start_time=start_time, end_time=end_time, level_infra=self),
+                trade_calendar=TradeCalendarManager(freq, start_time=start_time, end_time=end_time, level_infra=self)
             )
 
-    def set_sub_level_infra(self, sub_level_infra: LevelInfrastructure) -> None:
+    def set_sub_level_infra(self, sub_level_infra: LevelInfrastructure):
-        """this will make the calendar access easier when crossing multi-levels"""
+        """this will make the calendar access easier when acrossing multi-levels"""
         self.reset_infra(sub_level_infra=sub_level_infra)
 
 
-def get_start_end_idx(trade_calendar: TradeCalendarManager, outer_trade_decision: BaseTradeDecision) -> Tuple[int, int]:
+def get_start_end_idx(trade_calendar: TradeCalendarManager, outer_trade_decision: BaseTradeDecision) -> Union[int, int]:
     """
     A helper function for getting the decision-level index range limitation for inner strategy
     - NOTE: this function is not applicable to order-level

qlib/config.py

@@ -75,17 +75,6 @@ class Config:
     def set_conf_from_C(self, config_c):
         self.update(**config_c.__dict__["_config"])
 
-    def register_from_C(self, config, skip_register=True):
-        from .utils import set_log_with_config  # pylint: disable=C0415
-
-        if C.registered and skip_register:
-            return
-
-        C.set_conf_from_C(config)
-        if C.logging_config:
-            set_log_with_config(C.logging_config)
-        C.register()
-
 
 # pickle.dump protocol version: https://docs.python.org/3/library/pickle.html#data-stream-format
 PROTOCOL_VERSION = 4
@@ -113,7 +102,7 @@ _default_config = {
     #   "~/.qlib/stock_data/cn_data"
     #   # dict
     #   {"day": "~/.qlib/stock_data/cn_data", "1min": "~/.qlib/stock_data/cn_data_1min"}
-    # NOTE: provider_uri priority:
+    # NOTE: provider_uri priority：
     #   1. backend_config: backend_obj["kwargs"]["provider_uri"]
     #   2. backend_config: backend_obj["kwargs"]["provider_uri_map"]
     #   3. qlib.init: provider_uri
@@ -172,9 +161,6 @@ _default_config = {
             }
         },
         "loggers": {"qlib": {"level": logging.DEBUG, "handlers": ["console"]}},
-        # To let qlib work with other packages, we shouldn't disable existing loggers.
-        # Note that this param is default to True according to the documentation of logging.
-        "disable_existing_loggers": False,
     },
     # Default config for experiment manager
     "exp_manager": {
@@ -411,7 +397,8 @@ class QlibConfig(Config):
         if _logging_config:
             set_log_with_config(_logging_config)
 
-        logger = get_module_logger("Initialization", kwargs.get("logging_level", self.logging_level))
+        # FIXME: this logger ignored the level in config
+        logger = get_module_logger("Initialization", level=logging.INFO)
         logger.info(f"default_conf: {default_conf}.")
 
         self.set_mode(default_conf)

qlib/constant.py

@@ -2,21 +2,9 @@
 # Licensed under the MIT License.
 
 # REGION CONST
-from typing import TypeVar
-
-import numpy as np
-import pandas as pd
-
 REG_CN = "cn"
 REG_US = "us"
 REG_TW = "tw"
 
 # Epsilon for avoiding division by zero.
 EPS = 1e-12
-
-# Infinity in integer
-INF = int(1e18)
-ONE_DAY = pd.Timedelta("1day")
-ONE_MIN = pd.Timedelta("1min")
-EPS_T = pd.Timedelta("1s")  # use 1 second to exclude the right interval point
-float_or_ndarray = TypeVar("float_or_ndarray", float, np.ndarray)

qlib/contrib/data/dataset.py

@@ -63,20 +63,11 @@ def _get_date_parse_fn(target):
         get_date_parse_fn(20120101)('2017-01-01') => 20170101
     """
     if isinstance(target, int):
-
+        _fn = lambda x: int(str(x).replace("-", "")[:8])  # 20200201
-        def _fn(x):
-            return int(str(x).replace("-", "")[:8])  # 20200201
-
     elif isinstance(target, str) and len(target) == 8:
-
+        _fn = lambda x: str(x).replace("-", "")[:8]  # '20200201'
-        def _fn(x):
-            return str(x).replace("-", "")[:8]  # '20200201'
-
     else:
-
+        _fn = lambda x: x  # '2021-01-01'
-        def _fn(x):
-            return x  # '2021-01-01'
-
     return _fn
 
 
@@ -203,14 +194,8 @@ class MTSDatasetH(DatasetH):
 
     def _prepare_seg(self, slc, **kwargs):
         fn = _get_date_parse_fn(self._index[0][1])
-        if isinstance(slc, slice):
+        start_date = fn(slc.start)
-            start, stop = slc.start, slc.stop
+        end_date = fn(slc.stop)
-        elif isinstance(slc, (list, tuple)):
-            start, stop = slc
-        else:
-            raise NotImplementedError(f"This type of input is not supported")
-        start_date = pd.Timestamp(fn(start))
-        end_date = pd.Timestamp(fn(stop))
         obj = copy.copy(self)  # shallow copy
         # NOTE: Seriable will disable copy `self._data` so we manually assign them here
         obj._data = self._data  # reference (no copy)

qlib/contrib/data/handler.py

@@ -57,7 +57,7 @@ class Alpha360(DataHandlerLP):
         fit_end_time=None,
         filter_pipe=None,
         inst_processor=None,
-        **kwargs
+        **kwargs,
     ):
         infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time)
         learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time)
@@ -67,7 +67,7 @@ class Alpha360(DataHandlerLP):
             "kwargs": {
                 "config": {
                     "feature": self.get_feature_config(),
-                    "label": kwargs.pop("label", self.get_label_config()),
+                    "label": kwargs.get("label", self.get_label_config()),
                 },
                 "filter_pipe": filter_pipe,
                 "freq": freq,
@@ -82,14 +82,12 @@ class Alpha360(DataHandlerLP):
             data_loader=data_loader,
             learn_processors=learn_processors,
             infer_processors=infer_processors,
-            **kwargs
         )
 
     def get_label_config(self):
-        return ["Ref($close, -2)/Ref($close, -1) - 1"], ["LABEL0"]
+        return (["Ref($close, -2)/Ref($close, -1) - 1"], ["LABEL0"])
 
-    @staticmethod
+    def get_feature_config(self):
-    def get_feature_config():
         # NOTE:
         # Alpha360 tries to provide a dataset with original price data
         # the original price data includes the prices and volume in the last 60 days.
@@ -101,33 +99,33 @@ class Alpha360(DataHandlerLP):
         names = []
 
         for i in range(59, 0, -1):
-            fields += ["Ref($close, %d)/$close" % i]
+            fields += ["Ref($close, %d)/$close" % (i)]
-            names += ["CLOSE%d" % i]
+            names += ["CLOSE%d" % (i)]
         fields += ["$close/$close"]
         names += ["CLOSE0"]
         for i in range(59, 0, -1):
-            fields += ["Ref($open, %d)/$close" % i]
+            fields += ["Ref($open, %d)/$close" % (i)]
-            names += ["OPEN%d" % i]
+            names += ["OPEN%d" % (i)]
         fields += ["$open/$close"]
         names += ["OPEN0"]
         for i in range(59, 0, -1):
-            fields += ["Ref($high, %d)/$close" % i]
+            fields += ["Ref($high, %d)/$close" % (i)]
-            names += ["HIGH%d" % i]
+            names += ["HIGH%d" % (i)]
         fields += ["$high/$close"]
         names += ["HIGH0"]
         for i in range(59, 0, -1):
-            fields += ["Ref($low, %d)/$close" % i]
+            fields += ["Ref($low, %d)/$close" % (i)]
-            names += ["LOW%d" % i]
+            names += ["LOW%d" % (i)]
         fields += ["$low/$close"]
         names += ["LOW0"]
         for i in range(59, 0, -1):
-            fields += ["Ref($vwap, %d)/$close" % i]
+            fields += ["Ref($vwap, %d)/$close" % (i)]
-            names += ["VWAP%d" % i]
+            names += ["VWAP%d" % (i)]
         fields += ["$vwap/$close"]
         names += ["VWAP0"]
         for i in range(59, 0, -1):
-            fields += ["Ref($volume, %d)/($volume+1e-12)" % i]
+            fields += ["Ref($volume, %d)/($volume+1e-12)" % (i)]
-            names += ["VOLUME%d" % i]
+            names += ["VOLUME%d" % (i)]
         fields += ["$volume/($volume+1e-12)"]
         names += ["VOLUME0"]
 
@@ -136,7 +134,7 @@ class Alpha360(DataHandlerLP):
 
 class Alpha360vwap(Alpha360):
     def get_label_config(self):
-        return ["Ref($vwap, -2)/Ref($vwap, -1) - 1"], ["LABEL0"]
+        return (["Ref($vwap, -2)/Ref($vwap, -1) - 1"], ["LABEL0"])
 
 
 class Alpha158(DataHandlerLP):
@@ -153,7 +151,7 @@ class Alpha158(DataHandlerLP):
         process_type=DataHandlerLP.PTYPE_A,
         filter_pipe=None,
         inst_processor=None,
-        **kwargs
+        **kwargs,
     ):
         infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time)
         learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time)
@@ -163,7 +161,7 @@ class Alpha158(DataHandlerLP):
             "kwargs": {
                 "config": {
                     "feature": self.get_feature_config(),
-                    "label": kwargs.pop("label", self.get_label_config()),
+                    "label": kwargs.get("label", self.get_label_config()),
                 },
                 "filter_pipe": filter_pipe,
                 "freq": freq,
@@ -178,7 +176,6 @@ class Alpha158(DataHandlerLP):
             infer_processors=infer_processors,
             learn_processors=learn_processors,
             process_type=process_type,
-            **kwargs
         )
 
     def get_feature_config(self):
@@ -193,7 +190,7 @@ class Alpha158(DataHandlerLP):
         return self.parse_config_to_fields(conf)
 
     def get_label_config(self):
-        return ["Ref($close, -2)/Ref($close, -1) - 1"], ["LABEL0"]
+        return (["Ref($close, -2)/Ref($close, -1) - 1"], ["LABEL0"])
 
     @staticmethod
     def parse_config_to_fields(config):
@@ -258,123 +255,80 @@ class Alpha158(DataHandlerLP):
             exclude = config["rolling"].get("exclude", [])
             # `exclude` in dataset config unnecessary filed
             # `include` in dataset config necessary field
-
+            use = lambda x: x not in exclude and (include is None or x in include)
-            def use(x):
-                return x not in exclude and (include is None or x in include)
-
-            # Some factor ref: https://guorn.com/static/upload/file/3/134065454575605.pdf
             if use("ROC"):
-                # https://www.investopedia.com/terms/r/rateofchange.asp
-                # Rate of change, the price change in the past d days, divided by latest close price to remove unit
                 fields += ["Ref($close, %d)/$close" % d for d in windows]
                 names += ["ROC%d" % d for d in windows]
             if use("MA"):
-                # https://www.investopedia.com/ask/answers/071414/whats-difference-between-moving-average-and-weighted-moving-average.asp
-                # Simple Moving Average, the simple moving average in the past d days, divided by latest close price to remove unit
                 fields += ["Mean($close, %d)/$close" % d for d in windows]
                 names += ["MA%d" % d for d in windows]
             if use("STD"):
-                # The standard diviation of close price for the past d days, divided by latest close price to remove unit
                 fields += ["Std($close, %d)/$close" % d for d in windows]
                 names += ["STD%d" % d for d in windows]
             if use("BETA"):
-                # The rate of close price change in the past d days, divided by latest close price to remove unit
-                # For example, price increase 10 dollar per day in the past d days, then Slope will be 10.
                 fields += ["Slope($close, %d)/$close" % d for d in windows]
                 names += ["BETA%d" % d for d in windows]
             if use("RSQR"):
-                # The R-sqaure value of linear regression for the past d days, represent the trend linear
                 fields += ["Rsquare($close, %d)" % d for d in windows]
                 names += ["RSQR%d" % d for d in windows]
             if use("RESI"):
-                # The redisdual for linear regression for the past d days, represent the trend linearity for past d days.
                 fields += ["Resi($close, %d)/$close" % d for d in windows]
                 names += ["RESI%d" % d for d in windows]
             if use("MAX"):
-                # The max price for past d days, divided by latest close price to remove unit
                 fields += ["Max($high, %d)/$close" % d for d in windows]
                 names += ["MAX%d" % d for d in windows]
             if use("LOW"):
-                # The low price for past d days, divided by latest close price to remove unit
                 fields += ["Min($low, %d)/$close" % d for d in windows]
                 names += ["MIN%d" % d for d in windows]
             if use("QTLU"):
-                # The 80% quantile of past d day's close price, divided by latest close price to remove unit
-                # Used with MIN and MAX
                 fields += ["Quantile($close, %d, 0.8)/$close" % d for d in windows]
                 names += ["QTLU%d" % d for d in windows]
             if use("QTLD"):
-                # The 20% quantile of past d day's close price, divided by latest close price to remove unit
                 fields += ["Quantile($close, %d, 0.2)/$close" % d for d in windows]
                 names += ["QTLD%d" % d for d in windows]
             if use("RANK"):
-                # Get the percentile of current close price in past d day's close price.
-                # Represent the current price level comparing to past N days, add additional information to moving average.
                 fields += ["Rank($close, %d)" % d for d in windows]
                 names += ["RANK%d" % d for d in windows]
             if use("RSV"):
-                # Represent the price position between upper and lower resistent price for past d days.
                 fields += ["($close-Min($low, %d))/(Max($high, %d)-Min($low, %d)+1e-12)" % (d, d, d) for d in windows]
                 names += ["RSV%d" % d for d in windows]
             if use("IMAX"):
-                # The number of days between current date and previous highest price date.
-                # Part of Aroon Indicator https://www.investopedia.com/terms/a/aroon.asp
-                # The indicator measures the time between highs and the time between lows over a time period.
-                # The idea is that strong uptrends will regularly see new highs, and strong downtrends will regularly see new lows.
                 fields += ["IdxMax($high, %d)/%d" % (d, d) for d in windows]
                 names += ["IMAX%d" % d for d in windows]
             if use("IMIN"):
-                # The number of days between current date and previous lowest price date.
-                # Part of Aroon Indicator https://www.investopedia.com/terms/a/aroon.asp
-                # The indicator measures the time between highs and the time between lows over a time period.
-                # The idea is that strong uptrends will regularly see new highs, and strong downtrends will regularly see new lows.
                 fields += ["IdxMin($low, %d)/%d" % (d, d) for d in windows]
                 names += ["IMIN%d" % d for d in windows]
             if use("IMXD"):
-                # The time period between previous lowest-price date occur after highest price date.
-                # Large value suggest downward momemtum.
                 fields += ["(IdxMax($high, %d)-IdxMin($low, %d))/%d" % (d, d, d) for d in windows]
                 names += ["IMXD%d" % d for d in windows]
             if use("CORR"):
-                # The correlation between absolute close price and log scaled trading volume
                 fields += ["Corr($close, Log($volume+1), %d)" % d for d in windows]
                 names += ["CORR%d" % d for d in windows]
             if use("CORD"):
-                # The correlation between price change ratio and volume change ratio
                 fields += ["Corr($close/Ref($close,1), Log($volume/Ref($volume, 1)+1), %d)" % d for d in windows]
                 names += ["CORD%d" % d for d in windows]
             if use("CNTP"):
-                # The percentage of days in past d days that price go up.
                 fields += ["Mean($close>Ref($close, 1), %d)" % d for d in windows]
                 names += ["CNTP%d" % d for d in windows]
             if use("CNTN"):
-                # The percentage of days in past d days that price go down.
                 fields += ["Mean($close<Ref($close, 1), %d)" % d for d in windows]
                 names += ["CNTN%d" % d for d in windows]
             if use("CNTD"):
-                # The diff between past up day and past down day
                 fields += ["Mean($close>Ref($close, 1), %d)-Mean($close<Ref($close, 1), %d)" % (d, d) for d in windows]
                 names += ["CNTD%d" % d for d in windows]
             if use("SUMP"):
-                # The total gain / the absolute total price changed
-                # Similar to RSI indicator. https://www.investopedia.com/terms/r/rsi.asp
                 fields += [
                     "Sum(Greater($close-Ref($close, 1), 0), %d)/(Sum(Abs($close-Ref($close, 1)), %d)+1e-12)" % (d, d)
                     for d in windows
                 ]
                 names += ["SUMP%d" % d for d in windows]
             if use("SUMN"):
-                # The total lose / the absolute total price changed
-                # Can be derived from SUMP by SUMN = 1 - SUMP
-                # Similar to RSI indicator. https://www.investopedia.com/terms/r/rsi.asp
                 fields += [
                     "Sum(Greater(Ref($close, 1)-$close, 0), %d)/(Sum(Abs($close-Ref($close, 1)), %d)+1e-12)" % (d, d)
                     for d in windows
                 ]
                 names += ["SUMN%d" % d for d in windows]
             if use("SUMD"):
-                # The diff ratio between total gain and total lose
-                # Similar to RSI indicator. https://www.investopedia.com/terms/r/rsi.asp
                 fields += [
                     "(Sum(Greater($close-Ref($close, 1), 0), %d)-Sum(Greater(Ref($close, 1)-$close, 0), %d))"
                     "/(Sum(Abs($close-Ref($close, 1)), %d)+1e-12)" % (d, d, d)
@@ -382,15 +336,12 @@ class Alpha158(DataHandlerLP):
                 ]
                 names += ["SUMD%d" % d for d in windows]
             if use("VMA"):
-                # Simple Volume Moving average: https://www.barchart.com/education/technical-indicators/volume_moving_average
                 fields += ["Mean($volume, %d)/($volume+1e-12)" % d for d in windows]
                 names += ["VMA%d" % d for d in windows]
             if use("VSTD"):
-                # The standard deviation for volume in past d days.
                 fields += ["Std($volume, %d)/($volume+1e-12)" % d for d in windows]
                 names += ["VSTD%d" % d for d in windows]
             if use("WVMA"):
-                # The volume weighted price change volatility
                 fields += [
                     "Std(Abs($close/Ref($close, 1)-1)*$volume, %d)/(Mean(Abs($close/Ref($close, 1)-1)*$volume, %d)+1e-12)"
                     % (d, d)
@@ -398,7 +349,6 @@ class Alpha158(DataHandlerLP):
                 ]
                 names += ["WVMA%d" % d for d in windows]
             if use("VSUMP"):
-                # The total volume increase / the absolute total volume changed
                 fields += [
                     "Sum(Greater($volume-Ref($volume, 1), 0), %d)/(Sum(Abs($volume-Ref($volume, 1)), %d)+1e-12)"
                     % (d, d)
@@ -406,8 +356,6 @@ class Alpha158(DataHandlerLP):
                 ]
                 names += ["VSUMP%d" % d for d in windows]
             if use("VSUMN"):
-                # The total volume increase / the absolute total volume changed
-                # Can be derived from VSUMP by VSUMN = 1 - VSUMP
                 fields += [
                     "Sum(Greater(Ref($volume, 1)-$volume, 0), %d)/(Sum(Abs($volume-Ref($volume, 1)), %d)+1e-12)"
                     % (d, d)
@@ -415,8 +363,6 @@ class Alpha158(DataHandlerLP):
                 ]
                 names += ["VSUMN%d" % d for d in windows]
             if use("VSUMD"):
-                # The diff ratio between total volume increase and total volume decrease
-                # RSI indicator for volume
                 fields += [
                     "(Sum(Greater($volume-Ref($volume, 1), 0), %d)-Sum(Greater(Ref($volume, 1)-$volume, 0), %d))"
                     "/(Sum(Abs($volume-Ref($volume, 1)), %d)+1e-12)" % (d, d, d)
@@ -429,4 +375,4 @@ class Alpha158(DataHandlerLP):
 
 class Alpha158vwap(Alpha158):
     def get_label_config(self):
-        return ["Ref($vwap, -2)/Ref($vwap, -1) - 1"], ["LABEL0"]
+        return (["Ref($vwap, -2)/Ref($vwap, -1) - 1"], ["LABEL0"])

qlib/contrib/data/highfreq_handler.py

@@ -1,7 +1,5 @@
 from qlib.data.dataset.handler import DataHandler, DataHandlerLP
 
-from .handler import check_transform_proc
-
 EPSILON = 1e-4
 
 
@@ -17,9 +15,20 @@ class HighFreqHandler(DataHandlerLP):
         fit_end_time=None,
         drop_raw=True,
     ):
+        def check_transform_proc(proc_l):
+            new_l = []
+            for p in proc_l:
+                p["kwargs"].update(
+                    {
+                        "fit_start_time": fit_start_time,
+                        "fit_end_time": fit_end_time,
+                    }
+                )
+                new_l.append(p)
+            return new_l
 
-        infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time)
+        infer_processors = check_transform_proc(infer_processors)
-        learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time)
+        learn_processors = check_transform_proc(learn_processors)
 
         data_loader = {
             "class": "QlibDataLoader",
@@ -101,103 +110,6 @@ class HighFreqHandler(DataHandlerLP):
         return fields, names
 
 
-class HighFreqGeneralHandler(DataHandlerLP):
-    def __init__(
-        self,
-        instruments="csi300",
-        start_time=None,
-        end_time=None,
-        infer_processors=[],
-        learn_processors=[],
-        fit_start_time=None,
-        fit_end_time=None,
-        drop_raw=True,
-        day_length=240,
-    ):
-        self.day_length = day_length
-
-        infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time)
-        learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time)
-
-        data_loader = {
-            "class": "QlibDataLoader",
-            "kwargs": {
-                "config": self.get_feature_config(),
-                "swap_level": False,
-                "freq": "1min",
-            },
-        }
-        super().__init__(
-            instruments=instruments,
-            start_time=start_time,
-            end_time=end_time,
-            data_loader=data_loader,
-            infer_processors=infer_processors,
-            learn_processors=learn_processors,
-            drop_raw=drop_raw,
-        )
-
-    def get_feature_config(self):
-        fields = []
-        names = []
-
-        template_if = "If(IsNull({1}), {0}, {1})"
-        template_paused = f"Cut({{0}}, {self.day_length * 2}, None)"
-
-        def get_normalized_price_feature(price_field, shift=0):
-            # norm with the close price of 237th minute of yesterday.
-            if shift == 0:
-                template_norm = f"{{0}}/DayLast(Ref({{1}}, {self.day_length * 2}))"
-            else:
-                template_norm = f"Ref({{0}}, " + str(shift) + f")/DayLast(Ref({{1}}, {self.day_length}))"
-
-            template_fillnan = "FFillNan({0})"
-            # calculate -> ffill -> remove paused
-            feature_ops = template_paused.format(
-                template_fillnan.format(
-                    template_norm.format(template_if.format("$close", price_field), template_fillnan.format("$close"))
-                )
-            )
-            return feature_ops
-
-        fields += [get_normalized_price_feature("$open", 0)]
-        fields += [get_normalized_price_feature("$high", 0)]
-        fields += [get_normalized_price_feature("$low", 0)]
-        fields += [get_normalized_price_feature("$close", 0)]
-        fields += [get_normalized_price_feature("$vwap", 0)]
-        names += ["$open", "$high", "$low", "$close", "$vwap"]
-
-        fields += [get_normalized_price_feature("$open", self.day_length)]
-        fields += [get_normalized_price_feature("$high", self.day_length)]
-        fields += [get_normalized_price_feature("$low", self.day_length)]
-        fields += [get_normalized_price_feature("$close", self.day_length)]
-        fields += [get_normalized_price_feature("$vwap", self.day_length)]
-        names += ["$open_1", "$high_1", "$low_1", "$close_1", "$vwap_1"]
-
-        # calculate and fill nan with 0
-        fields += [
-            template_paused.format(
-                "If(IsNull({0}), 0, {0})".format(
-                    f"{{0}}/Ref(DayLast(Mean({{0}}, {self.day_length * 30})), {self.day_length})".format("$volume")
-                )
-            )
-        ]
-        names += ["$volume"]
-
-        fields += [
-            template_paused.format(
-                "If(IsNull({0}), 0, {0})".format(
-                    f"Ref({{0}}, {self.day_length})/Ref(DayLast(Mean({{0}}, {self.day_length * 30})), {self.day_length})".format(
-                        "$volume"
-                    )
-                )
-            )
-        ]
-        names += ["$volume_1"]
-
-        return fields, names
-
-
 class HighFreqBacktestHandler(DataHandler):
     def __init__(
         self,
@@ -220,266 +132,13 @@ class HighFreqBacktestHandler(DataHandler):
             data_loader=data_loader,
         )
 
-    def get_feature_config(self):
-        fields = []
-        names = []
-
-        template_if = "If(IsNull({1}), {0}, {1})"
-        template_paused = "Select(Gt($paused_num, 1.001), {0})"
-        template_fillnan = "FFillNan({0})"
-        fields += [
-            template_fillnan.format(template_paused.format("$close")),
-        ]
-        names += ["$close0"]
-
-        fields += [
-            template_paused.format(
-                template_if.format(
-                    template_fillnan.format("$close"),
-                    "$vwap",
-                )
-            )
-        ]
-        names += ["$vwap0"]
-
-        fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$volume"))]
-        names += ["$volume0"]
-
-        fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$factor"))]
-        names += ["$factor0"]
-
-        return fields, names
-
-
-class HighFreqGeneralBacktestHandler(DataHandler):
-    def __init__(
-        self,
-        instruments="csi300",
-        start_time=None,
-        end_time=None,
-        day_length=240,
-    ):
-        self.day_length = day_length
-        data_loader = {
-            "class": "QlibDataLoader",
-            "kwargs": {
-                "config": self.get_feature_config(),
-                "swap_level": False,
-                "freq": "1min",
-            },
-        }
-        super().__init__(
-            instruments=instruments,
-            start_time=start_time,
-            end_time=end_time,
-            data_loader=data_loader,
-        )
-
-    def get_feature_config(self):
-        fields = []
-        names = []
-
-        template_paused = f"Cut({{0}}, {self.day_length * 2}, None)"
-        template_fillnan = "FFillNan({0})"
-        template_if = "If(IsNull({1}), {0}, {1})"
-        fields += [
-            template_paused.format(template_fillnan.format("$close")),
-        ]
-        names += ["$close0"]
-
-        fields += [
-            template_paused.format(template_if.format(template_fillnan.format("$close"), "$vwap")),
-        ]
-        names += ["$vwap0"]
-
-        fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$volume"))]
-        names += ["$volume0"]
-
-        return fields, names
-
-
-class HighFreqOrderHandler(DataHandlerLP):
-    def __init__(
-        self,
-        instruments="csi300",
-        start_time=None,
-        end_time=None,
-        infer_processors=[],
-        learn_processors=[],
-        fit_start_time=None,
-        fit_end_time=None,
-        drop_raw=True,
-    ):
-
-        infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time)
-        learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time)
-
-        data_loader = {
-            "class": "QlibDataLoader",
-            "kwargs": {
-                "config": self.get_feature_config(),
-                "swap_level": False,
-                "freq": "1min",
-            },
-        }
-        super().__init__(
-            instruments=instruments,
-            start_time=start_time,
-            end_time=end_time,
-            data_loader=data_loader,
-            infer_processors=infer_processors,
-            learn_processors=learn_processors,
-            drop_raw=drop_raw,
-        )
-
-    def get_feature_config(self):
-        fields = []
-        names = []
-
-        template_if = "If(IsNull({1}), {0}, {1})"
-        template_ifinf = "If(IsInf({1}), {0}, {1})"
-        template_paused = "Select(Gt($paused_num, 1.001), {0})"
-
-        def get_normalized_price_feature(price_field, shift=0):
-            # norm with the close price of 237th minute of yesterday.
-            if shift == 0:
-                template_norm = "{0}/DayLast(Ref({1}, 243))"
-            else:
-                template_norm = "Ref({0}, " + str(shift) + ")/DayLast(Ref({1}, 243))"
-
-            template_fillnan = "FFillNan({0})"
-            # calculate -> ffill -> remove paused
-            feature_ops = template_paused.format(
-                template_fillnan.format(
-                    template_norm.format(template_if.format("$close", price_field), template_fillnan.format("$close"))
-                )
-            )
-            return feature_ops
-
-        def get_normalized_vwap_price_feature(price_field, shift=0):
-            # norm with the close price of 237th minute of yesterday.
-            if shift == 0:
-                template_norm = "{0}/DayLast(Ref({1}, 243))"
-            else:
-                template_norm = "Ref({0}, " + str(shift) + ")/DayLast(Ref({1}, 243))"
-
-            template_fillnan = "FFillNan({0})"
-            # calculate -> ffill -> remove paused
-            feature_ops = template_paused.format(
-                template_fillnan.format(
-                    template_norm.format(
-                        template_if.format("$close", template_ifinf.format("$close", price_field)),
-                        template_fillnan.format("$close"),
-                    )
-                )
-            )
-            return feature_ops
-
-        fields += [get_normalized_price_feature("$open", 0)]
-        fields += [get_normalized_price_feature("$high", 0)]
-        fields += [get_normalized_price_feature("$low", 0)]
-        fields += [get_normalized_price_feature("$close", 0)]
-        fields += [get_normalized_vwap_price_feature("$vwap", 0)]
-        names += ["$open", "$high", "$low", "$close", "$vwap"]
-
-        fields += [get_normalized_price_feature("$open", 240)]
-        fields += [get_normalized_price_feature("$high", 240)]
-        fields += [get_normalized_price_feature("$low", 240)]
-        fields += [get_normalized_price_feature("$close", 240)]
-        fields += [get_normalized_vwap_price_feature("$vwap", 240)]
-        names += ["$open_1", "$high_1", "$low_1", "$close_1", "$vwap_1"]
-
-        fields += [get_normalized_price_feature("$bid", 0)]
-        fields += [get_normalized_price_feature("$ask", 0)]
-        names += ["$bid", "$ask"]
-
-        fields += [get_normalized_price_feature("$bid", 240)]
-        fields += [get_normalized_price_feature("$ask", 240)]
-        names += ["$bid_1", "$ask_1"]
-
-        # calculate and fill nan with 0
-
-        def get_volume_feature(volume_field, shift=0):
-            template_gzero = "If(Ge({0}, 0), {0}, 0)"
-            if shift == 0:
-                feature_ops = template_gzero.format(
-                    template_paused.format(
-                        "If(IsInf({0}), 0, {0})".format(
-                            "If(IsNull({0}), 0, {0})".format(
-                                "{0}/Ref(DayLast(Mean({0}, 7200)), 240)".format(volume_field)
-                            )
-                        )
-                    )
-                )
-            else:
-                feature_ops = template_gzero.format(
-                    template_paused.format(
-                        "If(IsInf({0}), 0, {0})".format(
-                            "If(IsNull({0}), 0, {0})".format(
-                                f"Ref({{0}}, {shift})/Ref(DayLast(Mean({{0}}, 7200)), 240)".format(volume_field)
-                            )
-                        )
-                    )
-                )
-            return feature_ops
-
-        fields += [get_volume_feature("$volume", 0)]
-        names += ["$volume"]
-
-        fields += [get_volume_feature("$volume", 240)]
-        names += ["$volume_1"]
-
-        fields += [get_volume_feature("$bidV", 0)]
-        fields += [get_volume_feature("$bidV1", 0)]
-        fields += [get_volume_feature("$bidV3", 0)]
-        fields += [get_volume_feature("$bidV5", 0)]
-        fields += [get_volume_feature("$askV", 0)]
-        fields += [get_volume_feature("$askV1", 0)]
-        fields += [get_volume_feature("$askV3", 0)]
-        fields += [get_volume_feature("$askV5", 0)]
-        names += ["$bidV", "$bidV1", "$bidV3", "$bidV5", "$askV", "$askV1", "$askV3", "$askV5"]
-
-        fields += [get_volume_feature("$bidV", 240)]
-        fields += [get_volume_feature("$bidV1", 240)]
-        fields += [get_volume_feature("$bidV3", 240)]
-        fields += [get_volume_feature("$bidV5", 240)]
-        fields += [get_volume_feature("$askV", 240)]
-        fields += [get_volume_feature("$askV1", 240)]
-        fields += [get_volume_feature("$askV3", 240)]
-        fields += [get_volume_feature("$askV5", 240)]
-        names += ["$bidV_1", "$bidV1_1", "$bidV3_1", "$bidV5_1", "$askV_1", "$askV1_1", "$askV3_1", "$askV5_1"]
-
-        return fields, names
-
-
-class HighFreqBacktestOrderHandler(DataHandler):
-    def __init__(
-        self,
-        instruments="csi300",
-        start_time=None,
-        end_time=None,
-    ):
-        data_loader = {
-            "class": "QlibDataLoader",
-            "kwargs": {
-                "config": self.get_feature_config(),
-                "swap_level": False,
-                "freq": "1min",
-            },
-        }
-        super().__init__(
-            instruments=instruments,
-            start_time=start_time,
-            end_time=end_time,
-            data_loader=data_loader,
-        )
-
     def get_feature_config(self):
         fields = []
         names = []
 
         template_if = "If(IsNull({1}), {0}, {1})"
         template_paused = "Select(Gt($hx_paused_num, 1.001), {0})"
+        # template_paused = "{0}"
         template_fillnan = "FFillNan({0})"
         fields += [
             template_fillnan.format(template_paused.format("$close")),
@@ -499,34 +158,7 @@ class HighFreqBacktestOrderHandler(DataHandler):
         fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$volume"))]
         names += ["$volume0"]
 
-        fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$bid"))]
-        names += ["$bid0"]
-
-        fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$bidV"))]
-        names += ["$bidV0"]
-
-        fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$ask"))]
-        names += ["$ask0"]
-
-        fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$askV"))]
-        names += ["$askV0"]
-
-        fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("($bid + $ask) / 2"))]
-        names += ["$median0"]
-
         fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$factor"))]
         names += ["$factor0"]
 
-        fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$downlimitmarket"))]
-        names += ["$downlimitmarket0"]
-
-        fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$uplimitmarket"))]
-        names += ["$uplimitmarket0"]
-
-        fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$highmarket"))]
-        names += ["$highmarket0"]
-
-        fields += [template_paused.format("If(IsNull({0}), 0, {0})".format("$lowmarket"))]
-        names += ["$lowmarket0"]
-
         return fields, names

qlib/contrib/data/highfreq_provider.py

@@ -4,7 +4,6 @@ import datetime
 from typing import Optional
 
 import qlib
-from qlib import get_module_logger
 from qlib.data import D
 from qlib.config import REG_CN
 from qlib.utils import init_instance_by_config
@@ -13,6 +12,7 @@ from qlib.data.data import Cal
 from qlib.contrib.ops.high_freq import get_calendar_day, DayLast, FFillNan, BFillNan, Date, Select, IsNull, IsInf, Cut
 import pickle as pkl
 from joblib import Parallel, delayed
+from utilsd.logging import print_log
 
 
 class HighFreqProvider:
@@ -41,7 +41,6 @@ class HighFreqProvider:
         self.label_conf = label_conf
         self.backtest_conf = backtest_conf
         self.qlib_conf = qlib_conf
-        self.logger = get_module_logger("HighFreqProvider")
 
     def get_pre_datasets(self):
         """Generate the training, validation and test datasets for prediction
@@ -126,7 +125,7 @@ class HighFreqProvider:
             raise ValueError("Must specify the path to save the dataset.") from e
         if os.path.isfile(path):
             start = time.time()
-            self.logger.info("Dataset exists, load from disk.", __name__)
+            print_log("Dataset exists, load from disk.", __name__)
 
             # res = dataset.prepare(['train', 'valid', 'test'])
             with open(path, "rb") as f:
@@ -135,11 +134,11 @@ class HighFreqProvider:
                 res = [data[i] for i in datasets]
             else:
                 res = data.prepare(datasets)
-            self.logger.info(f"Data loaded, time cost: {time.time() - start:.2f}", __name__)
+            print_log(f"Data loaded, time cost: {time.time() - start:.2f}", __name__)
         else:
             if not os.path.exists(os.path.dirname(path)):
                 os.makedirs(os.path.dirname(path))
-            self.logger.info("Generating dataset", __name__)
+            print_log("Generating dataset", __name__)
             start_time = time.time()
             self._prepare_calender_cache()
             dataset = init_instance_by_config(config)
@@ -158,7 +157,7 @@ class HighFreqProvider:
             with open(path[:-4] + "test.pkl", "wb") as f:
                 pkl.dump(testset, f)
             res = [data[i] for i in datasets]
-            self.logger.info(f"Data generated, time cost: {(time.time() - start_time):.2f}", __name__)
+            print_log(f"Data generated, time cost: {(time.time() - start_time):.2f}", __name__)
         return res
 
     def _gen_data(self, config, datasets=["train", "valid", "test"]):
@@ -168,7 +167,7 @@ class HighFreqProvider:
             raise ValueError("Must specify the path to save the dataset.") from e
         if os.path.isfile(path):
             start = time.time()
-            self.logger.info("Dataset exists, load from disk.", __name__)
+            print_log("Dataset exists, load from disk.", __name__)
 
             # res = dataset.prepare(['train', 'valid', 'test'])
             with open(path, "rb") as f:
@@ -177,18 +176,18 @@ class HighFreqProvider:
                 res = [data[i] for i in datasets]
             else:
                 res = data.prepare(datasets)
-            self.logger.info(f"Data loaded, time cost: {time.time() - start:.2f}", __name__)
+            print_log(f"Data loaded, time cost: {time.time() - start:.2f}", __name__)
         else:
             if not os.path.exists(os.path.dirname(path)):
                 os.makedirs(os.path.dirname(path))
-            self.logger.info("Generating dataset", __name__)
+            print_log("Generating dataset", __name__)
             start_time = time.time()
             self._prepare_calender_cache()
             dataset = init_instance_by_config(config)
             dataset.config(dump_all=True, recursive=True)
             dataset.to_pickle(path)
             res = dataset.prepare(datasets)
-            self.logger.info(f"Data generated, time cost: {(time.time() - start_time):.2f}", __name__)
+            print_log(f"Data generated, time cost: {(time.time() - start_time):.2f}", __name__)
         return res
 
     def _gen_dataset(self, config):
@@ -198,21 +197,21 @@ class HighFreqProvider:
             raise ValueError("Must specify the path to save the dataset.") from e
         if os.path.isfile(path):
             start = time.time()
-            self.logger.info("Dataset exists, load from disk.", __name__)
+            print_log("Dataset exists, load from disk.", __name__)
 
             with open(path, "rb") as f:
                 dataset = pkl.load(f)
-            self.logger.info(f"Data loaded, time cost: {time.time() - start:.2f}", __name__)
+            print_log(f"Data loaded, time cost: {time.time() - start:.2f}", __name__)
         else:
             start = time.time()
             if not os.path.exists(os.path.dirname(path)):
                 os.makedirs(os.path.dirname(path))
-            self.logger.info("Generating dataset", __name__)
+            print_log("Generating dataset", __name__)
             self._prepare_calender_cache()
             dataset = init_instance_by_config(config)
-            self.logger.info(f"Dataset init, time cost: {time.time() - start:.2f}", __name__)
+            print_log(f"Dataset init, time cost: {time.time() - start:.2f}", __name__)
             dataset.prepare(["train", "valid", "test"])
-            self.logger.info(f"Dataset prepared, time cost: {time.time() - start:.2f}", __name__)
+            print_log(f"Dataset prepared, time cost: {time.time() - start:.2f}", __name__)
             dataset.config(dump_all=True, recursive=True)
             dataset.to_pickle(path)
         return dataset
@@ -225,15 +224,15 @@ class HighFreqProvider:
 
         if os.path.isfile(path + "tmp_dataset.pkl"):
             start = time.time()
-            self.logger.info("Dataset exists, load from disk.", __name__)
+            print_log("Dataset exists, load from disk.", __name__)
         else:
             start = time.time()
             if not os.path.exists(os.path.dirname(path)):
                 os.makedirs(os.path.dirname(path))
-            self.logger.info("Generating dataset", __name__)
+            print_log("Generating dataset", __name__)
             self._prepare_calender_cache()
             dataset = init_instance_by_config(config)
-            self.logger.info(f"Dataset init, time cost: {time.time() - start:.2f}", __name__)
+            print_log(f"Dataset init, time cost: {time.time() - start:.2f}", __name__)
             dataset.config(dump_all=False, recursive=True)
             dataset.to_pickle(path + "tmp_dataset.pkl")
 
@@ -266,15 +265,15 @@ class HighFreqProvider:
 
         if os.path.isfile(path + "tmp_dataset.pkl"):
             start = time.time()
-            self.logger.info("Dataset exists, load from disk.", __name__)
+            print_log("Dataset exists, load from disk.", __name__)
         else:
             start = time.time()
             if not os.path.exists(os.path.dirname(path)):
                 os.makedirs(os.path.dirname(path))
-            self.logger.info("Generating dataset", __name__)
+            print_log("Generating dataset", __name__)
             self._prepare_calender_cache()
             dataset = init_instance_by_config(config)
-            self.logger.info(f"Dataset init, time cost: {time.time() - start:.2f}", __name__)
+            print_log(f"Dataset init, time cost: {time.time() - start:.2f}", __name__)
             dataset.config(dump_all=False, recursive=True)
             dataset.to_pickle(path + "tmp_dataset.pkl")