docs: replace broken RD-Agent demo links in README (#2150)

Co-authored-by: monkeyjack123 <monkeyjack123@users.noreply.github.com>
Co-authored-by: Linlang Lv (iSoftStone Information) <v-llv@microsoft.com>

.commitlintrc.js

@@ -0,0 +1,21 @@
+module.exports = {
+    extends: ["@commitlint/config-conventional"],
+    rules: {
+        // Configuration Format: [level, applicability, value]
+        // level: Error level, usually expressed as a number:
+        //     0 - disable rule
+        //     1 - Warning (does not prevent commits)
+        //     2 - Error (will block the commit)
+        // applicability: the conditions under which the rule applies, commonly used values:
+        //     “always” - always apply the rule
+        //     “never” - never apply the rule
+        // value: the specific value of the rule, e.g. a maximum length of 100.
+        // Refs: https://commitlint.js.org/reference/rules-configuration.html
+      "header-max-length": [2, "always", 100],
+      "type-enum": [
+        2,
+        "always",
+        ["build", "chore", "ci", "docs", "feat", "fix", "perf", "refactor", "revert", "style", "test", "Release-As"]
+      ]
+    }
+  };

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,16 @@
+<!--- Thank you for submitting a Pull Request! In order to make our work smoother. -->
+<!--- please make sure your Pull Request meets the following requirements: -->
+<!---   1. Provide a general summary of your changes in the Title above; -->
+<!---   2. Add appropriate prefixes to titles, such as `build:`, `chore:`, `ci:`, `docs:`, `feat:`, `fix:`, `perf:`, `refactor:`, `revert:`, `style:`, `test:`(Ref: https://www.conventionalcommits.org/). -->
+<!--- Category: -->
+<!--- Patch Updates: `fix:` -->
+<!---   Example: fix(auth): correct login validation issue -->
+<!--- minor update (introduces new functionality): `feat` -->
+<!---   Example: feature(parser): add ability to parse arrays -->
+<!--- major update(destructive update): Include BREAKING CHANGE in the commit message footer, or add `! ` in the commit footer to indicate that there is a destructive update. -->
+<!---   Example: feat(auth)! : remove support for old authentication method -->
+<!--- Other updates: `build:`, `chore:`, `ci:`, `docs:`, `perf:`, `refactor:`, `revert:`, `style:`, `test:`. -->
+
 <!--- Provide a general summary of your changes in the Title above -->
 
 ## Description

.github/labeler.yml

@@ -1,6 +0,0 @@
-documentation:
-- 'docs/**/*'
-- '**/*.md'
-
-waiting for triage:
-- any: ['**/*', '!docs/**/*', '!**/*.md']

.github/workflows/labeler.yml

@@ -1,14 +0,0 @@
-name: "Add label automatically"
-on:
-- pull_request_target
-
-jobs:
-  triage:
-    permissions:
-      contents: read
-      pull-requests: write
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/labeler@v4
-      with:
-        repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/lint_title.yml

@@ -0,0 +1,35 @@
+name: Lint pull request title
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - edited
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  lint-title:
+    runs-on: ubuntu-latest
+    steps:
+      # This step is necessary because the lint title uses the .commitlintrc.js file in the project root directory.
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '16'
+
+      - name: Install commitlint
+        run: npm install --save-dev @commitlint/{config-conventional,cli}
+
+      - name: Validate PR Title with commitlint
+        env:
+          BODY: ${{ github.event.pull_request.title }}
+        run: |
+          echo "$BODY" | npx commitlint --config .commitlintrc.js

.github/workflows/python-publish.yml

@@ -1,65 +0,0 @@
-# This workflows will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
-name: Upload Python Package
-
-on:
-  release:
-    types: [published]
-
-jobs:
-  deploy_with_bdist_wheel:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [windows-latest, macos-13, macos-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-        exclude:
-          - os: macos-13
-            python-version: "3.11"
-          - os: macos-13
-            python-version: "3.12"
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        make dev
-    - name: Build wheel on ${{ matrix.os }}
-      run: |
-        make build
-    - name: Upload to PyPi
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-      run: |
-        twine check dist/*.whl
-        twine upload dist/*.whl --verbose
-
-  deploy_with_manylinux:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Build wheel on Linux
-      uses: RalfG/python-wheels-manylinux-build@v0.7.1-manylinux2014_x86_64
-      with:
-        python-versions: 'cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312'
-        build-requirements: 'numpy cython'
-    - name: Install dependencies
-      run: |
-        python -m pip install twine
-    - name: Upload to PyPi
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-      run: |
-        twine check dist/pyqlib-*-manylinux*.whl
-        twine upload dist/pyqlib-*-manylinux*.whl --verbose

.github/workflows/release-drafter.yml

@@ -1,22 +0,0 @@
-name: Release Drafter
-
-on:
-  push:
-    # branches to consider in the event; optional, defaults to all
-    branches:
-      - main
-
-permissions:
-  contents: read
-
-jobs:
-  update_release_draft:
-    permissions:
-      contents: write
-      pull-requests: read
-    runs-on: ubuntu-latest
-    steps:
-      # Drafts your next Release notes as Pull Requests are merged into "master"
-      - uses: release-drafter/release-drafter@v5.11.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,107 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    outputs:
+      release_created: ${{ steps.release_please.outputs.release_created }}
+
+    steps:
+      - name: Release please
+        id: release_please
+        uses: googleapis/release-please-action@v4
+        with:
+          token: ${{ secrets.PAT }}
+          release-type: simple
+
+  deploy_with_manylinux:
+    needs: release
+    permissions:
+      contents: write
+      pull-requests: read
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        if: needs.release.outputs.release_created == 'true'
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python ${{ matrix.python-version }}
+        if: needs.release.outputs.release_created == 'true'
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Build wheel on Linux
+        if: needs.release.outputs.release_created == 'true'
+        uses: RalfG/python-wheels-manylinux-build@v0.7.1-manylinux2014_x86_64
+        with:
+          python-versions: 'cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312'
+          build-requirements: 'numpy cython'
+
+      - name: Install dependencies
+        if: needs.release.outputs.release_created == 'true'
+        run: |
+          python -m pip install twine
+
+      - name: Upload to PyPi
+        if: needs.release.outputs.release_created == 'true'
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.TESTPYPI }}
+        run: |
+          twine check dist/pyqlib-*-manylinux*.whl
+          twine upload --repository-url https://test.pypi.org/legacy/ dist/pyqlib-*-manylinux*.whl --verbose
+
+  deploy_with_bdist_wheel:
+    needs: release
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # After testing, the whl files of pyqlib built by macos-14 and macos-15 in python environments of 3.8, 3.9, 3.10, 3.11, 3.12,
+        # the filenames are exactly duplicated, which will result in the duplicated whl files not being able to be uploaded to pypi,
+        # so we chose to just keep the latest macos-latest. macos-latest currently points to macos-15.
+        # Also, macos-13 will stop being supported on 2025-11-14.
+        # Refs: https://github.blog/changelog/2025-07-11-upcoming-changes-to-macos-hosted-runners-macos-latest-migration-and-xcode-support-policy-updates/
+        os: [windows-latest, macos-latest]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+        if: needs.release.outputs.release_created == 'true'
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python ${{ matrix.python-version }}
+        if: needs.release.outputs.release_created == 'true'
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        if: needs.release.outputs.release_created == 'true'
+        run: |
+          make dev
+
+      - name: Build wheel on ${{ matrix.os }}
+        if: needs.release.outputs.release_created == 'true'
+        run: |
+          make build
+
+      - name: Upload to PyPi
+        if: needs.release.outputs.release_created == 'true'
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.TESTPYPI }}
+        run: |
+          twine check dist/*.whl
+          twine upload --repository-url https://test.pypi.org/legacy/ dist/*.whl --verbose

.github/workflows/test_qlib_from_pip.yml

@@ -1,5 +1,9 @@
 name: Test qlib from pip
 
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
 on:
   push:
     branches: [ main ]
@@ -13,7 +17,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-13, macos-14, macos-15]
+        os: [windows-latest, ubuntu-24.04, ubuntu-22.04, macos-14, macos-15]
         # In github action, using python 3.7, pip install will not match the latest version of the package.
         # Also, python 3.7 is no longer supported from macos-14, and will be phased out from macos-13 in the near future.
         # All things considered, we have removed python 3.7.
@@ -21,7 +25,9 @@ jobs:
 
     steps:
     - name: Test qlib from pip
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
@@ -31,26 +37,22 @@ jobs:
     - name: Update pip to the latest version
       run: |
         python -m pip install --upgrade pip
-
+      
     - name: Qlib installation test
       run: |
         python -m pip install pyqlib
 
     - name: Install Lightgbm for MacOS
-      if: ${{ matrix.os == 'macos-13' || matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
+      if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
       run: |
-        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
-        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
-        # FIX MacOS error: Segmentation fault
-        # reference: https://github.com/microsoft/LightGBM/issues/4229
-        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
-        brew unlink libomp
-        brew install libomp.rb
+        brew update
+        brew install libomp || brew reinstall libomp
+        python -m pip install --no-binary=:all: lightgbm
 
     - name: Downloads dependencies data
       run: |
         cd ..
-        python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn
+        python -m qlib.cli.data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn
         cd qlib
 
     - name: Test workflow by config

.github/workflows/test_qlib_from_source.yml

@@ -1,5 +1,9 @@
 name: Test qlib from source
 
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
 on:
   push:
     branches: [ main ]
@@ -14,7 +18,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-13, macos-14, macos-15]
+        os: [windows-latest, ubuntu-24.04, ubuntu-22.04, macos-14, macos-15]
         # In github action, using python 3.7, pip install will not match the latest version of the package.
         # Also, python 3.7 is no longer supported from macos-14, and will be phased out from macos-13 in the near future.
         # All things considered, we have removed python 3.7.
@@ -22,7 +26,9 @@ jobs:
 
     steps:
     - name: Test qlib from source
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
@@ -34,12 +40,12 @@ jobs:
         python -m pip install --upgrade pip
 
     - name: Installing pytorch for macos
-      if: ${{ matrix.os == 'macos-13' || matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
+      if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
       run: |
         python -m pip install torch torchvision torchaudio
 
     - name: Installing pytorch for ubuntu
-      if: ${{ matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04' }}
+      if: ${{ matrix.os == 'ubuntu-24.04' || matrix.os == 'ubuntu-22.04' }}
       run: |
         python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
 
@@ -74,8 +80,11 @@ jobs:
       run: |
         make mypy
     
+    # Due to issues that cannot be automatically fixed when running `nbqa black . -l 120 --check --diff` on Jupyter notebooks,
+    # we reverted to a version of `black` earlier than 26.1.0 before performing the checks.
     - name: Check Qlib ipynb with nbqa
       run: |
+        python -m pip install "black<26.1"
         make nbqa
 
     - name: Test data downloads
@@ -84,15 +93,11 @@ jobs:
         python scripts/get_data.py download_data --file_name rl_data.zip --target_dir tests/.data/rl
 
     - name: Install Lightgbm for MacOS
-      if: ${{ matrix.os == 'macos-13' || matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
+      if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
       run: |
-        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
-        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
-        # FIX MacOS error: Segmentation fault
-        # reference: https://github.com/microsoft/LightGBM/issues/4229
-        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
-        brew unlink libomp
-        brew install libomp.rb
+        brew update
+        brew install libomp || brew reinstall libomp
+        python -m pip install --no-binary=:all: lightgbm
 
     - name: Check Qlib ipynb with nbconvert
       run: |
@@ -101,9 +106,26 @@ jobs:
     - name: Test workflow by config (install from source)
       run: |
         python -m pip install numba
-        python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+        python qlib/cli/run.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
 
-    - name: Unit tests with Pytest
+    - name: Unit tests with Pytest (MacOS)
+      if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
+      uses: nick-fields/retry@v2
+      with:
+        timeout_minutes: 60
+        max_attempts: 3
+        command: |
+          # Limit the number of threads in various libraries to prevent Segmentation faults caused by OpenMP multithreading conflicts under macOS.
+          export OMP_NUM_THREADS=1  # Limit the number of OpenMP threads
+          export MKL_NUM_THREADS=1  # Limit the number of Intel MKL threads
+          export NUMEXPR_NUM_THREADS=1  # Limit the number of NumExpr threads
+          export OPENBLAS_NUM_THREADS=1  # Limit the number of OpenBLAS threads
+          export VECLIB_MAXIMUM_THREADS=1  # Limit the number of macOS Accelerate/vecLib threads
+          cd tests
+          python -m pytest . -m "not slow" --durations=0
+
+    - name: Unit tests with Pytest (Ubuntu and Windows)
+      if: ${{ matrix.os != 'macos-13' && matrix.os != 'macos-14' && matrix.os != 'macos-15' }}
       uses: nick-fields/retry@v2
       with:
         timeout_minutes: 60

.github/workflows/test_qlib_from_source_slow.yml

@@ -1,5 +1,9 @@
 name: Test qlib from source slow
 
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
 on:
   push:
     branches: [ main ]
@@ -14,7 +18,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-13, macos-14, macos-15]
+        os: [windows-latest, ubuntu-24.04, ubuntu-22.04, macos-14, macos-15]
         # In github action, using python 3.7, pip install will not match the latest version of the package.
         # Also, python 3.7 is no longer supported from macos-14, and will be phased out from macos-13 in the near future.
         # All things considered, we have removed python 3.7.
@@ -22,7 +26,9 @@ jobs:
 
     steps:
     - name: Test qlib from source slow
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
@@ -38,15 +44,11 @@ jobs:
         python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
 
     - name: Install Lightgbm for MacOS
-      if: ${{ matrix.os == 'macos-13' || matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
+      if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
       run: |
-        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
-        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
-        # FIX MacOS error: Segmentation fault
-        # reference: https://github.com/microsoft/LightGBM/issues/4229
-        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
-        brew unlink libomp
-        brew install libomp.rb
+        brew update
+        brew install libomp || brew reinstall libomp
+        python -m pip install --no-binary=:all: lightgbm
 
     - name: Unit tests with Pytest
       uses: nick-fields/retry@v2

.gitignore

@@ -22,6 +22,7 @@ dist/
 qlib/VERSION.txt
 qlib/data/_libs/expanding.cpp
 qlib/data/_libs/rolling.cpp
+qlib/_version.py
 examples/estimator/estimator_example/
 examples/rl/data/
 examples/rl/checkpoints/

Makefile

@@ -12,6 +12,12 @@ PUBLIC_DIR := $(shell [ "$$READTHEDOCS" = "True" ] && echo "$$READTHEDOCS_OUTPUT
 SO_DIR := qlib/data/_libs
 SO_FILES := $(wildcard $(SO_DIR)/*.so)
 
+ifeq ($(OS),Windows_NT)
+    IS_WINDOWS = true
+else
+    IS_WINDOWS = false
+endif
+
 ########################################################################################
 # Development Environment Management
 ########################################################################################
@@ -48,6 +54,10 @@ deepclean: clean
 # What this code does is compile two Cython modules, rolling and expanding, using setuptools and Cython,
 # and builds them as binary expansion modules that can be imported directly into Python.
 # Since pyproject.toml can't do that, we compile it here.
+
+# pywinpty as a dependency of jupyter on windows, if you use pip install pywinpty installation,
+# will first download the tar.gz file, and then locally compiled and installed,
+# this will lead to some unnecessary trouble, so we choose to install the compiled whl file, to avoid trouble.
 prerequisite:
 	@if [ -n "$(SO_FILES)" ]; then \
 		echo "Shared library files exist, skipping build."; \
@@ -58,36 +68,43 @@ prerequisite:
 		python -c "from setuptools import setup, Extension; from Cython.Build import cythonize; import numpy; extensions = [Extension('qlib.data._libs.rolling', ['qlib/data/_libs/rolling.pyx'], language='c++', include_dirs=[numpy.get_include()]), Extension('qlib.data._libs.expanding', ['qlib/data/_libs/expanding.pyx'], language='c++', include_dirs=[numpy.get_include()])]; setup(ext_modules=cythonize(extensions, language_level='3'), script_args=['build_ext', '--inplace'])"; \
 	fi
 
+	@if [ "$(IS_WINDOWS)" = "true" ]; then \
+		python -m pip install pywinpty --only-binary=:all:; \
+	fi
+
 # Install the package in editable mode.
 dependencies:
-	python -m pip install -e .
+	python -m pip install --no-cache-dir -e .
 
 lightgbm:
-	python -m pip install lightgbm --prefer-binary
+	python -m pip install --no-cache-dir lightgbm --prefer-binary
 
 rl:
-	python -m pip install -e .[rl]
+	python -m pip install --no-cache-dir -e .[rl]
 
 develop:
-	python -m pip install -e .[dev]
+	python -m pip install --no-cache-dir -e .[dev]
 
 lint:
-	python -m pip install -e .[lint]
+	python -m pip install --no-cache-dir -e .[lint]
 
 docs:
-	python -m pip install -e .[docs]
+	python -m pip install --no-cache-dir -e .[docs]
 
 package:
-	python -m pip install -e .[package]
+	python -m pip install --no-cache-dir -e .[package]
 
 test:
-	python -m pip install -e .[test]
+	python -m pip install --no-cache-dir -e .[test]
 
 analysis:
-	python -m pip install -e .[analysis]
+	python -m pip install --no-cache-dir -e .[analysis]
+
+client:
+	python -m pip install --no-cache-dir -e .[client]
 
 all:
-	python -m pip install -e .[dev,lint,docs,package,test,analysis,rl]
+	python -m pip install --no-cache-dir -e .[pywinpty,dev,lint,docs,package,test,analysis,rl]
 
 install: prerequisite dependencies
 
@@ -99,7 +116,7 @@ dev: prerequisite all
 
 # Check lint with black.
 black:
-	black . -l 120 --check --diff
+	black . -l 120 --check --diff --exclude qlib/_version.py
 
 # Check code folder with pylint.
 # TODO: These problems we will solve in the future. Important among them are: W0221, W0223, W0237, E1102

README.md

@@ -17,19 +17,33 @@ We are excited to announce the release of **RD-Agent**📢, a powerful tool that
 
 RD-Agent is now available on [GitHub](https://github.com/microsoft/RD-Agent), and we welcome your star🌟!
 
-To learn more, please visit our [♾️Demo page](https://rdagent.azurewebsites.net/). Here, you will find demo videos in both English and Chinese to help you better understand the scenario and usage of RD-Agent.
+To learn more, please visit the [RD-Agent repository](https://github.com/microsoft/RD-Agent). We have prepared several public demo videos for you:
 
-We have prepared several demo videos for you:
 | Scenario | Demo video (English) | Demo video (中文) |
 | --                      | ------    | ------    |
-| Quant Factor Mining | [Link](https://rdagent.azurewebsites.net/factor_loop?lang=en) | [Link](https://rdagent.azurewebsites.net/factor_loop?lang=zh) |
-| Quant Factor Mining from reports | [Link](https://rdagent.azurewebsites.net/report_factor?lang=en) | [Link](https://rdagent.azurewebsites.net/report_factor?lang=zh) |
-| Quant Model Optimization | [Link](https://rdagent.azurewebsites.net/model_loop?lang=en) | [Link](https://rdagent.azurewebsites.net/model_loop?lang=zh) |
+| Quant Factor Mining | [YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s) | [YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s) |
+| Quant Factor Mining from reports | [YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c) | [YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c) |
+| Quant Model Optimization | [YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s) | [YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s) |
+
+- 📃**Paper**: [R&D-Agent-Quant: A Multi-Agent Framework for Data-Centric Factors and Model Joint Optimization](https://arxiv.org/abs/2505.15155)
+- 👾**Code**: https://github.com/microsoft/RD-Agent/
+```BibTeX
+@misc{li2025rdagentquant,
+    title={R\&D-Agent-Quant: A Multi-Agent Framework for Data-Centric Factors and Model Joint Optimization},
+    author={Yuante Li and Xu Yang and Xiao Yang and Minrui Xu and Xisen Wang and Weiqing Liu and Jiang Bian},
+    year={2025},
+    eprint={2505.15155},
+    archivePrefix={arXiv},
+    primaryClass={cs.AI}
+}
+```
+![image](https://github.com/user-attachments/assets/3198bc10-47ba-4ee0-8a8e-46d5ce44f45d)
 
 ***
 
 | Feature | Status |
 | --                      | ------    |
+| [R&D-Agent-Quant](https://arxiv.org/abs/2505.15155) Published | Apply R&D-Agent to Qlib for quant trading | 
 | BPQP for End-to-end learning | 📈Coming soon!([Under review](https://github.com/microsoft/qlib/pull/1863)) |
 | 🔥LLM-driven Auto Quant Factory🔥 | 🚀 Released in [♾️RD-Agent](https://github.com/microsoft/RD-Agent) on Aug 8, 2024 |
 | KRNN and Sandwich models | :chart_with_upwards_trend: [Released](https://github.com/microsoft/qlib/pull/1414/) on May 26, 2023 |
@@ -164,7 +178,6 @@ This table demonstrates the supported Python version of `Qlib`:
 **Note**: 
 1. **Conda** is suggested for managing your Python environment. In some cases, using Python outside of a `conda` environment may result in missing header files, causing the installation failure of certain packages.
 2. Please pay attention that installing cython in Python 3.6 will raise some error when installing ``Qlib`` from source. If users use Python 3.6 on their machines, it is recommended to *upgrade* Python to version 3.8 or higher, or use `conda`'s Python to install ``Qlib`` from source.
-3. For Python 3.9, `Qlib` supports running workflows such as training models, doing backtest and plot most of the related figures (those included in [notebook](examples/workflow_by_code.ipynb)). However, plotting for the *model performance* is not supported for now and we will fix this when the dependent packages are upgraded in the future.
 
 ### Install with pip
 Users can easily install ``Qlib`` by pip according to the following command.
@@ -196,10 +209,10 @@ Also, users can install the latest dev version ``Qlib`` by the source code accor
 **Tips for Mac**: If you are using Mac with M1, you might encounter issues in building the wheel for LightGBM, which is due to missing dependencies from OpenMP. To solve the problem, install openmp first with ``brew install libomp`` and then run ``pip install .`` to build it successfully. 
 
 ## Data Preparation
-❗ Due to more restrict data security policy. The offical dataset is disabled temporarily. You can try [this data source](https://github.com/chenditc/investment_data/releases) contributed by the community.
-Here is an example to download the data updated on 20240809.
+❗ Due to more restrict data security policy. The official dataset is disabled temporarily. You can try [this data source](https://github.com/chenditc/investment_data/releases) contributed by the community.
+Here is an example to download the latest data.
 ```bash
-wget https://github.com/chenditc/investment_data/releases/download/2024-08-09/qlib_bin.tar.gz
+wget https://github.com/chenditc/investment_data/releases/latest/download/qlib_bin.tar.gz
 mkdir -p ~/.qlib/qlib_data/cn_data
 tar -zxvf qlib_bin.tar.gz -C ~/.qlib/qlib_data/cn_data --strip-components=1
 rm -f qlib_bin.tar.gz
@@ -215,10 +228,10 @@ Load and prepare data by running the following code:
 ### Get with module
   ```bash
   # get 1d data
-  python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn
+  python -m qlib.cli.data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn
 
   # get 1min data
-  python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data_1min --region cn --interval 1min
+  python -m qlib.cli.data qlib_data --target_dir ~/.qlib/qlib_data/cn_data_1min --region cn --interval 1min
 
   ```
 
@@ -265,6 +278,16 @@ We recommend users to prepare their own data if they have a high-quality dataset
       * *trading_date*: start of trading day
       * *end_date*: end of trading day(not included)
 
+### Checking the health of the data
+  * We provide a script to check the health of the data, you can run the following commands to check whether the data is healthy or not.
+    ```
+    python scripts/check_data_health.py check_data --qlib_dir ~/.qlib/qlib_data/cn_data
+    ```
+  * Of course, you can also add some parameters to adjust the test results, such as this.
+    ```
+    python scripts/check_data_health.py check_data --qlib_dir ~/.qlib/qlib_data/cn_data --missing_data_num 30055 --large_step_threshold_volume 94485 --large_step_threshold_price 20
+    ```
+  * If you want more information about `check_data_health`, please refer to the [documentation](https://qlib.readthedocs.io/en/latest/component/data.html#checking-the-health-of-the-data).
 
 <!-- 
 - Run the initialization code and get stock data:
@@ -300,12 +323,12 @@ We recommend users to prepare their own data if they have a high-quality dataset
     ```
 2. Start a new Docker container
     ```bash
-    docker run -it --name <container name> -v <Mounted local directory>:/app qlib_image_stable
+    docker run -it --name <container name> -v <Mounted local directory>:/app pyqlib/qlib_image_stable:stable
     ```
 3. At this point you are in the docker environment and can run the qlib scripts. An example:
     ```bash
     >>> python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
-    >>> python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+    >>> python qlib/cli/run.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
     ```
 4. Exit the container
     ```bash
@@ -335,9 +358,9 @@ Qlib provides a tool named `qrun` to run the whole workflow automatically (inclu
     ```
     If users want to use `qrun` under debug mode, please use the following command:
     ```bash
-    python -m pdb qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+    python -m pdb qlib/cli/run.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
     ```
-    The result of `qrun` is as follows, please refer to [Intraday Trading](https://qlib.readthedocs.io/en/latest/component/backtest.html) for more details about the result. 
+    The result of `qrun` is as follows, please refer to [docs](https://qlib.readthedocs.io/en/latest/component/strategy.html#result) for more explanations about the result. 
 
     ```bash
 
@@ -453,6 +476,14 @@ python run_all_model.py run 10
 
 It also provides the API to run specific models at once. For more use cases, please refer to the file's [docstrings](examples/run_all_model.py). 
 
+### Break change
+In `pandas`, `group_key` is one of the parameters of the `groupby` method. From version 1.5 to 2.0 of `pandas`, the default value of `group_key` has been changed from `no default` to `True`, which will cause qlib to report an error during operation. So we set `group_key=False`, but it doesn't guarantee that some programmes will run correctly, including:
+* qlib\examples\rl_order_execution\scripts\gen_training_orders.py
+* qlib\examples\benchmarks\TRA\src\dataset.MTSDatasetH.py
+* qlib\examples\benchmarks\TFT\tft.py
+
+
+
 ## [Adapting to Market Dynamics](examples/benchmarks_dynamic)
 
 Due to the non-stationary nature of the environment of the financial market, the data distribution may change in different periods, which makes the performance of models build on training data decays in the future test data.
@@ -588,7 +619,7 @@ You can find some impefect implementation in Qlib by  `rg 'TODO|FIXME' qlib`
  
 If you would like to become one of Qlib's maintainers to contribute more (e.g. help merge PR, triage issues), please contact us by email([qlib@microsoft.com](mailto:qlib@microsoft.com)).  We are glad to help to upgrade your permission.
 
-## Licence
+## License
 Most contributions require you to agree to a
 Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
 the right to use your contribution. For details, visit https://cla.opensource.microsoft.com.

docs/advanced/alpha.rst

@@ -42,7 +42,7 @@ Example
 
     .. math::
 
-        DEA = \frac{EMA(DIF, 9)}{CLOSE}
+        DEA = EMA(DIF, 9)
 
 Users can use ``Data Handler`` to build formulaic alphas `MACD` in qlib:
 
@@ -51,7 +51,7 @@ Users can use ``Data Handler`` to build formulaic alphas `MACD` in qlib:
 .. code-block:: python
 
     >> from qlib.data.dataset.loader import QlibDataLoader
-    >> MACD_EXP = '(EMA($close, 12) - EMA($close, 26))/$close - EMA((EMA($close, 12) - EMA($close, 26))/$close, 9)/$close'
+    >> MACD_EXP = '2 * ((EMA($close, 12) - EMA($close, 26))/$close - EMA((EMA($close, 12) - EMA($close, 26))/$close, 9))'
     >> fields = [MACD_EXP] # MACD
     >> names = ['MACD']
     >> labels = ['Ref($close, -2)/Ref($close, -1) - 1'] # label
@@ -66,17 +66,17 @@ Users can use ``Data Handler`` to build formulaic alphas `MACD` in qlib:
                             feature     label
                                MACD     LABEL
     datetime   instrument
-    2010-01-04 SH600000   -0.011547 -0.019672
-               SH600004    0.002745 -0.014721
-               SH600006    0.010133  0.002911
-               SH600008   -0.001113  0.009818
-               SH600009    0.025878 -0.017758
+    2010-01-04 SH600000    0.008781 -0.019672
+               SH600004    0.006699 -0.014721
+               SH600006    0.005714  0.002911
+               SH600008    0.000798  0.009818
+               SH600009    0.017015 -0.017758
     ...                         ...       ...
-    2017-12-29 SZ300124    0.007306 -0.005074
-               SZ300136   -0.013492  0.056352
-               SZ300144   -0.000966  0.011853
-               SZ300251    0.004383  0.021739
-               SZ300315   -0.030557  0.012455
+    2017-12-29 SZ300124    0.015071 -0.005074
+               SZ300136   -0.015466  0.056352
+               SZ300144    0.013082  0.011853
+               SZ300251   -0.001026  0.021739
+               SZ300315   -0.007559  0.012455
 
 Reference
 =========

docs/component/data.rst

@@ -108,10 +108,10 @@ Automatic update of daily frequency data
 
 
 
-Converting CSV Format into Qlib Format
---------------------------------------
+Converting CSV and Parquet Format into Qlib Format
+--------------------------------------------------
 
-``Qlib`` has provided the script ``scripts/dump_bin.py`` to convert **any** data in CSV format into `.bin` files (``Qlib`` format) as long as they are in the correct format.
+``Qlib`` has provided the script ``scripts/dump_bin.py`` to convert **any** data in CSV or Parquet format into `.bin` files (``Qlib`` format) as long as they are in the correct format.
 
 Besides downloading the prepared demo data, users could download demo data directly from the Collector as follows for reference to the CSV format.
 Here are some example:
@@ -126,17 +126,17 @@ for 1min data:
 
     python scripts/data_collector/yahoo/collector.py download_data --source_dir ~/.qlib/stock_data/source/cn_1min --region CN --start 2021-05-20 --end 2021-05-23 --delay 0.1 --interval 1min --limit_nums 10
 
-Users can also provide their own data in CSV format. However, the CSV data **must satisfies** following criterions:
+Users can also provide their own data in CSV or Parquet format. However, the data **must satisfies** following criterions:
 
-- CSV file is named after a specific stock *or* the CSV file includes a column of the stock name
+- CSV or Parquet file is named after a specific stock *or* the CSV or Parquet file includes a column of the stock name
 
-    - Name the CSV file after a stock: `SH600000.csv`, `AAPL.csv` (not case sensitive).
+    - Name the CSV or Parquet file after a stock: `SH600000.csv`, `AAPL.csv` or `SH600000.parquet`, `AAPL.parquet` (not case sensitive).
 
-    - CSV file includes a column of the stock name. User **must** specify the column name when dumping the data. Here is an example:
+    - CSV or Parquet file includes a column of the stock name. User **must** specify the column name when dumping the data. Here is an example:
 
         .. code-block:: bash
 
-            python scripts/dump_bin.py dump_all ... --symbol_field_name symbol
+            python scripts/dump_bin.py dump_all ... --symbol_field_name symbol --file_suffix <.csv or .parquet>
 
         where the data are in the following format:
 
@@ -146,11 +146,11 @@ Users can also provide their own data in CSV format. However, the CSV data **mus
             | SH600000  | 120   |
             +-----------+-------+
 
-- CSV file **must** include a column for the date, and when dumping the data, user must specify the date column name. Here is an example:
+- CSV or Parquet file **must** include a column for the date, and when dumping the data, user must specify the date column name. Here is an example:
 
     .. code-block:: bash
 
-        python scripts/dump_bin.py dump_all ... --date_field_name date
+        python scripts/dump_bin.py dump_all ... --date_field_name date --file_suffix <.csv or .parquet>
 
     where the data are in the following format:
 
@@ -163,23 +163,23 @@ Users can also provide their own data in CSV format. However, the CSV data **mus
         +---------+------------+-------+------+----------+
 
 
-Supposed that users prepare their CSV format data in the directory ``~/.qlib/csv_data/my_data``, they can run the following command to start the conversion.
+Supposed that users prepare their CSV or Parquet format data in the directory ``~/.qlib/my_data``, they can run the following command to start the conversion.
 
 .. code-block:: bash
 
-    python scripts/dump_bin.py dump_all --csv_path  ~/.qlib/csv_data/my_data --qlib_dir ~/.qlib/qlib_data/my_data --include_fields open,close,high,low,volume,factor
+    python scripts/dump_bin.py dump_all --data_path  ~/.qlib/my_data --qlib_dir ~/.qlib/qlib_data/ --include_fields open,close,high,low,volume,factor --file_suffix <.csv or .parquet>
 
 For other supported parameters when dumping the data into `.bin` file, users can refer to the information by running the following commands:
 
 .. code-block:: bash
 
-    python dump_bin.py dump_all --help
+    python scripts/dump_bin.py dump_all --help
 
-After conversion, users can find their Qlib format data in the directory `~/.qlib/qlib_data/my_data`.
+After conversion, users can find their Qlib format data in the directory `~/.qlib/qlib_data/`.
 
 .. note::
 
-    The arguments of `--include_fields` should correspond with the column names of CSV files. The columns names of dataset provided by ``Qlib`` should include open, close, high, low, volume and factor at least.
+    The arguments of `--include_fields` should correspond with the column names of CSV or Parquet files. The columns names of dataset provided by ``Qlib`` should include open, close, high, low, volume and factor at least.
 
     - `open`
         The adjusted opening price
@@ -195,7 +195,58 @@ After conversion, users can find their Qlib format data in the directory `~/.qli
         The Restoration factor. Normally, ``factor = adjusted_price / original_price``, `adjusted price` reference: `split adjusted <https://www.investopedia.com/terms/s/splitadjusted.asp>`_
 
     In the convention of `Qlib` data processing, `open, close, high, low, volume, money and factor` will be set to NaN if the stock is suspended.
-    If you want to use your own alpha-factor which can't be calculate by OCHLV, like PE, EPS and so on, you could add it to the CSV files with OHCLV together and then dump it to the Qlib format data.
+    If you want to use your own alpha-factor which can't be calculate by OCHLV, like PE, EPS and so on, you could add it to the CSV or Parquet files with OHCLV together and then dump it to the Qlib format data.
+
+Checking the health of the data
+-------------------------------
+
+``Qlib`` provides a script to check the health of the data.
+
+- The main points to check are as follows
+
+    - Check if any data is missing in the DataFrame.
+
+    - Check if there are any large step changes above the threshold in the OHLCV columns.
+
+    - Check if any of the required columns (OLHCV) are missing in the DataFrame.
+
+    - Check if the 'factor' column is missing in the DataFrame.
+
+- You can run the following commands to check whether the data is healthy or not.
+
+    for daily data:
+        .. code-block:: bash
+
+            python scripts/check_data_health.py check_data --qlib_dir ~/.qlib/qlib_data/cn_data
+
+    for 1min data:
+        .. code-block:: bash
+
+            python scripts/check_data_health.py check_data --qlib_dir ~/.qlib/qlib_data/cn_data_1min --freq 1min
+
+- Of course, you can also add some parameters to adjust the test results.
+
+    - The available parameters are these.
+
+        - freq: Frequency of data.
+
+        - large_step_threshold_price: Maximum permitted price change
+
+        - large_step_threshold_volume: Maximum permitted volume change.
+
+        - missing_data_num: Maximum value for which data is allowed to be null.
+
+- You can run the following commands to check whether the data is healthy or not.
+
+    for daily data:
+        .. code-block:: bash
+
+            python scripts/check_data_health.py check_data --qlib_dir ~/.qlib/qlib_data/cn_data --missing_data_num 30055 --large_step_threshold_volume 94485 --large_step_threshold_price 20
+
+    for 1min data:
+        .. code-block:: bash
+
+            python scripts/check_data_health.py check_data --qlib_dir ~/.qlib/qlib_data/cn_data --freq 1min --missing_data_num 35806 --large_step_threshold_volume 3205452000000 --large_step_threshold_price 0.91
 
 Stock Pool (Market)
 -------------------

docs/component/highfreq.rst

@@ -25,7 +25,7 @@ The design of the framework is shown in the yellow part in the middle of the fig
 
 The frequency of the trading algorithm, decision content and execution environment can be customized by users (e.g. intraday trading, daily-frequency trading, weekly-frequency trading), and the execution environment can be nested with finer-grained trading algorithm and execution environment inside (i.e. sub-workflow in the figure, e.g. daily-frequency orders can be turned into finer-grained decisions by splitting orders within the day). The flexibility of the nested decision execution framework makes it easy for users to explore the effects of combining different levels of trading strategies and break down the optimization barriers between different levels of the trading algorithm.
 
-The optimization for the nested decision execution framework can be implemented with the support of `QlibRL <https://qlib.readthedocs.io/en/latest/component/rl.html>`_. To know more about how to use the QlibRL, go to API Reference: `RL API <../reference/api.html#rl>`_. 
+The optimization for the nested decision execution framework can be implemented with the support of `QlibRL <./rl/overall.html>`_. To know more about how to use the QlibRL, go to API Reference: `RL API <../reference/api.html#rl>`_. 
 
 Example
 =======

docs/component/workflow.rst

@@ -55,13 +55,16 @@ Below is a typical config file of ``qrun``.
                 n_drop: 5
                 signal: <PRED>
         backtest:
-            limit_threshold: 0.095
+            start_time: 2017-01-01
+            end_time: 2020-08-01
             account: 100000000
             benchmark: *benchmark
-            deal_price: close
-            open_cost: 0.0005
-            close_cost: 0.0015
-            min_cost: 5
+            exchange_kwargs:
+                limit_threshold: 0.095
+                deal_price: close
+                open_cost: 0.0005
+                close_cost: 0.0015
+                min_cost: 5
     task:
         model:
             class: LGBModel
@@ -107,7 +110,7 @@ If users want to use ``qrun`` under debug mode, please use the following command
 
 .. code-block:: bash
 
-    python -m pdb qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+    python -m pdb qlib/cli/run.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
 
 .. note::
 

docs/conf.py

@@ -21,8 +21,7 @@
 import os
 import sys
 
-import pkg_resources
-
+from importlib.metadata import version as ver
 
 # -- General configuration ------------------------------------------------
 
@@ -63,9 +62,9 @@ author = "Microsoft"
 # built documents.
 #
 # The short X.Y version.
-version = pkg_resources.get_distribution("pyqlib").version
+version = ver("pyqlib")
 # The full version, including alpha/beta/rc tags.
-release = pkg_resources.get_distribution("pyqlib").version
+release = version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/developer/how_to_build_image.rst

@@ -52,7 +52,7 @@ How to use qlib images
 .. code-block:: bash
 
     >>> python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
-    >>> python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
+    >>> python qlib/cli/run.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
 
 3. Exit the container
 

docs/start/getdata.rst

@@ -129,7 +129,7 @@ For example, it looks quite long and complicated:
 
 
 But using string is not the only way to implement the expression. You can also implement expression by code.
-Here is an exmaple which does the same thing as above examples.
+Here is an example which does the same thing as above examples.
 
 
 .. code-block:: python

docs/start/integration.rst

@@ -71,7 +71,7 @@ The Custom models need to inherit `qlib.model.base.Model <../reference/api.html#
                 )
 
 - Override the `predict` method
-    - The parameters must include the parameter `dataset`, which will be userd to get the test dataset.
+    - The parameters must include the parameter `dataset`, which will be used to get the test dataset.
     - Return the `prediction score`.
     - Please refer to `Model API <../reference/api.html#module-qlib.model.base>`_ for the parameter types of the fit method.
     - Code Example: In the following example, users need to use `LightGBM` to predict the label(such as `preds`) of test data `x_test` and return it.

examples/benchmarks/TFT/libs/tft_model.py

@@ -599,7 +599,7 @@ class TemporalFusionTransformer:
         print("Getting valid sampling locations.")
         valid_sampling_locations = []
         split_data_map = {}
-        for identifier, df in data.groupby(id_col):
+        for identifier, df in data.groupby(id_col, group_key=False):
             print("Getting locations for {}".format(identifier))
             num_entries = len(df)
             if num_entries >= self.time_steps:
@@ -678,7 +678,7 @@ class TemporalFusionTransformer:
         input_cols = [tup[0] for tup in self.column_definition if tup[2] not in {InputTypes.ID, InputTypes.TIME}]
 
         data_map = {}
-        for _, sliced in data.groupby(id_col):
+        for _, sliced in data.groupby(id_col, group_keys=False):
             col_mappings = {"identifier": [id_col], "time": [time_col], "outputs": [target_col], "inputs": input_cols}
 
             for k in col_mappings:

examples/benchmarks/TFT/tft.py

@@ -19,7 +19,6 @@ from qlib.model.base import ModelFT
 from qlib.data.dataset import DatasetH
 from qlib.data.dataset.handler import DataHandlerLP
 
-
 # To register new datasets, please add them here.
 ALLOW_DATASET = ["Alpha158", "Alpha360"]
 # To register new datasets, please add their configurations here.
@@ -78,13 +77,15 @@ DATASET_SETTING = {
 
 
 def get_shifted_label(data_df, shifts=5, col_shift="LABEL0"):
-    return data_df[[col_shift]].groupby("instrument").apply(lambda df: df.shift(shifts))
+    return data_df[[col_shift]].groupby("instrument", group_keys=False).apply(lambda df: df.shift(shifts))
 
 
 def fill_test_na(test_df):
     test_df_res = test_df.copy()
     feature_cols = ~test_df_res.columns.str.contains("label", case=False)
-    test_feature_fna = test_df_res.loc[:, feature_cols].groupby("datetime").apply(lambda df: df.fillna(df.mean()))
+    test_feature_fna = (
+        test_df_res.loc[:, feature_cols].groupby("datetime", group_keys=False).apply(lambda df: df.fillna(df.mean()))
+    )
     test_df_res.loc[:, feature_cols] = test_feature_fna
     return test_df_res
 

examples/benchmarks/TRA/src/dataset.py

@@ -8,7 +8,6 @@ import pandas as pd
 
 from qlib.data.dataset import DatasetH
 
-
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
 
@@ -29,7 +28,7 @@ def _create_ts_slices(index, seq_len):
     assert index.is_lexsorted(), "index should be sorted"
 
     # number of dates for each code
-    sample_count_by_codes = pd.Series(0, index=index).groupby(level=0).size().values
+    sample_count_by_codes = pd.Series(0, index=index).groupby(level=0, group_keys=False).size().values
 
     # start_index for each code
     start_index_of_codes = np.roll(np.cumsum(sample_count_by_codes), 1)

examples/benchmarks/TRA/workflow_config_tra_Alpha158.yaml

@@ -110,7 +110,6 @@ task:
         valid: [2015-01-01, 2016-12-31]
         test: [2017-01-01, 2020-08-01]
       seq_len: 60
-      horizon: 2
       input_size:
       num_states: *num_states
       batch_size: 1024

examples/benchmarks/TRA/workflow_config_tra_Alpha158_full.yaml

@@ -104,7 +104,6 @@ task:
         valid: [2015-01-01, 2016-12-31]
         test: [2017-01-01, 2020-08-01]
       seq_len: 60
-      horizon: 2
       input_size:
       num_states: *num_states
       batch_size: 1024

examples/benchmarks/TRA/workflow_config_tra_Alpha360.yaml

@@ -104,7 +104,6 @@ task:
         valid: [2015-01-01, 2016-12-31]
         test: [2017-01-01, 2020-08-01]
       seq_len: 60
-      horizon: 2
       input_size: 6
       num_states: *num_states
       batch_size: 1024

examples/benchmarks_dynamic/DDG-DA/vis_data.py

@@ -1,9 +1,10 @@
-import pickle
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
 import seaborn as sns
 
+from qlib.utils.pickle_utils import restricted_pickle_load
+
 sns.set(color_codes=True)
 plt.rcParams["font.sans-serif"] = "SimHei"
 plt.rcParams["axes.unicode_minus"] = False
@@ -18,7 +19,7 @@ from tqdm.auto import tqdm
 
 # +
 with open("./internal_data_s20.pkl", "rb") as f:
-    data = pickle.load(f)
+    data = restricted_pickle_load(f)
 
 data.data_ic_df.columns.names = ["start_date", "end_date"]
 
@@ -52,7 +53,7 @@ pd.DataFrame(meta_m.tn.twm.linear.weight.detach().numpy()).T[0].rolling(5).mean(
 
 # +
 with open("./tasks_s20.pkl", "rb") as f:
-    tasks = pickle.load(f)
+    tasks = restricted_pickle_load(f)
 
 task_df = {}
 for t in tasks:

examples/benchmarks_dynamic/README.md

@@ -7,7 +7,7 @@ The table below shows the performances of different solutions on different forec
 ## Alpha158 Dataset
 Here is the [crowd sourced version of qlib data](data_collector/crowd_source/README.md): https://github.com/chenditc/investment_data/releases
 ```bash
-wget https://github.com/chenditc/investment_data/releases/download/20220720/qlib_bin.tar.gz
+wget https://github.com/chenditc/investment_data/releases/latest/download/qlib_bin.tar.gz
 mkdir -p ~/.qlib/qlib_data/cn_data
 tar -zxvf qlib_bin.tar.gz -C ~/.qlib/qlib_data/cn_data --strip-components=2
 rm -f qlib_bin.tar.gz

examples/data_demo/data_cache_demo.py

@@ -1,8 +1,8 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 """
-    The motivation of this demo
-    - To show the data modules of Qlib is Serializable, users can dump processed data to disk to avoid duplicated data preprocessing
+The motivation of this demo
+- To show the data modules of Qlib is Serializable, users can dump processed data to disk to avoid duplicated data preprocessing
 """
 
 from copy import deepcopy

examples/data_demo/data_mem_resuse_demo.py

@@ -1,8 +1,8 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 """
-    The motivation of this demo
-    - To show the data modules of Qlib is Serializable, users can dump processed data to disk to avoid duplicated data preprocessing
+The motivation of this demo
+- To show the data modules of Qlib is Serializable, users can dump processed data to disk to avoid duplicated data preprocessing
 """
 
 from copy import deepcopy

examples/highfreq/highfreq_ops.py

@@ -25,7 +25,7 @@ class DayLast(ElemOperator):
     def _load_internal(self, instrument, start_index, end_index, freq):
         _calendar = get_calendar_day(freq=freq)
         series = self.feature.load(instrument, start_index, end_index, freq)
-        return series.groupby(_calendar[series.index]).transform("last")
+        return series.groupby(_calendar[series.index], group_keys=False).transform("last")
 
 
 class FFillNan(ElemOperator):
@@ -44,7 +44,7 @@ class FFillNan(ElemOperator):
 
     def _load_internal(self, instrument, start_index, end_index, freq):
         series = self.feature.load(instrument, start_index, end_index, freq)
-        return series.fillna(method="ffill")
+        return series.ffill()
 
 
 class BFillNan(ElemOperator):
@@ -63,7 +63,7 @@ class BFillNan(ElemOperator):
 
     def _load_internal(self, instrument, start_index, end_index, freq):
         series = self.feature.load(instrument, start_index, end_index, freq)
-        return series.fillna(method="bfill")
+        return series.bfill()
 
 
 class Date(ElemOperator):

examples/highfreq/workflow.py

@@ -4,11 +4,11 @@
 import fire
 
 import qlib
-import pickle
 from qlib.constant import REG_CN
 from qlib.config import HIGH_FREQ_CONFIG
 
 from qlib.utils import init_instance_by_config
+from qlib.utils.pickle_utils import restricted_pickle_load
 from qlib.data.dataset.handler import DataHandlerLP
 from qlib.data.ops import Operators
 from qlib.data.data import Cal
@@ -125,10 +125,10 @@ class HighfreqWorkflow:
         del dataset, dataset_backtest
         ##=============reload dataset=============
         with open("dataset.pkl", "rb") as file_dataset:
-            dataset = pickle.load(file_dataset)
+            dataset = restricted_pickle_load(file_dataset)
 
         with open("dataset_backtest.pkl", "rb") as file_dataset_backtest:
-            dataset_backtest = pickle.load(file_dataset_backtest)
+            dataset_backtest = restricted_pickle_load(file_dataset_backtest)
 
         self._prepare_calender_cache()
         ##=============reinit dataset=============

examples/model_interpreter/feature.py

@@ -9,7 +9,6 @@ from qlib.utils import init_instance_by_config
 from qlib.tests.data import GetData
 from qlib.tests.config import CSI300_GBDT_TASK
 
-
 if __name__ == "__main__":
     # use default data
     provider_uri = "~/.qlib/qlib_data/cn_data"  # target_dir

examples/nested_decision_execution/workflow.py

@@ -95,7 +95,6 @@ pos  0.000000
 [1706497:MainThread](2021-12-07 14:08:30,627) INFO - qlib.timer - [log.py:113] - Time cost: 0.014s | waiting `async_log` Done
 """
 
-
 from copy import deepcopy
 import qlib
 import fire

examples/online_srv/update_online_pred.py

@@ -7,6 +7,7 @@ There are two parts including first_train and update_online_pred.
 Firstly, we will finish the training and set the trained models to the `online` models.
 Next, we will finish updating online predictions.
 """
+
 import copy
 import fire
 import qlib

examples/orderbook_data/README.md

@@ -27,7 +27,7 @@ pip install arctic  # NOTE: pip may fail to resolve the right package dependency
 2. Please follow following steps to download example data
 ```bash
 cd examples/orderbook_data/
-gdown https://drive.google.com/uc?id=15nZF7tFT_eKVZAcMFL1qPS4jGyJflH7e  # Proxies may be necessary here.
+gdown https://drive.google.com/uc?id=15FuUqWn2rkCi8uhJYGEQWKakcEqLJNDG  # Proxies may be necessary here.
 python ../../scripts/get_data.py _unzip --file_path highfreq_orderbook_example_data.zip --target_dir .
 ```
 

examples/orderbook_data/create_dataset.py

@@ -1,11 +1,12 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 """
-    NOTE:
-    - This scripts is a demo to import example data import Qlib
-    - !!!!!!!!!!!!!!!TODO!!!!!!!!!!!!!!!!!!!:
-        - Its structure is not well designed and very ugly, your contribution is welcome to make importing dataset easier
+NOTE:
+- This scripts is a demo to import example data import Qlib
+- !!!!!!!!!!!!!!!TODO!!!!!!!!!!!!!!!!!!!:
+    - Its structure is not well designed and very ugly, your contribution is welcome to make importing dataset easier
 """
+
 from datetime import date, datetime as dt
 import os
 from pathlib import Path

examples/rl_order_execution/README.md

@@ -7,7 +7,7 @@ This folder comprises an example of Reinforcement Learning (RL) workflows for or
 ### Get Data
 
 ```
-python -m qlib.run.get_data qlib_data qlib_data --target_dir ./data/bin --region hs300 --interval 5min
+python -m qlib.cli.data qlib_data --target_dir ./data/bin --region hs300 --interval 5min
 ```
 
 ### Generate Pickle-Style Data

examples/rl_order_execution/scripts/gen_training_orders.py

@@ -19,9 +19,9 @@ def generate_order(stock: str, start_idx: int, end_idx: int) -> bool:
 
     df["date"] = df["datetime"].dt.date.astype("datetime64")
     df = df.set_index(["instrument", "datetime", "date"])
-    df = df.groupby("date").take(range(start_idx, end_idx)).droplevel(level=0)
+    df = df.groupby("date", group_keys=False).take(range(start_idx, end_idx)).droplevel(level=0)
 
-    order_all = pd.DataFrame(df.groupby(level=(2, 0)).mean().dropna())
+    order_all = pd.DataFrame(df.groupby(level=(2, 0), group_keys=False).mean().dropna())
     order_all["amount"] = np.random.lognormal(-3.28, 1.14) * order_all["$volume0"]
     order_all = order_all[order_all["amount"] > 0.0]
     order_all["order_type"] = 0

examples/rl_order_execution/scripts/merge_orders.py

@@ -1,13 +1,15 @@
-import pickle
 import os
 import pandas as pd
 from tqdm import tqdm
 
+from qlib.utils.pickle_utils import restricted_pickle_load
+
 for tag in ["test", "valid"]:
     files = os.listdir(os.path.join("data/orders/", tag))
     dfs = []
     for f in tqdm(files):
-        df = pickle.load(open(os.path.join("data/orders/", tag, f), "rb"))
+        with open(os.path.join("data/orders/", tag, f), "rb") as fr:
+            df = restricted_pickle_load(fr)
         df = df.drop(["$close0"], axis=1)
         dfs.append(df)
 

examples/rolling_process_data/workflow.py

@@ -3,12 +3,12 @@
 
 import qlib
 import fire
-import pickle
 
 from datetime import datetime
 from qlib.constant import REG_CN
 from qlib.data.dataset.handler import DataHandlerLP
 from qlib.utils import init_instance_by_config
+from qlib.utils.pickle_utils import restricted_pickle_load
 from qlib.tests.data import GetData
 
 
@@ -42,7 +42,7 @@ class RollingDataWorkflow:
 
     def _load_pre_handler(self, path):
         with open(path, "rb") as file_dataset:
-            pre_handler = pickle.load(file_dataset)
+            pre_handler = restricted_pickle_load(file_dataset)
         return pre_handler
 
     def rolling_process(self):

examples/tutorial/detailed_workflow.ipynb

@@ -171,7 +171,9 @@
    "outputs": [],
    "source": [
     "import plotly.graph_objects as go\n",
+    "import plotly.io as pio\n",
     "\n",
+    "pio.renderers.default = \"notebook\"\n",
     "fig = go.Figure(\n",
     "    data=[\n",
     "        go.Candlestick(\n",

examples/workflow_by_code.py

@@ -1,12 +1,13 @@
 #  Copyright (c) Microsoft Corporation.
 #  Licensed under the MIT License.
 """
-Qlib provides two kinds of interfaces. 
+Qlib provides two kinds of interfaces.
 (1) Users could define the Quant research workflow by a simple configuration.
 (2) Qlib is designed in a modularized way and supports creating research workflow by code just like building blocks.
 
 The interface of (1) is `qrun XXX.yaml`.  The interface of (2) is script like this, which nearly does the same thing as `qrun XXX.yaml`
 """
+
 import qlib
 from qlib.constant import REG_CN
 from qlib.utils import init_instance_by_config, flatten_dict
@@ -15,7 +16,6 @@ from qlib.workflow.record_temp import SignalRecord, PortAnaRecord, SigAnaRecord
 from qlib.tests.data import GetData
 from qlib.tests.config import CSI300_BENCH, CSI300_GBDT_TASK
 
-
 if __name__ == "__main__":
     # use default data
     provider_uri = "~/.qlib/qlib_data/cn_data"  # target_dir

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools", "cython", "numpy>=1.24.0"]
+requires = ["setuptools", "setuptools-scm", "cython", "numpy>=1.24.0"]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -22,11 +22,18 @@ dynamic = ["version"]
 description = "A Quantitative-research Platform"
 requires-python = ">=3.8.0"
 readme = {file = "README.md", content-type = "text/markdown"}
+license = { text = "MIT" }
 
 dependencies = [
   "pyyaml",
   "numpy",
-  "pandas",
+  # Since version 1.1.0, pandas supports the ffill and bfill methods.
+  # Since version 2.1.0, pandas has deprecated the method parameter of the fillna method. 
+  # qlib has updated the fillna method in PR 1987 and limited the minimum version of pandas.
+  "pandas>=1.1",
+  # I encoutered an Error that the set_uri does not work when downloading artifacts in mlflow 3.1.1;
+  # But earlier versions of mlflow does not have this problem.
+  # But when I switch to 2.*.* version, another error occurs, which is even more strange...
   "mlflow",
   "filelock>=3.16.0",
   "redis",
@@ -44,6 +51,9 @@ dependencies = [
   "matplotlib",
   "jupyter",
   "nbconvert",
+  "pyarrow",
+  "pydantic-settings",
+  "setuptools-scm",
 ]
 
 [project.optional-dependencies]
@@ -59,6 +69,7 @@ rl = [
   "torch",
   "numpy<2.0.0",
 ]
+
 lint = [
   "black",
   "pylint",
@@ -66,10 +77,17 @@ lint = [
   "flake8",
   "nbqa",
 ]
+# snowballstemmer, a dependency of sphinx, was released on 2025-05-08 with version 3.0.0,
+# which causes errors in the build process. So we've limited the version for now.
 docs = [
+  # After upgrading scipy to version 1.16.0,
+  # we encountered ImportError: cannot import name '_lazywhere', in the build documentation,
+  # so we restricted the version of scipy to: 1.15.3
+  "scipy<=1.15.3",
   "sphinx",
   "sphinx_rtd_theme",
   "readthedocs_sphinx_ext",
+  "snowballstemmer<3.0",
 ]
 package = [
   "twine",
@@ -82,12 +100,26 @@ test = [
 ]
 analysis = [
   "plotly",
+  "statsmodels",
+]
+client = [
+  "python-socketio<6",
+  "tables",
 ]
 
+# In the process of releasing a new version, when checking the manylinux package with twine, an error is reported:
+# InvalidDistribution: Invalid distribution metadata: unrecognized or malformed field 'license-file'
+# To solve this problem, we added license-files here. Refs: https://github.com/pypa/twine/issues/1216
 [tool.setuptools]
 packages = [
   "qlib",
 ]
+license-files = []
 
 [project.scripts]
-qrun = "qlib.workflow.cli:run"
+qrun = "qlib.cli.run:run"
+
+[tool.setuptools_scm]
+local_scheme = "no-local-version"
+version_scheme = "guess-next-dev"
+write_to = "qlib/_version.py"

qlib/__init__.py

@@ -2,14 +2,22 @@
 # Licensed under the MIT License.
 from pathlib import Path
 
-__version__ = "0.9.6.99"
+from setuptools_scm import get_version
+
+try:
+    from ._version import version as __version__
+except ImportError:
+    __version__ = get_version(root="..", relative_to=__file__)
 __version__bak = __version__  # This version is backup for QlibConfig.reset_qlib_version
-import os
-from typing import Union
-from ruamel.yaml import YAML
 import logging
+import os
 import platform
+import re
 import subprocess
+from typing import Union
+
+from ruamel.yaml import YAML
+
 from .log import get_module_logger
 
 
@@ -80,34 +88,41 @@ def _mount_nfs_uri(provider_uri, mount_path, auto_mount: bool = False):
     LOG = get_module_logger("mount nfs", level=logging.INFO)
     if mount_path is None:
         raise ValueError(f"Invalid mount path: {mount_path}!")
+    if not re.match(r"^[a-zA-Z0-9.:/\-_]+$", provider_uri):
+        raise ValueError(f"Invalid provider_uri format: {provider_uri}")
     # FIXME: the C["provider_uri"] is modified in this function
     # If it is not modified, we can pass only  provider_uri or mount_path instead of C
-    mount_command = "sudo mount.nfs %s %s" % (provider_uri, mount_path)
+    mount_command = ["sudo", "mount.nfs", provider_uri, mount_path]
     # If the provider uri looks like this 172.23.233.89//data/csdesign'
     # It will be a nfs path. The client provider will be used
     if not auto_mount:  # pylint: disable=R1702
         if not Path(mount_path).exists():
             raise FileNotFoundError(
-                f"Invalid mount path: {mount_path}! Please mount manually: {mount_command} or Set init parameter `auto_mount=True`"
+                f"Invalid mount path: {mount_path}! Please mount manually: {' '.join(mount_command)} or Set init parameter `auto_mount=True`"
             )
     else:
         # Judging system type
         sys_type = platform.system()
         if "windows" in sys_type.lower():
             # system: window
-            exec_result = os.popen(f"mount -o anon {provider_uri} {mount_path}")
-            result = exec_result.read()
-            if "85" in result:
-                LOG.warning(f"{provider_uri} on Windows:{mount_path} is already mounted")
-            elif "53" in result:
-                raise OSError("not find network path")
-            elif "error" in result or "错误" in result:
-                raise OSError("Invalid mount path")
-            elif provider_uri in result:
-                LOG.info("window success mount..")
-            else:
-                raise OSError(f"unknown error: {result}")
-
+            try:
+                subprocess.run(
+                    ["mount", "-o", "anon", provider_uri, mount_path],
+                    capture_output=True,
+                    text=True,
+                    check=True,
+                )
+                LOG.info("Mount finished.")
+            except subprocess.CalledProcessError as e:
+                error_output = (e.stdout or "") + (e.stderr or "")
+                if e.returncode == 85:
+                    LOG.warning(f"{provider_uri} already mounted at {mount_path}")
+                elif e.returncode == 53:
+                    raise OSError("Network path not found") from e
+                elif "error" in error_output.lower() or "错误" in error_output:
+                    raise OSError("Invalid mount path") from e
+                else:
+                    raise OSError(f"Unknown mount error: {error_output.strip()}") from e
         else:
             # system: linux/Unix/Mac
             # check mount
@@ -119,15 +134,19 @@ def _mount_nfs_uri(provider_uri, mount_path, auto_mount: bool = False):
             _is_mount = False
             while _check_level_num:
                 with subprocess.Popen(
-                    'mount | grep "{}"'.format(_remote_uri),
-                    shell=True,
+                    ["mount"],
+                    text=True,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.STDOUT,
                 ) as shell_r:
                     _command_log = shell_r.stdout.readlines()
+                    _command_log = [line for line in _command_log if _remote_uri in line]
                 if len(_command_log) > 0:
                     for _c in _command_log:
-                        _temp_mount = _c.decode("utf-8").split(" ")[2]
+                        if isinstance(_c, str):
+                            _temp_mount = _c.split(" ")[2]
+                        else:
+                            _temp_mount = _c.decode("utf-8").split(" ")[2]
                         _temp_mount = _temp_mount[:-1] if _temp_mount.endswith("/") else _temp_mount
                         if _temp_mount == _mount_path:
                             _is_mount = True
@@ -152,16 +171,16 @@ def _mount_nfs_uri(provider_uri, mount_path, auto_mount: bool = False):
                 if not command_res:
                     raise OSError("nfs-common is not found, please install it by execute: sudo apt install nfs-common")
                 # manually mount
-                command_status = os.system(mount_command)
-                if command_status == 256:
-                    raise OSError(
-                        f"mount {provider_uri} on {mount_path} error! Needs SUDO! Please mount manually: {mount_command}"
-                    )
-                elif command_status == 32512:
-                    # LOG.error("Command error")
-                    raise OSError(f"mount {provider_uri} on {mount_path} error! Command error")
-                elif command_status == 0:
-                    LOG.info("Mount finished")
+                try:
+                    subprocess.run(mount_command, check=True, capture_output=True, text=True)
+                    LOG.info("Mount finished.")
+                except subprocess.CalledProcessError as e:
+                    if e.returncode == 256:
+                        raise OSError("Mount failed: requires sudo or permission denied") from e
+                    elif e.returncode == 32512:
+                        raise OSError(f"mount {provider_uri} on {mount_path} error! Command error") from e
+                    else:
+                        raise OSError(f"Mount failed: {e.stderr}") from e
             else:
                 LOG.warning(f"{_remote_uri} on {_mount_path} is already mounted")
 

qlib/backtest/backtest.py

@@ -18,7 +18,6 @@ from tqdm.auto import tqdm
 
 from ..utils.time import Freq
 
-
 PORT_METRIC = Dict[str, Tuple[pd.DataFrame, dict]]
 INDICATOR_METRIC = Dict[str, Tuple[pd.DataFrame, Indicator]]
 

qlib/backtest/exchange.py

@@ -897,6 +897,7 @@ class Exchange:
             # if we don't know current position, we choose to sell all
             # Otherwise, we clip the amount based on current position
             if position is not None:
+                # TODO: make the trading shortable
                 current_amount = (
                     position.get_stock_amount(order.stock_id) if position.check_stock(order.stock_id) else 0
                 )

qlib/backtest/high_performance_ds.py

@@ -104,7 +104,7 @@ class PandasQuote(BaseQuote):
     def __init__(self, quote_df: pd.DataFrame, freq: str) -> None:
         super().__init__(quote_df=quote_df, freq=freq)
         quote_dict = {}
-        for stock_id, stock_val in quote_df.groupby(level="instrument"):
+        for stock_id, stock_val in quote_df.groupby(level="instrument", group_keys=False):
             quote_dict[stock_id] = stock_val.droplevel(level="instrument")
         self.data = quote_dict
 
@@ -137,7 +137,7 @@ class NumpyQuote(BaseQuote):
         """
         super().__init__(quote_df=quote_df, freq=freq)
         quote_dict = {}
-        for stock_id, stock_val in quote_df.groupby(level="instrument"):
+        for stock_id, stock_val in quote_df.groupby(level="instrument", group_keys=False):
             quote_dict[stock_id] = idd.MultiData(stock_val.droplevel(level="instrument"))
             quote_dict[stock_id].sort_index()  # To support more flexible slicing, we must sort data first
         self.data = quote_dict

qlib/backtest/position.py

@@ -311,7 +311,7 @@ class Position(BasePosition):
             freq=freq,
             disk_cache=True,
         ).dropna()
-        price_dict = price_df.groupby(["instrument"]).tail(1).reset_index(level=1, drop=True)["$close"].to_dict()
+        price_dict = price_df.groupby(["instrument"], group_keys=False).tail(1)["$close"].to_dict()
 
         if len(price_dict) < len(stock_list):
             lack_stock = set(stock_list) - set(price_dict)

qlib/backtest/profit_attribution.py

@@ -281,13 +281,13 @@ def brinson_pa(
 
     stock_group_field = stock_df[group_field].unstack().T
     # FIXME: some attributes of some suspend stock is NAN.
-    stock_group_field = stock_group_field.fillna(method="ffill")
+    stock_group_field = stock_group_field.ffill()
     stock_group_field = stock_group_field.loc[start_date:end_date]
 
     stock_group = get_stock_group(stock_group_field, bench_stock_weight, group_method, group_n)
 
     deal_price_df = stock_df["deal_price"].unstack().T
-    deal_price_df = deal_price_df.fillna(method="ffill")
+    deal_price_df = deal_price_df.ffill()
 
     # NOTE:
     # The return will be slightly different from the of the return in the report.

qlib/backtest/report.py

@@ -114,7 +114,11 @@ class PortfolioMetrics:
             _temp_result, _ = get_higher_eq_freq_feature(_codes, fields, start_time, end_time, freq=freq)
             if len(_temp_result) == 0:
                 raise ValueError(f"The benchmark {_codes} does not exist. Please provide the right benchmark")
-            return _temp_result.groupby(level="datetime")[_temp_result.columns.tolist()[0]].mean().fillna(0)
+            return (
+                _temp_result.groupby(level="datetime", group_keys=False)[_temp_result.columns.tolist()[0]]
+                .mean()
+                .fillna(0)
+            )
 
     def _sample_benchmark(
         self,
@@ -427,6 +431,10 @@ class Indicator:
         # NOTE ~(price_s < 1e-08) is different from price_s >= 1e-8
         #   ~(np.nan < 1e-8) -> ~(False)  -> True
 
+        # if price_s is empty
+        if price_s.empty:
+            return None, None
+
         assert isinstance(price_s, idd.SingleData)
         if agg == "vwap":
             volume_s = trade_exchange.get_volume(inst, trade_start_time, trade_end_time, method=None)

qlib/cli/data.py

@@ -4,6 +4,5 @@
 import fire
 from qlib.tests.data import GetData
 
-
 if __name__ == "__main__":
     fire.Fire(GetData)

qlib/cli/run.py

@@ -87,7 +87,7 @@ def workflow(config_path, experiment_name="workflow", uri_folder="mlruns"):
     """
     This is a Qlib CLI entrance.
     User can run the whole Quant research workflow defined by a configure file
-    - the code is located here ``qlib/workflow/cli.py`
+    - the code is located here ``qlib/cli/run.py``
 
     User can specify a base_config file in your workflow.yml file by adding "BASE_CONFIG_PATH".
     Qlib will load the configuration in BASE_CONFIG_PATH first, and the user only needs to update the custom fields

qlib/config.py

@@ -10,6 +10,7 @@ Two modes are supported
 - server
 
 """
+
 from __future__ import annotations
 
 import os
@@ -27,6 +28,38 @@ from qlib.constant import REG_CN, REG_US, REG_TW
 if TYPE_CHECKING:
     from qlib.utils.time import Freq
 
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class MLflowSettings(BaseSettings):
+    uri: str = "file:" + str(Path(os.getcwd()).resolve() / "mlruns")
+    default_exp_name: str = "Experiment"
+
+
+class QSettings(BaseSettings):
+    """
+    Qlib's settings.
+    It tries to provide a default settings for most of Qlib's components.
+    But it would be a long journey to provide a comprehensive settings for all of Qlib's components.
+
+    Here is some design guidelines:
+    - The priority of settings is
+        - Actively passed-in settings, like `qlib.init(provider_uri=...)`
+        - The default settings
+            - QSettings tries to provide default settings for most of Qlib's components.
+    """
+
+    mlflow: MLflowSettings = MLflowSettings()
+    provider_uri: str = "~/.qlib/qlib_data/cn_data"
+
+    model_config = SettingsConfigDict(
+        env_prefix="QLIB_",
+        env_nested_delimiter="_",
+    )
+
+
+QSETTINGS = QSettings()
+
 
 class Config:
     def __init__(self, default_conf):
@@ -187,8 +220,8 @@ _default_config = {
         "class": "MLflowExpManager",
         "module_path": "qlib.workflow.expm",
         "kwargs": {
-            "uri": "file:" + str(Path(os.getcwd()).resolve() / "mlruns"),
-            "default_exp_name": "Experiment",
+            "uri": QSETTINGS.mlflow.uri,
+            "default_exp_name": QSETTINGS.mlflow.default_exp_name,
         },
     },
     "pit_record_type": {
@@ -230,7 +263,7 @@ MODE_CONF = {
     },
     "client": {
         # config it in user's own code
-        "provider_uri": "~/.qlib/qlib_data/cn_data",
+        "provider_uri": QSETTINGS.provider_uri,
         # cache
         # Using parameter 'remote' to announce the client is using server_cache, and the writing access will be disabled.
         # Disable cache by default. Avoid introduce advanced features for beginners

qlib/contrib/data/dataset.py

@@ -6,10 +6,11 @@ import torch
 import warnings
 import numpy as np
 import pandas as pd
+from qlib.utils.data import guess_horizon
+from qlib.utils import init_instance_by_config
 
 from qlib.data.dataset import DatasetH
 
-
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
 
@@ -32,7 +33,7 @@ def _create_ts_slices(index, seq_len):
     assert index.is_monotonic_increasing, "index should be sorted"
 
     # number of dates for each instrument
-    sample_count_by_insts = index.to_series().groupby(level=0).size().values
+    sample_count_by_insts = index.to_series().groupby(level=0, group_keys=False).size().values
 
     # start index for each instrument
     start_index_of_insts = np.roll(np.cumsum(sample_count_by_insts), 1)
@@ -130,6 +131,14 @@ class MTSDatasetH(DatasetH):
         input_size=None,
         **kwargs,
     ):
+        if horizon == 0:
+            # Try to guess horizon
+            if isinstance(handler, (dict, str)):
+                handler = init_instance_by_config(handler)
+            assert "label" in getattr(handler.data_loader, "fields", None)
+            label = handler.data_loader.fields["label"][0][0]
+            horizon = guess_horizon([label])
+
         assert num_states == 0 or horizon > 0, "please specify `horizon` to avoid data leakage"
         assert memory_mode in ["sample", "daily"], "unsupported memory mode"
         assert memory_mode == "sample" or batch_size < 0, "daily memory requires daily sampling (`batch_size < 0`)"

qlib/contrib/data/processor.py

@@ -55,14 +55,18 @@ class ConfigSectionProcessor(Processor):
 
         # Label
         cols = df_focus.columns[df_focus.columns.str.contains("^LABEL")]
-        df_focus[cols] = df_focus[cols].groupby(level="datetime").apply(_label_norm)
+        df_focus[cols] = df_focus[cols].groupby(level="datetime", group_keys=False).apply(_label_norm)
 
         # Features
         cols = df_focus.columns[df_focus.columns.str.contains("^KLEN|^KLOW|^KUP")]
-        df_focus[cols] = df_focus[cols].apply(lambda x: x**0.25).groupby(level="datetime").apply(_feature_norm)
+        df_focus[cols] = (
+            df_focus[cols].apply(lambda x: x**0.25).groupby(level="datetime", group_keys=False).apply(_feature_norm)
+        )
 
         cols = df_focus.columns[df_focus.columns.str.contains("^KLOW2|^KUP2")]
-        df_focus[cols] = df_focus[cols].apply(lambda x: x**0.5).groupby(level="datetime").apply(_feature_norm)
+        df_focus[cols] = (
+            df_focus[cols].apply(lambda x: x**0.5).groupby(level="datetime", group_keys=False).apply(_feature_norm)
+        )
 
         _cols = [
             "KMID",
@@ -88,25 +92,35 @@ class ConfigSectionProcessor(Processor):
         ]
         pat = "|".join(["^" + x for x in _cols])
         cols = df_focus.columns[df_focus.columns.str.contains(pat) & (~df_focus.columns.isin(["HIGH0", "LOW0"]))]
-        df_focus[cols] = df_focus[cols].groupby(level="datetime").apply(_feature_norm)
+        df_focus[cols] = df_focus[cols].groupby(level="datetime", group_keys=False).apply(_feature_norm)
 
         cols = df_focus.columns[df_focus.columns.str.contains("^STD|^VOLUME|^VMA|^VSTD")]
-        df_focus[cols] = df_focus[cols].apply(np.log).groupby(level="datetime").apply(_feature_norm)
+        df_focus[cols] = df_focus[cols].apply(np.log).groupby(level="datetime", group_keys=False).apply(_feature_norm)
 
         cols = df_focus.columns[df_focus.columns.str.contains("^RSQR")]
-        df_focus[cols] = df_focus[cols].fillna(0).groupby(level="datetime").apply(_feature_norm)
+        df_focus[cols] = df_focus[cols].fillna(0).groupby(level="datetime", group_keys=False).apply(_feature_norm)
 
         cols = df_focus.columns[df_focus.columns.str.contains("^MAX|^HIGH0")]
-        df_focus[cols] = df_focus[cols].apply(lambda x: (x - 1) ** 0.5).groupby(level="datetime").apply(_feature_norm)
+        df_focus[cols] = (
+            df_focus[cols]
+            .apply(lambda x: (x - 1) ** 0.5)
+            .groupby(level="datetime", group_keys=False)
+            .apply(_feature_norm)
+        )
 
         cols = df_focus.columns[df_focus.columns.str.contains("^MIN|^LOW0")]
-        df_focus[cols] = df_focus[cols].apply(lambda x: (1 - x) ** 0.5).groupby(level="datetime").apply(_feature_norm)
+        df_focus[cols] = (
+            df_focus[cols]
+            .apply(lambda x: (1 - x) ** 0.5)
+            .groupby(level="datetime", group_keys=False)
+            .apply(_feature_norm)
+        )
 
         cols = df_focus.columns[df_focus.columns.str.contains("^CORR|^CORD")]
-        df_focus[cols] = df_focus[cols].apply(np.exp).groupby(level="datetime").apply(_feature_norm)
+        df_focus[cols] = df_focus[cols].apply(np.exp).groupby(level="datetime", group_keys=False).apply(_feature_norm)
 
         cols = df_focus.columns[df_focus.columns.str.contains("^WVMA")]
-        df_focus[cols] = df_focus[cols].apply(np.log1p).groupby(level="datetime").apply(_feature_norm)
+        df_focus[cols] = df_focus[cols].apply(np.log1p).groupby(level="datetime", group_keys=False).apply(_feature_norm)
 
         df[selected_cols] = df_focus.values
 

qlib/contrib/eva/alpha.py

@@ -39,7 +39,7 @@ def calc_long_short_prec(
         long precision and short precision in time level
     """
     if is_alpha:
-        label = label - label.mean(level=date_col)
+        label = label - label.groupby(level=date_col, group_keys=False).mean()
     if int(1 / quantile) >= len(label.index.get_level_values(1).unique()):
         raise ValueError("Need more instruments to calculate precision")
 
@@ -47,23 +47,25 @@ def calc_long_short_prec(
     if dropna:
         df.dropna(inplace=True)
 
-    group = df.groupby(level=date_col)
+    group = df.groupby(level=date_col, group_keys=False)
 
     def N(x):
         return int(len(x) * quantile)
 
     # find the top/low quantile of prediction and treat them as long and short target
-    long = group.apply(lambda x: x.nlargest(N(x), columns="pred").label).reset_index(level=0, drop=True)
-    short = group.apply(lambda x: x.nsmallest(N(x), columns="pred").label).reset_index(level=0, drop=True)
+    long = group.apply(lambda x: x.nlargest(N(x), columns="pred").label)
+    short = group.apply(lambda x: x.nsmallest(N(x), columns="pred").label)
 
-    groupll = long.groupby(date_col)
+    groupll = long.groupby(date_col, group_keys=False)
     l_dom = groupll.apply(lambda x: x > 0)
     l_c = groupll.count()
 
-    groups = short.groupby(date_col)
+    groups = short.groupby(date_col, group_keys=False)
     s_dom = groups.apply(lambda x: x < 0)
     s_c = groups.count()
-    return (l_dom.groupby(date_col).sum() / l_c), (s_dom.groupby(date_col).sum() / s_c)
+    return (l_dom.groupby(date_col, group_keys=False).sum() / l_c), (
+        s_dom.groupby(date_col, group_keys=False).sum() / s_c
+    )
 
 
 def calc_long_short_return(
@@ -100,7 +102,7 @@ def calc_long_short_return(
     df = pd.DataFrame({"pred": pred, "label": label})
     if dropna:
         df.dropna(inplace=True)
-    group = df.groupby(level=date_col)
+    group = df.groupby(level=date_col, group_keys=False)
 
     def N(x):
         return int(len(x) * quantile)
@@ -173,8 +175,8 @@ def calc_ic(pred: pd.Series, label: pd.Series, date_col="datetime", dropna=False
         ic and rank ic
     """
     df = pd.DataFrame({"pred": pred, "label": label})
-    ic = df.groupby(date_col).apply(lambda df: df["pred"].corr(df["label"]))
-    ric = df.groupby(date_col).apply(lambda df: df["pred"].corr(df["label"], method="spearman"))
+    ic = df.groupby(date_col, group_keys=False).apply(lambda df: df["pred"].corr(df["label"]))
+    ric = df.groupby(date_col, group_keys=False).apply(lambda df: df["pred"].corr(df["label"], method="spearman"))
     if dropna:
         return ic.dropna(), ric.dropna()
     else:

qlib/contrib/evaluate.py

@@ -7,7 +7,7 @@ from __future__ import print_function
 import numpy as np
 import pandas as pd
 import warnings
-from typing import Union
+from typing import Union, Literal
 
 from ..log import get_module_logger
 from ..utils import get_date_range
@@ -20,20 +20,17 @@ from ..data import D
 from ..config import C
 from ..data.dataset.utils import get_level_index
 
-
 logger = get_module_logger("Evaluate")
 
 
-def risk_analysis(r, N: int = None, freq: str = "day"):
+def risk_analysis(r, N: int = None, freq: str = "day", mode: Literal["sum", "product"] = "sum"):
     """Risk Analysis
     NOTE:
-    The calculation of annulaized return is different from the definition of annualized return.
+    The calculation of annualized return is different from the definition of annualized return.
     It is implemented by design.
-    Qlib tries to cumulated returns by summation instead of production to avoid the cumulated curve being skewed exponentially.
+    Qlib tries to cumulate returns by summation instead of production to avoid the cumulated curve being skewed exponentially.
     All the calculation of annualized returns follows this principle in Qlib.
 
-    TODO: add a parameter to enable calculating metrics with production accumulation of return.
-
     Parameters
     ----------
     r : pandas.Series
@@ -42,11 +39,14 @@ def risk_analysis(r, N: int = None, freq: str = "day"):
         scaler for annualizing information_ratio (day: 252, week: 50, month: 12), at least one of `N` and `freq` should exist
     freq: str
         analysis frequency used for calculating the scaler, at least one of `N` and `freq` should exist
+    mode: Literal["sum", "product"]
+        the method by which returns are accumulated:
+        - "sum": Arithmetic accumulation (linear returns).
+        - "product": Geometric accumulation (compounded returns).
     """
 
     def cal_risk_analysis_scaler(freq):
         _count, _freq = Freq.parse(freq)
-        # len(D.calendar(start_time='2010-01-01', end_time='2019-12-31', freq='day')) = 2384
         _freq_scaler = {
             Freq.NORM_FREQ_MINUTE: 240 * 238,
             Freq.NORM_FREQ_DAY: 238,
@@ -62,11 +62,26 @@ def risk_analysis(r, N: int = None, freq: str = "day"):
     if N is None:
         N = cal_risk_analysis_scaler(freq)
 
-    mean = r.mean()
-    std = r.std(ddof=1)
-    annualized_return = mean * N
+    if mode == "sum":
+        mean = r.mean()
+        std = r.std(ddof=1)
+        annualized_return = mean * N
+        max_drawdown = (r.cumsum() - r.cumsum().cummax()).min()
+    elif mode == "product":
+        cumulative_curve = (1 + r).cumprod()
+        # geometric mean (compound annual growth rate)
+        mean = cumulative_curve.iloc[-1] ** (1 / len(r)) - 1
+        # volatility of log returns
+        std = np.log(1 + r).std(ddof=1)
+
+        cumulative_return = cumulative_curve.iloc[-1] - 1
+        annualized_return = (1 + cumulative_return) ** (N / len(r)) - 1
+        # max percentage drawdown from peak cumulative product
+        max_drawdown = (cumulative_curve / cumulative_curve.cummax() - 1).min()
+    else:
+        raise ValueError(f"risk_analysis accumulation mode {mode} is not supported. Expected `sum` or `product`.")
+
     information_ratio = mean / std * np.sqrt(N)
-    max_drawdown = (r.cumsum() - r.cumsum().cummax()).min()
     data = {
         "mean": mean,
         "std": std,

qlib/contrib/meta/__init__.py

@@ -3,5 +3,4 @@
 
 from .data_selection import MetaTaskDS, MetaDatasetDS, MetaModelDS
 
-
 __all__ = ["MetaTaskDS", "MetaDatasetDS", "MetaModelDS"]

qlib/contrib/meta/data_selection/__init__.py

@@ -4,5 +4,4 @@
 from .dataset import MetaDatasetDS, MetaTaskDS
 from .model import MetaModelDS
 
-
 __all__ = ["MetaDatasetDS", "MetaTaskDS", "MetaModelDS"]

qlib/contrib/meta/data_selection/dataset.py

@@ -106,7 +106,7 @@ class InternalData:
 
     def _calc_perf(self, pred, label):
         df = pd.DataFrame({"pred": pred, "label": label})
-        df = df.groupby("datetime").corr(method="spearman")
+        df = df.groupby("datetime", group_keys=False).corr(method="spearman")
         corr = df.loc(axis=0)[:, "pred"]["label"].droplevel(axis=0, level=-1)
         return corr
 
@@ -161,7 +161,7 @@ class MetaTaskDS(MetaTask):
                 raise ValueError(f"Most of samples are dropped. Please check this task: {task}")
 
             assert (
-                d_test.groupby("datetime").size().shape[0] >= 5
+                d_test.groupby("datetime", group_keys=False).size().shape[0] >= 5
             ), "In this segment, this trading dates is less than 5, you'd better check the data."
 
             sample_time_belong = np.zeros((d_train.shape[0], time_perf.shape[1]))

qlib/contrib/meta/data_selection/model.py

@@ -125,7 +125,11 @@ class MetaModelDS(MetaTaskModel):
         loss_l.setdefault(phase, []).append(running_loss)
 
         pred_y_all = pd.concat(pred_y_all)
-        ic = pred_y_all.groupby("datetime").apply(lambda df: df["pred"].corr(df["label"], method="spearman")).mean()
+        ic = (
+            pred_y_all.groupby("datetime", group_keys=False)
+            .apply(lambda df: df["pred"].corr(df["label"], method="spearman"))
+            .mean()
+        )
 
         R.log_metrics(**{f"loss/{phase}": running_loss, "step": epoch})
         R.log_metrics(**{f"ic/{phase}": ic, "step": epoch})

qlib/contrib/model/double_ensemble.py

@@ -166,7 +166,7 @@ class DEnsembleModel(Model, FeatureInt):
 
         # calculate weights
         h["bins"] = pd.cut(h["h_value"], self.bins_sr)
-        h_avg = h.groupby("bins")["h_value"].mean()
+        h_avg = h.groupby("bins", group_keys=False, observed=False)["h_value"].mean()
         weights = pd.Series(np.zeros(N, dtype=float))
         for b in h_avg.index:
             weights[h["bins"] == b] = 1.0 / (self.decay**k_th * h_avg[b] + 0.1)

qlib/contrib/model/gbdt.py

@@ -51,7 +51,7 @@ class LGBModel(ModelFT, LightGBMFInt):
                     w = reweighter.reweight(df)
                 else:
                     raise ValueError("Unsupported reweighter type.")
-                ds_l.append((lgb.Dataset(x.values, label=y, weight=w), key))
+                ds_l.append((lgb.Dataset(x.values, label=y, weight=w, free_raw_data=False), key))
         return ds_l
 
     def fit(
@@ -109,8 +109,10 @@ class LGBModel(ModelFT, LightGBMFInt):
             verbose level
         """
         # Based on existing model and finetune by train more rounds
-        dtrain, _ = self._prepare_data(dataset, reweighter)  # pylint: disable=W0632
-        if dtrain.empty:
+        ds_l = self._prepare_data(dataset, reweighter)
+        dtrain, _ = ds_l[0]
+
+        if dtrain.construct().num_data() == 0:
             raise ValueError("Empty data from dataset, please check your dataset config.")
         verbose_eval_callback = lgb.log_evaluation(period=verbose_eval)
         self.model = lgb.train(

qlib/contrib/model/highfreq_gdbt_model.py

@@ -90,8 +90,14 @@ class HFLGBModel(ModelFT, LightGBMFInt):
         if y_train.values.ndim == 2 and y_train.values.shape[1] == 1:
             l_name = df_train["label"].columns[0]
             # Convert label into alpha
-            df_train["label"][l_name] = df_train["label"][l_name] - df_train["label"][l_name].mean(level=0)
-            df_valid["label"][l_name] = df_valid["label"][l_name] - df_valid["label"][l_name].mean(level=0)
+            df_train.loc[:, ("label", l_name)] = (
+                df_train.loc[:, ("label", l_name)]
+                - df_train.loc[:, ("label", l_name)].groupby(level=0, group_keys=False).mean()
+            )
+            df_valid.loc[:, ("label", l_name)] = (
+                df_valid.loc[:, ("label", l_name)]
+                - df_valid.loc[:, ("label", l_name)].groupby(level=0, group_keys=False).mean()
+            )
 
             def mapping_fn(x):
                 return 0 if x < 0 else 1

qlib/contrib/model/pytorch_adarnn.py

@@ -214,8 +214,10 @@ class ADARNN(Model):
     def calc_all_metrics(pred):
         """pred is a pandas dataframe that has two attributes: score (pred) and label (real)"""
         res = {}
-        ic = pred.groupby(level="datetime").apply(lambda x: x.label.corr(x.score))
-        rank_ic = pred.groupby(level="datetime").apply(lambda x: x.label.corr(x.score, method="spearman"))
+        ic = pred.groupby(level="datetime", group_keys=False).apply(lambda x: x.label.corr(x.score))
+        rank_ic = pred.groupby(level="datetime", group_keys=False).apply(
+            lambda x: x.label.corr(x.score, method="spearman")
+        )
         res["ic"] = ic.mean()
         res["icir"] = ic.mean() / ic.std()
         res["ric"] = rank_ic.mean()

qlib/contrib/model/pytorch_add.py

@@ -226,7 +226,7 @@ class ADD(Model):
 
     def get_daily_inter(self, df, shuffle=False):
         # organize the train data into daily batches
-        daily_count = df.groupby(level=0).size().values
+        daily_count = df.groupby(level=0, group_keys=False).size().values
         daily_index = np.roll(np.cumsum(daily_count), 1)
         daily_index[0] = 0
         if shuffle:
@@ -349,7 +349,7 @@ class ADD(Model):
         return best_score
 
     def gen_market_label(self, df, raw_label):
-        market_label = raw_label.groupby("datetime").mean().squeeze()
+        market_label = raw_label.groupby("datetime", group_keys=False).mean().squeeze()
         bins = [-np.inf, self.lo, self.hi, np.inf]
         market_label = pd.cut(market_label, bins, labels=False)
         market_label.name = ("market_return", "market_return")
@@ -357,7 +357,7 @@ class ADD(Model):
         return df
 
     def fit_thresh(self, train_label):
-        market_label = train_label.groupby("datetime").mean().squeeze()
+        market_label = train_label.groupby("datetime", group_keys=False).mean().squeeze()
         self.lo, self.hi = market_label.quantile([1 / 3, 2 / 3])
 
     def fit(

qlib/contrib/model/pytorch_gats.py

@@ -163,7 +163,7 @@ class GATs(Model):
 
     def get_daily_inter(self, df, shuffle=False):
         # organize the train data into daily batches
-        daily_count = df.groupby(level=0).size().values
+        daily_count = df.groupby(level=0, group_keys=False).size().values
         daily_index = np.roll(np.cumsum(daily_count), 1)
         daily_index[0] = 0
         if shuffle:

qlib/contrib/model/pytorch_gats_ts.py

@@ -27,7 +27,9 @@ class DailyBatchSampler(Sampler):
     def __init__(self, data_source):
         self.data_source = data_source
         # calculate number of samples in each batch
-        self.daily_count = pd.Series(index=self.data_source.get_index()).groupby("datetime").size().values
+        self.daily_count = (
+            pd.Series(index=self.data_source.get_index()).groupby("datetime", group_keys=False).size().values
+        )
         self.daily_index = np.roll(np.cumsum(self.daily_count), 1)  # calculate begin index of each batch
         self.daily_index[0] = 0
 
@@ -181,7 +183,7 @@ class GATs(Model):
 
     def get_daily_inter(self, df, shuffle=False):
         # organize the train data into daily batches
-        daily_count = df.groupby(level=0).size().values
+        daily_count = df.groupby(level=0, group_keys=False).size().values
         daily_index = np.roll(np.cumsum(daily_count), 1)
         daily_index[0] = 0
         if shuffle:

qlib/contrib/model/pytorch_general_nn.py

@@ -13,6 +13,7 @@ import copy
 
 import torch
 import torch.optim as optim
+from torch.optim.lr_scheduler import ReduceLROnPlateau
 
 from qlib.data.dataset.weight import Reweighter
 
@@ -136,6 +137,10 @@ class GeneralPTNN(Model):
         else:
             raise NotImplementedError("optimizer {} is not supported!".format(optimizer))
 
+        # === ReduceLROnPlateau learning rate scheduler ===
+        self.lr_scheduler = ReduceLROnPlateau(
+            self.train_optimizer, mode="min", factor=0.5, patience=5, min_lr=1e-6, threshold=1e-5
+        )
         self.fitted = False
         self.dnn_model.to(self.device)
 
@@ -154,7 +159,7 @@ class GeneralPTNN(Model):
             weight = torch.ones_like(label)
 
         if self.loss == "mse":
-            return self.mse(pred[mask], label[mask], weight[mask])
+            return self.mse(pred[mask], label[mask].view(-1, 1), weight[mask])
 
         raise ValueError("unknown loss `%s`" % self.loss)
 
@@ -162,7 +167,7 @@ class GeneralPTNN(Model):
         mask = torch.isfinite(label)
 
         if self.metric in ("", "loss"):
-            return -self.loss_fn(pred[mask], label[mask])
+            return self.loss_fn(pred[mask], label[mask])
 
         raise ValueError("unknown metric `%s`" % self.metric)
 
@@ -238,6 +243,8 @@ class GeneralPTNN(Model):
 
         dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
         dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
+        self.logger.info(f"Train samples: {len(dl_train)}")
+        self.logger.info(f"Valid samples: {len(dl_valid)}")
         if dl_train.empty or dl_valid.empty:
             raise ValueError("Empty data from dataset, please check your dataset config.")
 
@@ -279,7 +286,7 @@ class GeneralPTNN(Model):
 
         stop_steps = 0
         train_loss = 0
-        best_score = -np.inf
+        best_score = np.inf
         best_epoch = 0
         evals_result["train"] = []
         evals_result["valid"] = []
@@ -295,13 +302,18 @@ class GeneralPTNN(Model):
             self.logger.info("evaluating...")
             train_loss, train_score = self.test_epoch(train_loader)
             val_loss, val_score = self.test_epoch(valid_loader)
-            self.logger.info("train %.6f, valid %.6f" % (train_score, val_score))
+            self.logger.info("Epoch%d: train %.6f, valid %.6f" % (step, train_score, val_score))
             evals_result["train"].append(train_score)
             evals_result["valid"].append(val_score)
 
+            # current_lr = self.train_optimizer.param_groups[0]["lr"]
+            # self.logger.info("Current learning rate: %.6e" % current_lr)
+
+            self.lr_scheduler.step(val_score)
+
             if step == 0:
                 best_param = copy.deepcopy(self.dnn_model.state_dict())
-            if val_score > best_score:
+            if val_score < best_score:
                 best_score = val_score
                 stop_steps = 0
                 best_epoch = step
@@ -312,7 +324,7 @@ class GeneralPTNN(Model):
                     self.logger.info("early stop")
                     break
 
-        self.logger.info("best score: %.6lf @ %d" % (best_score, best_epoch))
+        self.logger.info("best score: %.6lf @ %d epoch" % (best_score, best_epoch))
         self.dnn_model.load_state_dict(best_param)
         torch.save(best_param, save_path)
 
@@ -329,6 +341,7 @@ class GeneralPTNN(Model):
             raise ValueError("model is not fitted yet!")
 
         dl_test = dataset.prepare("test", col_set=["feature", "label"], data_key=DataHandlerLP.DK_I)
+        self.logger.info(f"Test samples: {len(dl_test)}")
 
         if isinstance(dataset, TSDatasetH):
             dl_test.config(fillna_type="ffill+bfill")  # process nan brought by dataloader

qlib/contrib/model/pytorch_hist.py

@@ -177,7 +177,7 @@ class HIST(Model):
 
     def get_daily_inter(self, df, shuffle=False):
         # organize the train data into daily batches
-        daily_count = df.groupby(level=0).size().values
+        daily_count = df.groupby(level=0, group_keys=False).size().values
         daily_index = np.roll(np.cumsum(daily_count), 1)
         daily_index[0] = 0
         if shuffle:

qlib/contrib/model/pytorch_igmtf.py

@@ -170,7 +170,7 @@ class IGMTF(Model):
 
     def get_daily_inter(self, df, shuffle=False):
         # organize the train data into daily batches
-        daily_count = df.groupby(level=0).size().values
+        daily_count = df.groupby(level=0, group_keys=False).size().values
         daily_index = np.roll(np.cumsum(daily_count), 1)
         daily_index[0] = 0
         if shuffle:

qlib/contrib/model/pytorch_krnn.py

@@ -368,7 +368,7 @@ class KRNN(Model):
 
     def get_daily_inter(self, df, shuffle=False):
         # organize the train data into daily batches
-        daily_count = df.groupby(level=0).size().values
+        daily_count = df.groupby(level=0, group_keys=False).size().values
         daily_index = np.roll(np.cumsum(daily_count), 1)
         daily_index[0] = 0
         if shuffle:

qlib/contrib/model/pytorch_nn.py

@@ -10,6 +10,7 @@ import os
 import gc
 import numpy as np
 import pandas as pd
+from packaging import version
 from typing import Callable, Optional, Text, Union
 from sklearn.metrics import roc_auc_score, mean_squared_error
 
@@ -146,19 +147,34 @@ class DNNModelPytorch(Model):
             raise NotImplementedError("optimizer {} is not supported!".format(optimizer))
 
         if scheduler == "default":
-            # Reduce learning rate when loss has stopped decrease
-            self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
-                self.train_optimizer,
-                mode="min",
-                factor=0.5,
-                patience=10,
-                verbose=True,
-                threshold=0.0001,
-                threshold_mode="rel",
-                cooldown=0,
-                min_lr=0.00001,
-                eps=1e-08,
-            )
+            # In torch version 2.7.0, the verbose parameter has been removed. Reference Link:
+            # https://github.com/pytorch/pytorch/pull/147301/files#diff-036a7470d5307f13c9a6a51c3a65dd014f00ca02f476c545488cd856bea9bcf2L1313
+            if version.parse(str(torch.__version__).split("+", maxsplit=1)[0]) <= version.parse("2.6.0"):
+                # Reduce learning rate when loss has stopped decrease
+                self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(  # pylint: disable=E1123
+                    self.train_optimizer,
+                    mode="min",
+                    factor=0.5,
+                    patience=10,
+                    verbose=True,
+                    threshold=0.0001,
+                    threshold_mode="rel",
+                    cooldown=0,
+                    min_lr=0.00001,
+                    eps=1e-08,
+                )
+            else:
+                self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
+                    self.train_optimizer,
+                    mode="min",
+                    factor=0.5,
+                    patience=10,
+                    threshold=0.0001,
+                    threshold_mode="rel",
+                    cooldown=0,
+                    min_lr=0.00001,
+                    eps=1e-08,
+                )
         elif scheduler is None:
             self.scheduler = None
         else:

qlib/contrib/model/pytorch_tabnet.py

@@ -317,7 +317,7 @@ class TabnetModel(Model):
             feature = x_train_values.float().to(self.device)
             label = y_train_values.float().to(self.device)
             priors = 1 - S_mask
-            (vec, sparse_loss) = self.tabnet_model(feature, priors)
+            vec, sparse_loss = self.tabnet_model(feature, priors)
             f = self.tabnet_decoder(vec)
             loss = self.pretrain_loss_fn(label, f, S_mask)
 
@@ -348,7 +348,7 @@ class TabnetModel(Model):
             S_mask = S_mask.to(self.device)
             priors = 1 - S_mask
             with torch.no_grad():
-                (vec, sparse_loss) = self.tabnet_model(feature, priors)
+                vec, sparse_loss = self.tabnet_model(feature, priors)
                 f = self.tabnet_decoder(vec)
 
                 loss = self.pretrain_loss_fn(label, f, S_mask)

qlib/contrib/online/utils.py

@@ -12,6 +12,7 @@ from ...data import D
 from ...config import C
 from ...log import get_module_logger
 from ...utils import get_next_trading_date
+from ...utils.pickle_utils import restricted_pickle_load
 from ...backtest.exchange import Exchange
 
 log = get_module_logger("utils")
@@ -30,7 +31,7 @@ def load_instance(file_path):
     if not file_path.exists():
         raise ValueError("Cannot find file {}".format(file_path))
     with file_path.open("rb") as fr:
-        instance = pickle.load(fr)
+        instance = restricted_pickle_load(fr)
     return instance
 
 

qlib/contrib/ops/high_freq.py

@@ -96,7 +96,7 @@ class DayCumsum(ElemOperator):
     def _load_internal(self, instrument, start_index, end_index, freq):
         _calendar = get_calendar_day(freq=freq)
         series = self.feature.load(instrument, start_index, end_index, freq)
-        return series.groupby(_calendar[series.index]).transform(self.period_cusum)
+        return series.groupby(_calendar[series.index], group_keys=False).transform(self.period_cusum)
 
 
 class DayLast(ElemOperator):
@@ -116,7 +116,7 @@ class DayLast(ElemOperator):
     def _load_internal(self, instrument, start_index, end_index, freq):
         _calendar = get_calendar_day(freq=freq)
         series = self.feature.load(instrument, start_index, end_index, freq)
-        return series.groupby(_calendar[series.index]).transform("last")
+        return series.groupby(_calendar[series.index], group_keys=False).transform("last")
 
 
 class FFillNan(ElemOperator):
@@ -135,7 +135,7 @@ class FFillNan(ElemOperator):
 
     def _load_internal(self, instrument, start_index, end_index, freq):
         series = self.feature.load(instrument, start_index, end_index, freq)
-        return series.fillna(method="ffill")
+        return series.ffill()
 
 
 class BFillNan(ElemOperator):
@@ -154,7 +154,7 @@ class BFillNan(ElemOperator):
 
     def _load_internal(self, instrument, start_index, end_index, freq):
         series = self.feature.load(instrument, start_index, end_index, freq)
-        return series.fillna(method="bfill")
+        return series.bfill()
 
 
 class Date(ElemOperator):

qlib/contrib/report/analysis_model/__init__.py

@@ -3,5 +3,4 @@
 
 from .analysis_model_performance import model_performance_graph
 
-
 __all__ = ["model_performance_graph"]

qlib/contrib/report/analysis_model/analysis_model_performance.py

@@ -38,7 +38,7 @@ def _group_return(pred_label: pd.DataFrame = None, reverse: bool = False, N: int
     t_df = pd.DataFrame(
         {
             "Group%d"
-            % (i + 1): pred_label_drop.groupby(level="datetime")["label"].apply(
+            % (i + 1): pred_label_drop.groupby(level="datetime", group_keys=False)["label"].apply(
                 lambda x: x[len(x) // N * i : len(x) // N * (i + 1)].mean()  # pylint: disable=W0640
             )
             for i in range(N)
@@ -50,7 +50,7 @@ def _group_return(pred_label: pd.DataFrame = None, reverse: bool = False, N: int
     t_df["long-short"] = t_df["Group1"] - t_df["Group%d" % N]
 
     # Long-Average
-    t_df["long-average"] = t_df["Group1"] - pred_label.groupby(level="datetime")["label"].mean()
+    t_df["long-average"] = t_df["Group1"] - pred_label.groupby(level="datetime", group_keys=False)["label"].mean()
 
     t_df = t_df.dropna(how="all")  # for days which does not contain label
     # Cumulative Return By Group
@@ -137,7 +137,9 @@ def _pred_ic(
 
     ic_df = pd.concat(
         [
-            pred_label.groupby(level="datetime").apply(partial(_corr_series, method=_methods_mapping[m])).rename(m)
+            pred_label.groupby(level="datetime", group_keys=False)
+            .apply(partial(_corr_series, method=_methods_mapping[m]))
+            .rename(m)
             for m in methods
         ],
         axis=1,
@@ -145,7 +147,7 @@ def _pred_ic(
     _ic = ic_df.iloc(axis=1)[0]
 
     _index = _ic.index.get_level_values(0).astype("str").str.replace("-", "").str.slice(0, 6)
-    _monthly_ic = _ic.groupby(_index).mean()
+    _monthly_ic = _ic.groupby(_index, group_keys=False).mean()
     _monthly_ic.index = pd.MultiIndex.from_arrays(
         [_monthly_ic.index.str.slice(0, 4), _monthly_ic.index.str.slice(4, 6)],
         names=["year", "month"],
@@ -220,8 +222,10 @@ def _pred_ic(
 
 def _pred_autocorr(pred_label: pd.DataFrame, lag=1, **kwargs) -> tuple:
     pred = pred_label.copy()
-    pred["score_last"] = pred.groupby(level="instrument")["score"].shift(lag)
-    ac = pred.groupby(level="datetime").apply(lambda x: x["score"].rank(pct=True).corr(x["score_last"].rank(pct=True)))
+    pred["score_last"] = pred.groupby(level="instrument", group_keys=False)["score"].shift(lag)
+    ac = pred.groupby(level="datetime", group_keys=False).apply(
+        lambda x: x["score"].rank(pct=True).corr(x["score_last"].rank(pct=True))
+    )
     _df = ac.to_frame("value")
     ac_figure = ScatterGraph(
         _df,
@@ -235,13 +239,13 @@ def _pred_autocorr(pred_label: pd.DataFrame, lag=1, **kwargs) -> tuple:
 
 def _pred_turnover(pred_label: pd.DataFrame, N=5, lag=1, **kwargs) -> tuple:
     pred = pred_label.copy()
-    pred["score_last"] = pred.groupby(level="instrument")["score"].shift(lag)
-    top = pred.groupby(level="datetime").apply(
+    pred["score_last"] = pred.groupby(level="instrument", group_keys=False)["score"].shift(lag)
+    top = pred.groupby(level="datetime", group_keys=False).apply(
         lambda x: 1
         - x.nlargest(len(x) // N, columns="score").index.isin(x.nlargest(len(x) // N, columns="score_last").index).sum()
         / (len(x) // N)
     )
-    bottom = pred.groupby(level="datetime").apply(
+    bottom = pred.groupby(level="datetime", group_keys=False).apply(
         lambda x: 1
         - x.nsmallest(len(x) // N, columns="score")
         .index.isin(x.nsmallest(len(x) // N, columns="score_last").index)
@@ -313,7 +317,7 @@ def model_performance_graph(
                                 2017-12-15  -0.102778       -0.102778
 
 
-    :param lag: `pred.groupby(level='instrument')['score'].shift(lag)`. It will be only used in the auto-correlation computing.
+    :param lag: `pred.groupby(level='instrument', group_keys=False)['score'].shift(lag)`. It will be only used in the auto-correlation computing.
     :param N: group number, default 5.
     :param reverse: if `True`, `pred['score'] *= -1`.
     :param rank: if **True**, calculate rank ic.

qlib/contrib/report/analysis_position/__init__.py

@@ -7,5 +7,4 @@ from .report import report_graph
 from .rank_label import rank_label_graph
 from .risk_analysis import risk_analysis_graph
 
-
 __all__ = ["cumulative_return_graph", "score_ic_graph", "report_graph", "rank_label_graph", "risk_analysis_graph"]

qlib/contrib/report/analysis_position/cumulative_return.py

@@ -38,7 +38,7 @@ def _get_cum_return_data_with_position(
 
     _cumulative_return_df["label"] = _cumulative_return_df["label"] - _cumulative_return_df["bench"]
     _cumulative_return_df = _cumulative_return_df.dropna()
-    df_gp = _cumulative_return_df.groupby(level="datetime")
+    df_gp = _cumulative_return_df.groupby(level="datetime", group_keys=False)
     result_list = []
     for gp in df_gp:
         date = gp[0]

qlib/contrib/report/analysis_position/parse_position.py

@@ -33,7 +33,7 @@ def parse_position(position: dict = None) -> pd.DataFrame:
 
     position_weight_df = get_stock_weight_df(position)
     # If the day does not exist, use the last weight
-    position_weight_df.fillna(method="ffill", inplace=True)
+    position_weight_df.ffill(inplace=True)
 
     previous_data = {"date": None, "code_list": []}
 
@@ -132,7 +132,7 @@ def _calculate_label_rank(df: pd.DataFrame) -> pd.DataFrame:
         g_df["excess_return"] = g_df[_label_name] - g_df[_label_name].mean()
         return g_df
 
-    return df.groupby(level="datetime").apply(_calculate_day_value)
+    return df.groupby(level="datetime", group_keys=False).apply(_calculate_day_value)
 
 
 def get_position_data(

qlib/contrib/report/analysis_position/rank_label.py

@@ -31,7 +31,7 @@ def _get_figure_with_position(
     )
 
     res_dict = dict()
-    _pos_gp = _position_df.groupby(level=1)
+    _pos_gp = _position_df.groupby(level=1, group_keys=False)
     for _item in _pos_gp:
         _date = _item[0]
         _day_df = _item[1]

qlib/contrib/report/analysis_position/risk_analysis.py

@@ -63,9 +63,11 @@ def _get_monthly_risk_analysis_with_report(report_normal_df: pd.DataFrame) -> pd
     """
 
     # Group by month
-    report_normal_gp = report_normal_df.groupby([report_normal_df.index.year, report_normal_df.index.month])
+    report_normal_gp = report_normal_df.groupby(
+        [report_normal_df.index.year, report_normal_df.index.month], group_keys=False
+    )
     # report_long_short_gp = report_long_short_df.groupby(
-    #     [report_long_short_df.index.year, report_long_short_df.index.month]
+    #     [report_long_short_df.index.year, report_long_short_df.index.month], group_keys=False
     # )
 
     gp_month = sorted(set(report_normal_gp.size().index))
@@ -97,7 +99,7 @@ def _get_monthly_analysis_with_feature(monthly_df: pd.DataFrame, feature: str =
     :param feature:
     :return:
     """
-    _monthly_df_gp = monthly_df.reset_index().groupby(["level_1"])
+    _monthly_df_gp = monthly_df.reset_index().groupby(["level_1"], group_keys=False)
 
     _name_df = _monthly_df_gp.get_group(feature).set_index(["level_0", "level_1"])
     _temp_df = _name_df.pivot_table(index="date", values=["risk"], columns=_name_df.index)

qlib/contrib/report/analysis_position/score_ic.py

@@ -15,8 +15,10 @@ def _get_score_ic(pred_label: pd.DataFrame):
     """
     concat_data = pred_label.copy()
     concat_data.dropna(axis=0, how="any", inplace=True)
-    _ic = concat_data.groupby(level="datetime").apply(lambda x: x["label"].corr(x["score"]))
-    _rank_ic = concat_data.groupby(level="datetime").apply(lambda x: x["label"].corr(x["score"], method="spearman"))
+    _ic = concat_data.groupby(level="datetime", group_keys=False).apply(lambda x: x["label"].corr(x["score"]))
+    _rank_ic = concat_data.groupby(level="datetime", group_keys=False).apply(
+        lambda x: x["label"].corr(x["score"], method="spearman")
+    )
     return pd.DataFrame({"ic": _ic, "rank_ic": _rank_ic})
 
 

qlib/contrib/report/data/ana.py

@@ -12,6 +12,7 @@ Here is an example.
     fa.plot_all(wspace=0.3, sub_figsize=(12, 3), col_n=5)
 
 """
+
 import pandas as pd
 import numpy as np
 from qlib.contrib.report.data.base import FeaAnalyser
@@ -72,10 +73,10 @@ class ValueCNT(FeaAnalyser):
         self._val_cnt = {}
         for col, item in self._dataset.items():
             if not super().skip(col):
-                self._val_cnt[col] = item.groupby(DT_COL_NAME).apply(lambda s: len(s.unique()))
+                self._val_cnt[col] = item.groupby(DT_COL_NAME, group_keys=False).apply(lambda s: len(s.unique()))
         self._val_cnt = pd.DataFrame(self._val_cnt)
         if self.ratio:
-            self._val_cnt = self._val_cnt.div(self._dataset.groupby(DT_COL_NAME).size(), axis=0)
+            self._val_cnt = self._val_cnt.div(self._dataset.groupby(DT_COL_NAME, group_keys=False).size(), axis=0)
 
         # TODO: transfer this feature to other analysers
         ymin, ymax = self._val_cnt.min().min(), self._val_cnt.max().max()
@@ -98,7 +99,7 @@ class FeaInfAna(NumFeaAnalyser):
         self._inf_cnt = {}
         for col, item in self._dataset.items():
             if not super().skip(col):
-                self._inf_cnt[col] = item.apply(np.isinf).astype(np.int).groupby(DT_COL_NAME).sum()
+                self._inf_cnt[col] = item.apply(np.isinf).astype(np.int).groupby(DT_COL_NAME, group_keys=False).sum()
         self._inf_cnt = pd.DataFrame(self._inf_cnt)
 
     def skip(self, col):
@@ -111,7 +112,7 @@ class FeaInfAna(NumFeaAnalyser):
 
 class FeaNanAna(FeaAnalyser):
     def calc_stat_values(self):
-        self._nan_cnt = self._dataset.isna().groupby(DT_COL_NAME).sum()
+        self._nan_cnt = self._dataset.isna().groupby(DT_COL_NAME, group_keys=False).sum()
 
     def skip(self, col):
         return (col not in self._nan_cnt) or (self._nan_cnt[col].sum() == 0)
@@ -123,8 +124,8 @@ class FeaNanAna(FeaAnalyser):
 
 class FeaNanAnaRatio(FeaAnalyser):
     def calc_stat_values(self):
-        self._nan_cnt = self._dataset.isna().groupby(DT_COL_NAME).sum()
-        self._total_cnt = self._dataset.groupby(DT_COL_NAME).size()
+        self._nan_cnt = self._dataset.isna().groupby(DT_COL_NAME, group_keys=False).sum()
+        self._total_cnt = self._dataset.groupby(DT_COL_NAME, group_keys=False).size()
 
     def skip(self, col):
         return (col not in self._nan_cnt) or (self._nan_cnt[col].sum() == 0)
@@ -176,8 +177,8 @@ class FeaSkewTurt(NumFeaAnalyser):
 
 class FeaMeanStd(NumFeaAnalyser):
     def calc_stat_values(self):
-        self._std = self._dataset.groupby(DT_COL_NAME).std()
-        self._mean = self._dataset.groupby(DT_COL_NAME).mean()
+        self._std = self._dataset.groupby(DT_COL_NAME, group_keys=False).std()
+        self._mean = self._dataset.groupby(DT_COL_NAME, group_keys=False).mean()
 
     def plot_single(self, col, ax):
         self._mean[col].plot(ax=ax, label="mean")

qlib/contrib/report/data/base.py

@@ -7,6 +7,7 @@ Assumptions
 - The analyse each feature individually
 
 """
+
 import pandas as pd
 from qlib.log import TimeInspector
 from qlib.contrib.report.utils import sub_fig_generator

qlib/contrib/rolling/ddgda.py

@@ -14,6 +14,7 @@ from qlib.model.meta.task import MetaTask
 from qlib.model.trainer import TrainerR
 from qlib.typehint import Literal
 from qlib.utils import init_instance_by_config
+from qlib.utils.pickle_utils import restricted_pickle_load
 from qlib.workflow import R
 from qlib.workflow.task.utils import replace_task_handler_with_cache
 
@@ -298,7 +299,7 @@ class DDGDA(Rolling):
         # but their task test segment are not aligned! It worked in my previous experiment.
         # So the misalignment will not affect the effectiveness of the method.
         with self._internal_data_path.open("rb") as f:
-            internal_data = pickle.load(f)
+            internal_data = restricted_pickle_load(f)
 
         md = MetaDatasetDS(exp_name=internal_data, **kwargs)
 
@@ -360,7 +361,7 @@ class DDGDA(Rolling):
         )
 
         with self._internal_data_path.open("rb") as f:
-            internal_data = pickle.load(f)
+            internal_data = restricted_pickle_load(f)
         mds = MetaDatasetDS(exp_name=internal_data, **kwargs)
 
         # 3) meta model make inference and get new qlib task

qlib/contrib/strategy/__init__.py

@@ -16,7 +16,6 @@ from .rule_strategy import (
 
 from .cost_control import SoftTopkStrategy
 
-
 __all__ = [
     "TopkDropoutStrategy",
     "WeightStrategyBase",

qlib/contrib/strategy/cost_control.py

@@ -1,101 +1,117 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
-"""
-This strategy is not well maintained
-"""
-
 
 from .order_generator import OrderGenWInteract
 from .signal_strategy import WeightStrategyBase
-import copy
 
 
 class SoftTopkStrategy(WeightStrategyBase):
     def __init__(
         self,
-        model,
-        dataset,
-        topk,
+        model=None,
+        dataset=None,
+        topk=None,
         order_generator_cls_or_obj=OrderGenWInteract,
         max_sold_weight=1.0,
+        trade_impact_limit=None,
         risk_degree=0.95,
         buy_method="first_fill",
-        trade_exchange=None,
-        level_infra=None,
-        common_infra=None,
         **kwargs,
     ):
         """
+        Refactored SoftTopkStrategy with a budget-constrained rebalancing engine.
+
         Parameters
         ----------
         topk : int
-            top-N stocks to buy
+            The number of top-N stocks to be held in the portfolio.
+        trade_impact_limit : float
+            Maximum weight change for each stock in one trade. If None, fallback to max_sold_weight.
+        max_sold_weight : float
+            Backward-compatible alias for trade_impact_limit. Use 1.0 to effectively disable the limit.
         risk_degree : float
-            position percentage of total value buy_method:
-
-                rank_fill: assign the weight stocks that rank high first(1/topk max)
-                average_fill: assign the weight to the stocks rank high averagely.
+            The target percentage of total value to be invested.
         """
         super(SoftTopkStrategy, self).__init__(
-            model, dataset, order_generator_cls_or_obj, trade_exchange, level_infra, common_infra, **kwargs
+            model=model, dataset=dataset, order_generator_cls_or_obj=order_generator_cls_or_obj, **kwargs
         )
+
         self.topk = topk
-        self.max_sold_weight = max_sold_weight
+        self.trade_impact_limit = trade_impact_limit if trade_impact_limit is not None else max_sold_weight
         self.risk_degree = risk_degree
         self.buy_method = buy_method
 
     def get_risk_degree(self, trade_step=None):
-        """get_risk_degree
-        Return the proportion of your total value you will used in investment.
-        Dynamically risk_degree will result in Market timing
-        """
-        # It will use 95% amount of your total value by default
         return self.risk_degree
 
-    def generate_target_weight_position(self, score, current, trade_start_time, trade_end_time):
+    def generate_target_weight_position(self, score, current, trade_start_time, trade_end_time, **kwargs):
         """
-        Parameters
-        ----------
-        score:
-            pred score for this trade date, pd.Series, index is stock_id, contain 'score' column
-        current:
-            current position, use Position() class
-        trade_date:
-            trade date
-
-            generate target position from score for this date and the current position
-
-            The cache is not considered in the position
+        Generates target position using Proportional Budget Allocation.
+        Ensures deterministic sells and synchronized buys under impact limits.
         """
-        # TODO:
-        # If the current stock list is more than topk(eg. The weights are modified
-        # by risk control), the weight will not be handled correctly.
-        buy_signal_stocks = set(score.sort_values(ascending=False).iloc[: self.topk].index)
-        cur_stock_weight = current.get_stock_weight_dict(only_stock=True)
 
-        if len(cur_stock_weight) == 0:
-            final_stock_weight = {code: 1 / self.topk for code in buy_signal_stocks}
-        else:
-            final_stock_weight = copy.deepcopy(cur_stock_weight)
-            sold_stock_weight = 0.0
-            for stock_id in final_stock_weight:
-                if stock_id not in buy_signal_stocks:
-                    sw = min(self.max_sold_weight, final_stock_weight[stock_id])
-                    sold_stock_weight += sw
-                    final_stock_weight[stock_id] -= sw
-            if self.buy_method == "first_fill":
-                for stock_id in buy_signal_stocks:
-                    add_weight = min(
-                        max(1 / self.topk - final_stock_weight.get(stock_id, 0), 0.0),
-                        sold_stock_weight,
-                    )
-                    final_stock_weight[stock_id] = final_stock_weight.get(stock_id, 0.0) + add_weight
-                    sold_stock_weight -= add_weight
-            elif self.buy_method == "average_fill":
-                for stock_id in buy_signal_stocks:
-                    final_stock_weight[stock_id] = final_stock_weight.get(stock_id, 0.0) + sold_stock_weight / len(
-                        buy_signal_stocks
-                    )
-            else:
-                raise ValueError("Buy method not found")
-        return final_stock_weight
+        if self.topk is None or self.topk <= 0:
+            return {}
+
+        def apply_impact_limit(weight):
+            return weight if self.trade_impact_limit is None else min(weight, self.trade_impact_limit)
+
+        ideal_per_stock = self.risk_degree / self.topk
+        ideal_list = score.sort_values(ascending=False).iloc[: self.topk].index.tolist()
+
+        cur_weights = current.get_stock_weight_dict(only_stock=True)
+        initial_total_weight = sum(cur_weights.values())
+
+        # --- Case A: Cold Start ---
+        if not cur_weights:
+            fill = apply_impact_limit(ideal_per_stock)
+            return {code: fill for code in ideal_list}
+
+        # --- Case B: Rebalancing ---
+        all_tickers = set(cur_weights.keys()) | set(ideal_list)
+        next_weights = {t: cur_weights.get(t, 0.0) for t in all_tickers}
+
+        # Phase 1: Deterministic Sell Phase
+        released_cash = 0.0
+        for t in list(next_weights.keys()):
+            cur = next_weights[t]
+            if cur <= 1e-8:
+                continue
+
+            if t not in ideal_list:
+                sell = apply_impact_limit(cur)
+                next_weights[t] -= sell
+                released_cash += sell
+            elif cur > ideal_per_stock + 1e-8:
+                excess = cur - ideal_per_stock
+                sell = apply_impact_limit(excess)
+                next_weights[t] -= sell
+                released_cash += sell
+
+        # Phase 2: Budget Calculation
+        # Budget = Cash from sells + Available space from target risk degree
+        total_budget = released_cash + (self.risk_degree - initial_total_weight)
+
+        # Phase 3: Proportional Buy Allocation
+        if total_budget > 1e-8:
+            shortfalls = {
+                t: (ideal_per_stock - next_weights.get(t, 0.0))
+                for t in ideal_list
+                if next_weights.get(t, 0.0) < ideal_per_stock - 1e-8
+            }
+
+            if shortfalls:
+                total_shortfall = sum(shortfalls.values())
+                # Normalize total_budget to not exceed total_shortfall
+                available_to_spend = min(total_budget, total_shortfall)
+
+                for t, shortfall in shortfalls.items():
+                    # Every stock gets its fair share based on its distance to target
+                    share_of_budget = (shortfall / total_shortfall) * available_to_spend
+
+                    # Capped by impact limit
+                    max_buy_cap = apply_impact_limit(shortfall)
+
+                    next_weights[t] += min(share_of_budget, max_buy_cap)
+
+        return {k: v for k, v in next_weights.items() if v > 1e-8}

qlib/contrib/strategy/optimizer/__init__.py

@@ -5,5 +5,4 @@ from .base import BaseOptimizer
 from .optimizer import PortfolioOptimizer
 from .enhanced_indexing import EnhancedIndexingOptimizer
 
-
 __all__ = ["BaseOptimizer", "PortfolioOptimizer", "EnhancedIndexingOptimizer"]

qlib/contrib/strategy/optimizer/enhanced_indexing.py

@@ -9,7 +9,6 @@ from typing import Union, Optional, Dict, Any, List
 from qlib.log import get_module_logger
 from .base import BaseOptimizer
 
-
 logger = get_module_logger("EnhancedIndexingOptimizer")
 
 

qlib/contrib/strategy/order_generator.py

@@ -4,6 +4,7 @@
 """
 This order generator is for strategies based on WeightStrategyBase
 """
+
 from ...backtest.position import Position
 from ...backtest.exchange import Exchange
 

qlib/contrib/strategy/rule_strategy.py

@@ -326,8 +326,10 @@ class SBBStrategyEMA(SBBStrategyBase):
         if instruments is None:
             warnings.warn("`instruments` is not set, will load all stocks")
             self.instruments = "all"
-        if isinstance(instruments, str):
+        elif isinstance(instruments, str):
             self.instruments = D.instruments(instruments)
+        elif isinstance(instruments, List):
+            self.instruments = instruments
         self.freq = freq
         super(SBBStrategyEMA, self).__init__(
             outer_trade_decision, level_infra, common_infra, trade_exchange=trade_exchange, **kwargs
@@ -345,7 +347,7 @@ class SBBStrategyEMA(SBBStrategyBase):
         self.signal = {}
 
         if not signal_df.empty:
-            for stock_id, stock_val in signal_df.groupby(level="instrument"):
+            for stock_id, stock_val in signal_df.groupby(level="instrument", group_keys=False):
                 self.signal[stock_id] = stock_val["signal"].droplevel(level="instrument")
 
     def reset_level_infra(self, level_infra):
@@ -432,7 +434,7 @@ class ACStrategy(BaseStrategy):
         self.signal = {}
 
         if not signal_df.empty:
-            for stock_id, stock_val in signal_df.groupby(level="instrument"):
+            for stock_id, stock_val in signal_df.groupby(level="instrument", group_keys=False):
                 self.signal[stock_id] = stock_val["volatility"].droplevel(level="instrument")
 
     def reset_level_infra(self, level_infra):

qlib/contrib/torch.py

@@ -1,10 +1,11 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 """
-    This module is not a necessary part of Qlib.
-    They are just some tools for convenience
-    It is should not imported into the core part of qlib
+This module is not a necessary part of Qlib.
+They are just some tools for convenience
+It is should not imported into the core part of qlib
 """
+
 import torch
 import numpy as np
 import pandas as pd

qlib/contrib/tuner/launcher.py

@@ -13,7 +13,6 @@ import yaml
 
 from .config import TunerConfigManager
 
-
 args_parser = argparse.ArgumentParser(prog="tuner")
 args_parser.add_argument(
     "-c",