docs: replace broken RD-Agent demo links in README (#2150)

Co-authored-by: monkeyjack123 <monkeyjack123@users.noreply.github.com>
Co-authored-by: Linlang Lv (iSoftStone Information) <v-llv@microsoft.com>

.github/workflows/test_qlib_from_pip.yml

@@ -1,5 +1,9 @@
 name: Test qlib from pip
 
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
 on:
   push:
     branches: [ main ]

.github/workflows/test_qlib_from_source.yml

@@ -1,5 +1,9 @@
 name: Test qlib from source
 
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
 on:
   push:
     branches: [ main ]
@@ -76,8 +80,11 @@ jobs:
       run: |
         make mypy
     
+    # Due to issues that cannot be automatically fixed when running `nbqa black . -l 120 --check --diff` on Jupyter notebooks,
+    # we reverted to a version of `black` earlier than 26.1.0 before performing the checks.
     - name: Check Qlib ipynb with nbqa
       run: |
+        python -m pip install "black<26.1"
         make nbqa
 
     - name: Test data downloads

.github/workflows/test_qlib_from_source_slow.yml

@@ -1,5 +1,9 @@
 name: Test qlib from source slow
 
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
 on:
   push:
     branches: [ main ]

README.md

@@ -17,14 +17,13 @@ We are excited to announce the release of **RD-Agent**📢, a powerful tool that
 
 RD-Agent is now available on [GitHub](https://github.com/microsoft/RD-Agent), and we welcome your star🌟!
 
-To learn more, please visit our [♾️Demo page](https://rdagent.azurewebsites.net/). Here, you will find demo videos in both English and Chinese to help you better understand the scenario and usage of RD-Agent.
+To learn more, please visit the [RD-Agent repository](https://github.com/microsoft/RD-Agent). We have prepared several public demo videos for you:
 
-We have prepared several demo videos for you:
 | Scenario | Demo video (English) | Demo video (中文) |
 | --                      | ------    | ------    |
-| Quant Factor Mining | [Link](https://rdagent.azurewebsites.net/factor_loop?lang=en) | [Link](https://rdagent.azurewebsites.net/factor_loop?lang=zh) |
-| Quant Factor Mining from reports | [Link](https://rdagent.azurewebsites.net/report_factor?lang=en) | [Link](https://rdagent.azurewebsites.net/report_factor?lang=zh) |
-| Quant Model Optimization | [Link](https://rdagent.azurewebsites.net/model_loop?lang=en) | [Link](https://rdagent.azurewebsites.net/model_loop?lang=zh) |
+| Quant Factor Mining | [YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s) | [YouTube](https://www.youtube.com/watch?v=X4DK2QZKaKY&t=6s) |
+| Quant Factor Mining from reports | [YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c) | [YouTube](https://www.youtube.com/watch?v=ECLTXVcSx-c) |
+| Quant Model Optimization | [YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s) | [YouTube](https://www.youtube.com/watch?v=dm0dWL49Bc0&t=104s) |
 
 - 📃**Paper**: [R&D-Agent-Quant: A Multi-Agent Framework for Data-Centric Factors and Model Joint Optimization](https://arxiv.org/abs/2505.15155)
 - 👾**Code**: https://github.com/microsoft/RD-Agent/

docs/conf.py

@@ -21,8 +21,7 @@
 import os
 import sys
 
-import pkg_resources
-
+from importlib.metadata import version as ver
 
 # -- General configuration ------------------------------------------------
 
@@ -63,9 +62,9 @@ author = "Microsoft"
 # built documents.
 #
 # The short X.Y version.
-version = pkg_resources.get_distribution("pyqlib").version
+version = ver("pyqlib")
 # The full version, including alpha/beta/rc tags.
-release = pkg_resources.get_distribution("pyqlib").version
+release = version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

examples/benchmarks/TFT/tft.py

@@ -19,7 +19,6 @@ from qlib.model.base import ModelFT
 from qlib.data.dataset import DatasetH
 from qlib.data.dataset.handler import DataHandlerLP
 
-
 # To register new datasets, please add them here.
 ALLOW_DATASET = ["Alpha158", "Alpha360"]
 # To register new datasets, please add their configurations here.

examples/benchmarks/TRA/src/dataset.py

@@ -8,7 +8,6 @@ import pandas as pd
 
 from qlib.data.dataset import DatasetH
 
-
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
 

examples/benchmarks_dynamic/DDG-DA/vis_data.py

@@ -1,9 +1,10 @@
-import pickle
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
 import seaborn as sns
 
+from qlib.utils.pickle_utils import restricted_pickle_load
+
 sns.set(color_codes=True)
 plt.rcParams["font.sans-serif"] = "SimHei"
 plt.rcParams["axes.unicode_minus"] = False
@@ -18,7 +19,7 @@ from tqdm.auto import tqdm
 
 # +
 with open("./internal_data_s20.pkl", "rb") as f:
-    data = pickle.load(f)
+    data = restricted_pickle_load(f)
 
 data.data_ic_df.columns.names = ["start_date", "end_date"]
 
@@ -52,7 +53,7 @@ pd.DataFrame(meta_m.tn.twm.linear.weight.detach().numpy()).T[0].rolling(5).mean(
 
 # +
 with open("./tasks_s20.pkl", "rb") as f:
-    tasks = pickle.load(f)
+    tasks = restricted_pickle_load(f)
 
 task_df = {}
 for t in tasks:

examples/highfreq/workflow.py

@@ -4,11 +4,11 @@
 import fire
 
 import qlib
-import pickle
 from qlib.constant import REG_CN
 from qlib.config import HIGH_FREQ_CONFIG
 
 from qlib.utils import init_instance_by_config
+from qlib.utils.pickle_utils import restricted_pickle_load
 from qlib.data.dataset.handler import DataHandlerLP
 from qlib.data.ops import Operators
 from qlib.data.data import Cal
@@ -125,10 +125,10 @@ class HighfreqWorkflow:
         del dataset, dataset_backtest
         ##=============reload dataset=============
         with open("dataset.pkl", "rb") as file_dataset:
-            dataset = pickle.load(file_dataset)
+            dataset = restricted_pickle_load(file_dataset)
 
         with open("dataset_backtest.pkl", "rb") as file_dataset_backtest:
-            dataset_backtest = pickle.load(file_dataset_backtest)
+            dataset_backtest = restricted_pickle_load(file_dataset_backtest)
 
         self._prepare_calender_cache()
         ##=============reinit dataset=============

examples/model_interpreter/feature.py

@@ -9,7 +9,6 @@ from qlib.utils import init_instance_by_config
 from qlib.tests.data import GetData
 from qlib.tests.config import CSI300_GBDT_TASK
 
-
 if __name__ == "__main__":
     # use default data
     provider_uri = "~/.qlib/qlib_data/cn_data"  # target_dir

examples/nested_decision_execution/workflow.py

@@ -95,7 +95,6 @@ pos  0.000000
 [1706497:MainThread](2021-12-07 14:08:30,627) INFO - qlib.timer - [log.py:113] - Time cost: 0.014s | waiting `async_log` Done
 """
 
-
 from copy import deepcopy
 import qlib
 import fire

examples/online_srv/update_online_pred.py

@@ -7,6 +7,7 @@ There are two parts including first_train and update_online_pred.
 Firstly, we will finish the training and set the trained models to the `online` models.
 Next, we will finish updating online predictions.
 """
+
 import copy
 import fire
 import qlib

examples/orderbook_data/create_dataset.py

@@ -6,6 +6,7 @@ NOTE:
 - !!!!!!!!!!!!!!!TODO!!!!!!!!!!!!!!!!!!!:
     - Its structure is not well designed and very ugly, your contribution is welcome to make importing dataset easier
 """
+
 from datetime import date, datetime as dt
 import os
 from pathlib import Path

examples/rl_order_execution/scripts/merge_orders.py

@@ -1,13 +1,15 @@
-import pickle
 import os
 import pandas as pd
 from tqdm import tqdm
 
+from qlib.utils.pickle_utils import restricted_pickle_load
+
 for tag in ["test", "valid"]:
     files = os.listdir(os.path.join("data/orders/", tag))
     dfs = []
     for f in tqdm(files):
-        df = pickle.load(open(os.path.join("data/orders/", tag, f), "rb"))
+        with open(os.path.join("data/orders/", tag, f), "rb") as fr:
+            df = restricted_pickle_load(fr)
         df = df.drop(["$close0"], axis=1)
         dfs.append(df)
 

examples/rolling_process_data/workflow.py

@@ -3,12 +3,12 @@
 
 import qlib
 import fire
-import pickle
 
 from datetime import datetime
 from qlib.constant import REG_CN
 from qlib.data.dataset.handler import DataHandlerLP
 from qlib.utils import init_instance_by_config
+from qlib.utils.pickle_utils import restricted_pickle_load
 from qlib.tests.data import GetData
 
 
@@ -42,7 +42,7 @@ class RollingDataWorkflow:
 
     def _load_pre_handler(self, path):
         with open(path, "rb") as file_dataset:
-            pre_handler = pickle.load(file_dataset)
+            pre_handler = restricted_pickle_load(file_dataset)
         return pre_handler
 
     def rolling_process(self):

examples/workflow_by_code.py

@@ -7,6 +7,7 @@ Qlib provides two kinds of interfaces.
 
 The interface of (1) is `qrun XXX.yaml`.  The interface of (2) is script like this, which nearly does the same thing as `qrun XXX.yaml`
 """
+
 import qlib
 from qlib.constant import REG_CN
 from qlib.utils import init_instance_by_config, flatten_dict
@@ -15,7 +16,6 @@ from qlib.workflow.record_temp import SignalRecord, PortAnaRecord, SigAnaRecord
 from qlib.tests.data import GetData
 from qlib.tests.config import CSI300_BENCH, CSI300_GBDT_TASK
 
-
 if __name__ == "__main__":
     # use default data
     provider_uri = "~/.qlib/qlib_data/cn_data"  # target_dir

pyproject.toml

@@ -69,10 +69,9 @@ rl = [
   "torch",
   "numpy<2.0.0",
 ]
-# We exclude black version 26.1.0 due to known issues with nbqa when formatting Jupyter notebooks,
-# which can cause false-positive --check results and inconsistent notebook formatting.
+
 lint = [
-  "black!=26.1.0",
+  "black",
   "pylint",
   "mypy<1.5.0",
   "flake8",

qlib/backtest/backtest.py

@@ -18,7 +18,6 @@ from tqdm.auto import tqdm
 
 from ..utils.time import Freq
 
-
 PORT_METRIC = Dict[str, Tuple[pd.DataFrame, dict]]
 INDICATOR_METRIC = Dict[str, Tuple[pd.DataFrame, Indicator]]
 

qlib/cli/data.py

@@ -4,6 +4,5 @@
 import fire
 from qlib.tests.data import GetData
 
-
 if __name__ == "__main__":
     fire.Fire(GetData)

qlib/config.py

@@ -10,6 +10,7 @@ Two modes are supported
 - server
 
 """
+
 from __future__ import annotations
 
 import os

qlib/contrib/data/dataset.py

@@ -11,7 +11,6 @@ from qlib.utils import init_instance_by_config
 
 from qlib.data.dataset import DatasetH
 
-
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
 

qlib/contrib/evaluate.py

@@ -20,7 +20,6 @@ from ..data import D
 from ..config import C
 from ..data.dataset.utils import get_level_index
 
-
 logger = get_module_logger("Evaluate")
 
 

qlib/contrib/meta/__init__.py

@@ -3,5 +3,4 @@
 
 from .data_selection import MetaTaskDS, MetaDatasetDS, MetaModelDS
 
-
 __all__ = ["MetaTaskDS", "MetaDatasetDS", "MetaModelDS"]

qlib/contrib/meta/data_selection/__init__.py

@@ -4,5 +4,4 @@
 from .dataset import MetaDatasetDS, MetaTaskDS
 from .model import MetaModelDS
 
-
 __all__ = ["MetaDatasetDS", "MetaTaskDS", "MetaModelDS"]

qlib/contrib/model/pytorch_tabnet.py

@@ -317,7 +317,7 @@ class TabnetModel(Model):
             feature = x_train_values.float().to(self.device)
             label = y_train_values.float().to(self.device)
             priors = 1 - S_mask
-            (vec, sparse_loss) = self.tabnet_model(feature, priors)
+            vec, sparse_loss = self.tabnet_model(feature, priors)
             f = self.tabnet_decoder(vec)
             loss = self.pretrain_loss_fn(label, f, S_mask)
 
@@ -348,7 +348,7 @@ class TabnetModel(Model):
             S_mask = S_mask.to(self.device)
             priors = 1 - S_mask
             with torch.no_grad():
-                (vec, sparse_loss) = self.tabnet_model(feature, priors)
+                vec, sparse_loss = self.tabnet_model(feature, priors)
                 f = self.tabnet_decoder(vec)
 
                 loss = self.pretrain_loss_fn(label, f, S_mask)

qlib/contrib/online/utils.py

@@ -12,6 +12,7 @@ from ...data import D
 from ...config import C
 from ...log import get_module_logger
 from ...utils import get_next_trading_date
+from ...utils.pickle_utils import restricted_pickle_load
 from ...backtest.exchange import Exchange
 
 log = get_module_logger("utils")
@@ -30,7 +31,7 @@ def load_instance(file_path):
     if not file_path.exists():
         raise ValueError("Cannot find file {}".format(file_path))
     with file_path.open("rb") as fr:
-        instance = pickle.load(fr)
+        instance = restricted_pickle_load(fr)
     return instance
 
 

qlib/contrib/report/analysis_model/__init__.py

@@ -3,5 +3,4 @@
 
 from .analysis_model_performance import model_performance_graph
 
-
 __all__ = ["model_performance_graph"]

qlib/contrib/report/analysis_position/__init__.py

@@ -7,5 +7,4 @@ from .report import report_graph
 from .rank_label import rank_label_graph
 from .risk_analysis import risk_analysis_graph
 
-
 __all__ = ["cumulative_return_graph", "score_ic_graph", "report_graph", "rank_label_graph", "risk_analysis_graph"]

qlib/contrib/report/data/ana.py

@@ -12,6 +12,7 @@ Here is an example.
     fa.plot_all(wspace=0.3, sub_figsize=(12, 3), col_n=5)
 
 """
+
 import pandas as pd
 import numpy as np
 from qlib.contrib.report.data.base import FeaAnalyser

qlib/contrib/report/data/base.py

@@ -7,6 +7,7 @@ Assumptions
 - The analyse each feature individually
 
 """
+
 import pandas as pd
 from qlib.log import TimeInspector
 from qlib.contrib.report.utils import sub_fig_generator

qlib/contrib/strategy/__init__.py

@@ -16,7 +16,6 @@ from .rule_strategy import (
 
 from .cost_control import SoftTopkStrategy
 
-
 __all__ = [
     "TopkDropoutStrategy",
     "WeightStrategyBase",

qlib/contrib/strategy/cost_control.py

@@ -1,101 +1,117 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
-"""
-This strategy is not well maintained
-"""
-
 
 from .order_generator import OrderGenWInteract
 from .signal_strategy import WeightStrategyBase
-import copy
 
 
 class SoftTopkStrategy(WeightStrategyBase):
     def __init__(
         self,
-        model,
-        dataset,
-        topk,
+        model=None,
+        dataset=None,
+        topk=None,
         order_generator_cls_or_obj=OrderGenWInteract,
         max_sold_weight=1.0,
+        trade_impact_limit=None,
         risk_degree=0.95,
         buy_method="first_fill",
-        trade_exchange=None,
-        level_infra=None,
-        common_infra=None,
         **kwargs,
     ):
         """
+        Refactored SoftTopkStrategy with a budget-constrained rebalancing engine.
+
         Parameters
         ----------
         topk : int
-            top-N stocks to buy
+            The number of top-N stocks to be held in the portfolio.
+        trade_impact_limit : float
+            Maximum weight change for each stock in one trade. If None, fallback to max_sold_weight.
+        max_sold_weight : float
+            Backward-compatible alias for trade_impact_limit. Use 1.0 to effectively disable the limit.
         risk_degree : float
-            position percentage of total value buy_method:
-
-                rank_fill: assign the weight stocks that rank high first(1/topk max)
-                average_fill: assign the weight to the stocks rank high averagely.
+            The target percentage of total value to be invested.
         """
         super(SoftTopkStrategy, self).__init__(
-            model, dataset, order_generator_cls_or_obj, trade_exchange, level_infra, common_infra, **kwargs
+            model=model, dataset=dataset, order_generator_cls_or_obj=order_generator_cls_or_obj, **kwargs
         )
+
         self.topk = topk
-        self.max_sold_weight = max_sold_weight
+        self.trade_impact_limit = trade_impact_limit if trade_impact_limit is not None else max_sold_weight
         self.risk_degree = risk_degree
         self.buy_method = buy_method
 
     def get_risk_degree(self, trade_step=None):
-        """get_risk_degree
-        Return the proportion of your total value you will used in investment.
-        Dynamically risk_degree will result in Market timing
-        """
-        # It will use 95% amount of your total value by default
         return self.risk_degree
 
-    def generate_target_weight_position(self, score, current, trade_start_time, trade_end_time):
+    def generate_target_weight_position(self, score, current, trade_start_time, trade_end_time, **kwargs):
         """
-        Parameters
-        ----------
-        score:
-            pred score for this trade date, pd.Series, index is stock_id, contain 'score' column
-        current:
-            current position, use Position() class
-        trade_date:
-            trade date
-
-            generate target position from score for this date and the current position
-
-            The cache is not considered in the position
+        Generates target position using Proportional Budget Allocation.
+        Ensures deterministic sells and synchronized buys under impact limits.
         """
-        # TODO:
-        # If the current stock list is more than topk(eg. The weights are modified
-        # by risk control), the weight will not be handled correctly.
-        buy_signal_stocks = set(score.sort_values(ascending=False).iloc[: self.topk].index)
-        cur_stock_weight = current.get_stock_weight_dict(only_stock=True)
 
-        if len(cur_stock_weight) == 0:
-            final_stock_weight = {code: 1 / self.topk for code in buy_signal_stocks}
-        else:
-            final_stock_weight = copy.deepcopy(cur_stock_weight)
-            sold_stock_weight = 0.0
-            for stock_id in final_stock_weight:
-                if stock_id not in buy_signal_stocks:
-                    sw = min(self.max_sold_weight, final_stock_weight[stock_id])
-                    sold_stock_weight += sw
-                    final_stock_weight[stock_id] -= sw
-            if self.buy_method == "first_fill":
-                for stock_id in buy_signal_stocks:
-                    add_weight = min(
-                        max(1 / self.topk - final_stock_weight.get(stock_id, 0), 0.0),
-                        sold_stock_weight,
-                    )
-                    final_stock_weight[stock_id] = final_stock_weight.get(stock_id, 0.0) + add_weight
-                    sold_stock_weight -= add_weight
-            elif self.buy_method == "average_fill":
-                for stock_id in buy_signal_stocks:
-                    final_stock_weight[stock_id] = final_stock_weight.get(stock_id, 0.0) + sold_stock_weight / len(
-                        buy_signal_stocks
-                    )
-            else:
-                raise ValueError("Buy method not found")
-        return final_stock_weight
+        if self.topk is None or self.topk <= 0:
+            return {}
+
+        def apply_impact_limit(weight):
+            return weight if self.trade_impact_limit is None else min(weight, self.trade_impact_limit)
+
+        ideal_per_stock = self.risk_degree / self.topk
+        ideal_list = score.sort_values(ascending=False).iloc[: self.topk].index.tolist()
+
+        cur_weights = current.get_stock_weight_dict(only_stock=True)
+        initial_total_weight = sum(cur_weights.values())
+
+        # --- Case A: Cold Start ---
+        if not cur_weights:
+            fill = apply_impact_limit(ideal_per_stock)
+            return {code: fill for code in ideal_list}
+
+        # --- Case B: Rebalancing ---
+        all_tickers = set(cur_weights.keys()) | set(ideal_list)
+        next_weights = {t: cur_weights.get(t, 0.0) for t in all_tickers}
+
+        # Phase 1: Deterministic Sell Phase
+        released_cash = 0.0
+        for t in list(next_weights.keys()):
+            cur = next_weights[t]
+            if cur <= 1e-8:
+                continue
+
+            if t not in ideal_list:
+                sell = apply_impact_limit(cur)
+                next_weights[t] -= sell
+                released_cash += sell
+            elif cur > ideal_per_stock + 1e-8:
+                excess = cur - ideal_per_stock
+                sell = apply_impact_limit(excess)
+                next_weights[t] -= sell
+                released_cash += sell
+
+        # Phase 2: Budget Calculation
+        # Budget = Cash from sells + Available space from target risk degree
+        total_budget = released_cash + (self.risk_degree - initial_total_weight)
+
+        # Phase 3: Proportional Buy Allocation
+        if total_budget > 1e-8:
+            shortfalls = {
+                t: (ideal_per_stock - next_weights.get(t, 0.0))
+                for t in ideal_list
+                if next_weights.get(t, 0.0) < ideal_per_stock - 1e-8
+            }
+
+            if shortfalls:
+                total_shortfall = sum(shortfalls.values())
+                # Normalize total_budget to not exceed total_shortfall
+                available_to_spend = min(total_budget, total_shortfall)
+
+                for t, shortfall in shortfalls.items():
+                    # Every stock gets its fair share based on its distance to target
+                    share_of_budget = (shortfall / total_shortfall) * available_to_spend
+
+                    # Capped by impact limit
+                    max_buy_cap = apply_impact_limit(shortfall)
+
+                    next_weights[t] += min(share_of_budget, max_buy_cap)
+
+        return {k: v for k, v in next_weights.items() if v > 1e-8}

qlib/contrib/strategy/optimizer/__init__.py

@@ -5,5 +5,4 @@ from .base import BaseOptimizer
 from .optimizer import PortfolioOptimizer
 from .enhanced_indexing import EnhancedIndexingOptimizer
 
-
 __all__ = ["BaseOptimizer", "PortfolioOptimizer", "EnhancedIndexingOptimizer"]

qlib/contrib/strategy/optimizer/enhanced_indexing.py

@@ -9,7 +9,6 @@ from typing import Union, Optional, Dict, Any, List
 from qlib.log import get_module_logger
 from .base import BaseOptimizer
 
-
 logger = get_module_logger("EnhancedIndexingOptimizer")
 
 

qlib/contrib/strategy/order_generator.py

@@ -4,6 +4,7 @@
 """
 This order generator is for strategies based on WeightStrategyBase
 """
+
 from ...backtest.position import Position
 from ...backtest.exchange import Exchange
 

qlib/contrib/torch.py

@@ -5,6 +5,7 @@ This module is not a necessary part of Qlib.
 They are just some tools for convenience
 It is should not imported into the core part of qlib
 """
+
 import torch
 import numpy as np
 import pandas as pd

qlib/contrib/tuner/launcher.py

@@ -13,7 +13,6 @@ import yaml
 
 from .config import TunerConfigManager
 
-
 args_parser = argparse.ArgumentParser(prog="tuner")
 args_parser.add_argument(
     "-c",

qlib/contrib/tuner/space.py

@@ -6,7 +6,6 @@
 
 from hyperopt import hp
 
-
 TopkAmountStrategySpace = {
     "topk": hp.choice("topk", [30, 35, 40]),
     "buffer_margin": hp.choice("buffer_margin", [200, 250, 300]),

qlib/contrib/workflow/__init__.py

@@ -3,5 +3,4 @@
 from .record_temp import MultiSegRecord
 from .record_temp import SignalMseRecord
 
-
 __all__ = ["MultiSegRecord", "SignalMseRecord"]

qlib/data/__init__.py

@@ -36,7 +36,6 @@ from .cache import (
     MemoryCalendarCache,
 )
 
-
 __all__ = [
     "D",
     "CalendarProvider",

qlib/data/dataset/handler.py

@@ -19,7 +19,6 @@ from .loader import DataLoader
 from . import processor as processor_module
 from . import loader as data_loader_module
 
-
 DATA_KEY_TYPE = Literal["raw", "infer", "learn"]
 
 

qlib/data/pit.py

@@ -13,6 +13,7 @@ The calculation of both <period_time, feature> and <observe_time, feature> data
 2) concatenate all th collasped data, we will get data with format <observe_time, feature>.
 Qlib will use the operator `P` to perform the collapse.
 """
+
 import numpy as np
 import pandas as pd
 from qlib.data.ops import ElemOperator

qlib/data/storage/__init__.py

@@ -3,5 +3,4 @@
 
 from .storage import CalendarStorage, InstrumentStorage, FeatureStorage, CalVT, InstVT, InstKT
 
-
 __all__ = ["CalendarStorage", "InstrumentStorage", "FeatureStorage", "CalVT", "InstVT", "InstKT"]

qlib/data/storage/file_storage.py

@@ -156,7 +156,7 @@ class FileCalendarStorage(FileStorageMixin, CalendarStorage):
     def index(self, value: CalVT) -> int:
         self.check()
         calendar = self._read_calendar()
-        return int(np.argwhere(calendar == value)[0])
+        return calendar.index(value)
 
     def insert(self, index: int, value: CalVT):
         calendar = self._read_calendar()

qlib/model/__init__.py

@@ -5,5 +5,4 @@ import warnings
 
 from .base import Model
 
-
 __all__ = ["Model", "warnings"]

qlib/model/meta/__init__.py

@@ -4,5 +4,4 @@
 from .task import MetaTask
 from .dataset import MetaTaskDataset
 
-
 __all__ = ["MetaTask", "MetaTaskDataset"]

qlib/model/riskmodel/__init__.py

@@ -6,7 +6,6 @@ from .poet import POETCovEstimator
 from .shrink import ShrinkCovEstimator
 from .structured import StructuredCovEstimator
 
-
 __all__ = [
     "RiskModel",
     "POETCovEstimator",

qlib/rl/contrib/naive_config_parser.py

@@ -9,7 +9,6 @@ import tempfile
 from importlib import import_module
 from ruamel.yaml import YAML
 
-
 DELETE_KEY = "_delete_"
 
 

qlib/utils/data.py

@@ -3,6 +3,7 @@
 """
 This module covers some utility functions that operate on data or basic object
 """
+
 from copy import deepcopy
 from typing import List, Union
 

qlib/utils/time.py

@@ -3,6 +3,7 @@
 """
 Time related utils are compiled in this script
 """
+
 import bisect
 from datetime import datetime, time, date, timedelta
 from typing import List, Optional, Tuple, Union
@@ -14,7 +15,6 @@ import pandas as pd
 from qlib.config import C
 from qlib.constant import REG_CN, REG_TW, REG_US
 
-
 CN_TIME = [
     datetime.strptime("9:30", "%H:%M"),
     datetime.strptime("11:30", "%H:%M"),

qlib/workflow/expm.py

@@ -16,7 +16,6 @@ from .recorder import Recorder
 from ..log import get_module_logger
 from ..utils.exceptions import ExpAlreadyExistError
 
-
 logger = get_module_logger("workflow")
 
 

qlib/workflow/record_temp.py

@@ -22,7 +22,6 @@ from ..utils.data import deepcopy_basic_type
 from ..utils.exceptions import QlibException
 from ..contrib.eva.alpha import calc_ic, calc_long_short_return, calc_long_short_prec
 
-
 logger = get_module_logger("workflow", logging.INFO)
 
 
@@ -476,7 +475,13 @@ class PortAnaRecord(ACRecordTemp):
         if self.backtest_config["start_time"] is None:
             self.backtest_config["start_time"] = dt_values.min()
         if self.backtest_config["end_time"] is None:
-            self.backtest_config["end_time"] = get_date_by_shift(dt_values.max(), 1)
+            self.backtest_config["end_time"] = get_date_by_shift(dt_values.max(), -1)
+            warnings.warn(
+                "No explicit backtest end_time provided. "
+                "Qlib requires one extra calendar step to determine the right boundary of a bar. "
+                "Therefore the end_time is shifted backward by one trading day from "
+                f"{dt_values.max()} -> {self.backtest_config['end_time']}."
+            )
 
         artifact_objects = {}
         # custom strategy and get backtest

qlib/workflow/task/gen.py

@@ -3,6 +3,7 @@
 """
 TaskGenerator module can generate many tasks based on TaskGen and some task templates.
 """
+
 import abc
 import copy
 import pandas as pd

qlib/workflow/task/manage.py

@@ -12,6 +12,7 @@ A task in TaskManager consists of 3 parts
 - tasks status: the status of the task
 - tasks result: A user can get the task with the task description and task result.
 """
+
 import concurrent
 import pickle
 import time

scripts/data_collector/base.py

@@ -280,11 +280,20 @@ class Normalize:
         self._symbol_field_name = symbol_field_name
         self._end_date = kwargs.get("end_date", None)
         self._max_workers = max_workers
+        self.interval = kwargs.get("interval", "1d")
 
         self._normalize_obj = normalize_class(
             date_field_name=date_field_name, symbol_field_name=symbol_field_name, **kwargs
         )
 
+    def format_data(self, df: pd.DataFrame):
+        if self.interval == "1d":
+            try:
+                pd.to_datetime(df.iloc[-1]["date"], format="%Y-%m-%d", errors="raise")
+            except Exception:
+                df = df.iloc[:-1]
+        return df
+
     def _executor(self, file_path: Path):
         file_path = Path(file_path)
 
@@ -300,14 +309,18 @@ class Normalize:
             keep_default_na=False,
             na_values={col: symbol_na if col == self._symbol_field_name else default_na for col in columns},
         )
+        df = self.format_data(df=df)
 
-        # NOTE: It has been reported that there may be some problems here, and the specific issues will be dealt with when they are identified.
-        df = self._normalize_obj.normalize(df)
-        if df is not None and not df.empty:
-            if self._end_date is not None:
-                _mask = pd.to_datetime(df[self._date_field_name]) <= pd.Timestamp(self._end_date)
-                df = df[_mask]
-            df.to_csv(self._target_dir.joinpath(file_path.name), index=False)
+        if not df.empty:
+            # NOTE: It has been reported that there may be some problems here, and the specific issues will be dealt with when they are identified.
+            df = self._normalize_obj.normalize(df)
+            if df is not None and not df.empty:
+                if self._end_date is not None:
+                    _mask = pd.to_datetime(df[self._date_field_name]) <= pd.Timestamp(self._end_date)
+                    df = df[_mask]
+                df.to_csv(self._target_dir.joinpath(file_path.name), index=False)
+        else:
+            logger.warning(f"{file_path.stem} source data is empty and will not undergo normalization processing.")
 
     def normalize(self):
         logger.info("normalize data......")

scripts/data_collector/cn_index/collector.py

@@ -22,7 +22,6 @@ from data_collector.index import IndexBase
 from data_collector.utils import get_calendar_list, get_trading_date_by_shift, deco_retry
 from data_collector.utils import get_instruments
 
-
 NEW_COMPANIES_URL = (
     "https://oss-ch.csindex.com.cn/static/html/csindex/public/uploads/file/autofile/cons/{index_code}cons.xls"
 )

scripts/data_collector/crypto/collector.py

@@ -19,7 +19,6 @@ from time import mktime
 from datetime import datetime as dt
 import time
 
-
 _CG_CRYPTO_SYMBOLS = None
 
 

scripts/data_collector/us_index/collector.py

@@ -16,7 +16,6 @@ from tqdm import tqdm
 from loguru import logger
 from fake_useragent import UserAgent
 
-
 CUR_DIR = Path(__file__).resolve().parent
 sys.path.append(str(CUR_DIR.parent.parent))
 
@@ -24,7 +23,6 @@ from data_collector.index import IndexBase
 from data_collector.utils import deco_retry, get_calendar_list, get_trading_date_by_shift
 from data_collector.utils import get_instruments
 
-
 WIKI_URL = "https://en.wikipedia.org/wiki"
 
 WIKI_INDEX_NAME_MAP = {

scripts/data_collector/utils.py

@@ -3,12 +3,10 @@
 
 import re
 import copy
-import datetime
 import importlib
 import time
 import bisect
 import pickle
-import random
 import requests
 import functools
 from pathlib import Path
@@ -22,10 +20,13 @@ from tqdm import tqdm
 from functools import partial
 from concurrent.futures import ProcessPoolExecutor
 from bs4 import BeautifulSoup
+import baostock as bs
+
+from qlib.utils.pickle_utils import restricted_pickle_load
 
 HS_SYMBOLS_URL = "http://app.finance.ifeng.com/hq/list.php?type=stock_a&class={s_type}"
 
-CALENDAR_URL_BASE = "http://push2his.eastmoney.com/api/qt/stock/kline/get?secid={market}.{bench_code}&fields1=f1%2Cf2%2Cf3%2Cf4%2Cf5&fields2=f51%2Cf52%2Cf53%2Cf54%2Cf55%2Cf56%2Cf57%2Cf58&klt=101&fqt=0"
+CALENDAR_URL_BASE = "http://push2his.eastmoney.com/api/qt/stock/kline/get?secid={market}.{bench_code}&fields1=f1%2Cf2%2Cf3%2Cf4%2Cf5&fields2=f51%2Cf52%2Cf53%2Cf54%2Cf55%2Cf56%2Cf57%2Cf58&klt=101&fqt=0&beg=19900101&end=20991231"
 SZSE_CALENDAR_URL = "http://www.szse.cn/api/report/exchange/onepersistenthour/monthList?month={month}&random={random}"
 
 CALENDAR_BENCH_URL_MAP = {
@@ -40,24 +41,6 @@ CALENDAR_BENCH_URL_MAP = {
     "BR_ALL": "^BVSP",
 }
 
-CHROME_UA_POOL = [
-    # Windows
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-    "AppleWebKit/537.36 (KHTML, like Gecko) "
-    "Chrome/120.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-    "AppleWebKit/537.36 (KHTML, like Gecko) "
-    "Chrome/121.0.6167.85 Safari/537.36",
-    # macOS
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
-    "AppleWebKit/537.36 (KHTML, like Gecko) "
-    "Chrome/121.0.0.0 Safari/537.36",
-    # Linux
-    "Mozilla/5.0 (X11; Linux x86_64) "  # pylint: disable=W1404
-    "AppleWebKit/537.36 (KHTML, like Gecko) "
-    "Chrome/120.0.0.0 Safari/537.36",
-]
-
 _BENCH_CALENDAR_LIST = None
 _ALL_CALENDAR_LIST = None
 _HS_SYMBOLS = None
@@ -71,16 +54,6 @@ _CALENDAR_MAP = {}
 MINIMUM_SYMBOLS_NUM = 3900
 
 
-def build_headers():
-    return {
-        "User-Agent": random.choice(CHROME_UA_POOL),
-        "Accept": "application/json,text/plain,*/*",
-        "Accept-Language": "zh-CN,zh;q=0.9",
-        "Referer": "https://quote.eastmoney.com/",
-        "Connection": "keep-alive",
-    }
-
-
 def get_calendar_list(bench_code="CSI300") -> List[pd.Timestamp]:
     """get SH/SZ history calendar list
 
@@ -96,38 +69,16 @@ def get_calendar_list(bench_code="CSI300") -> List[pd.Timestamp]:
 
     logger.info(f"get calendar list: {bench_code}......")
 
-    def _get_calendar(url, max_retry=3):
-        session = requests.Session()
-        session.headers.update(build_headers())
-        current_datetime = datetime.datetime.now()
-        cur_year = current_datetime.year
-        res_list = []
-        failed_years = []
-        for year in range(2000, cur_year + 1):
-            start = f"{year}0101"
-            end = f"{year}1231"
-            formatted_url = url + f"&beg={start}&end={end}".format(start=start, end=end)
-            for attempt in range(max_retry):
-                try:
-                    resp = session.get(formatted_url, timeout=10)
-                    resp.raise_for_status()
-
-                    data = resp.json().get("data")
-                    if not data or "klines" not in data:
-                        raise ValueError("missing klines")
-
-                    res_list.extend(pd.Timestamp(x.split(",")[0]) for x in data["klines"])
-                    break
-
-                except Exception as e:
-                    time.sleep(random.uniform(0.8, 1.5))
-            else:
-                failed_years.append(year)
-
-        if failed_years:
-            logger.warning(f"Calendar incomplete, failed years: {failed_years}")
-
-        return sorted(set(res_list))
+    def _get_calendar(end_date):
+        bs.login()
+        rs = bs.query_trade_dates(start_date="2005-01-01", end_date=end_date)
+        data_list = []
+        while (rs.error_code == "0") & rs.next():
+            data_list.append(rs.get_row_data())
+        bs.logout()
+        df = pd.DataFrame(data_list, columns=rs.fields)
+        trade_days = df[df["is_trading_day"] == "1"]["calendar_date"]
+        return sorted(map(pd.Timestamp, trade_days.to_list()))
 
     calendar = _CALENDAR_MAP.get(bench_code, None)
     if calendar is None:
@@ -147,7 +98,8 @@ def get_calendar_list(bench_code="CSI300") -> List[pd.Timestamp]:
                 filtered_dates = dates[(dates >= "2000-01-04") & (dates <= pd.Timestamp.today().normalize())]
                 calendar = filtered_dates.tolist()
             else:
-                calendar = _get_calendar(CALENDAR_BENCH_URL_MAP[bench_code])
+                end_date = time.strftime("%Y-%m-%d", time.localtime())
+                calendar = _get_calendar(end_date=end_date)
         _CALENDAR_MAP[bench_code] = calendar
     logger.info(f"end of get calendar list: {bench_code}.")
     return calendar
@@ -305,11 +257,7 @@ def get_hs_stock_symbols() -> list:
 
         # Add suffix after the stock code to conform to yahooquery standard, otherwise the data will not be fetched.
         _symbols = [
-            (
-                _symbol + ".ss"
-                if _symbol.startswith("6")
-                else _symbol + ".sz" if _symbol.startswith(("0", "3")) else None
-            )
+            _symbol + ".ss" if _symbol.startswith("6") else _symbol + ".sz" if _symbol.startswith(("0", "3")) else None
             for _symbol in _symbols
         ]
         _symbols = [_symbol for _symbol in _symbols if _symbol is not None]
@@ -328,7 +276,7 @@ def get_hs_stock_symbols() -> list:
         symbol_cache_path.parent.mkdir(parents=True, exist_ok=True)
         if symbol_cache_path.exists():
             with symbol_cache_path.open("rb") as fp:
-                cache_symbols = pickle.load(fp)
+                cache_symbols = restricted_pickle_load(fp)
                 symbols |= cache_symbols
         with symbol_cache_path.open("wb") as fp:
             pickle.dump(symbols, fp)
@@ -345,20 +293,14 @@ def get_us_stock_symbols(qlib_data_path: [str, Path] = None) -> list:
     -------
         stock symbols
     """
+    import akshare as ak  # pylint: disable=C0415
+
     global _US_SYMBOLS  # pylint: disable=W0603
 
     @deco_retry
     def _get_eastmoney():
-        url = "http://4.push2.eastmoney.com/api/qt/clist/get?pn=1&pz=10000&fs=m:105,m:106,m:107&fields=f12"
-        resp = requests.get(url, timeout=None)
-        if resp.status_code != 200:
-            raise ValueError("request error")
-
-        try:
-            _symbols = [_v["f12"].replace("_", "-P") for _v in resp.json()["data"]["diff"].values()]
-        except Exception as e:
-            logger.warning(f"request error: {e}")
-            raise
+        df = ak.get_us_stock_name()
+        _symbols = df["symbol"].to_list()
 
         if len(_symbols) < 8000:
             raise ValueError("request error")
@@ -420,14 +362,7 @@ def get_us_stock_symbols(qlib_data_path: [str, Path] = None) -> list:
             s_ = s_.strip("*")
             return s_
 
-        _US_SYMBOLS = sorted(
-            set(
-                map(
-                    _format,
-                    filter(lambda x: len(x) < 8 and not x.endswith("WS"), _all_symbols),
-                )
-            )
-        )
+        _US_SYMBOLS = sorted(set(map(_format, filter(lambda x: len(x) < 8 and not x.endswith("WS"), _all_symbols))))
 
     return _US_SYMBOLS
 
@@ -541,10 +476,7 @@ def get_en_fund_symbols(qlib_data_path: [str, Path] = None) -> list:
             raise ValueError("request error")
         try:
             _symbols = []
-            for sub_data in re.findall(
-                r"[\[](.*?)[\]]",
-                resp.content.decode().split("= [")[-1].replace("];", ""),
-            ):
+            for sub_data in re.findall(r"[\[](.*?)[\]]", resp.content.decode().split("= [")[-1].replace("];", "")):
                 data = sub_data.replace('"', "").replace("'", "")
                 # TODO: do we need other information, like fund_name from ['000001', 'HXCZHH', '华夏成长混合', '混合型', 'HUAXIACHENGZHANGHUNHE']
                 _symbols.append(data.split(",")[0])
@@ -725,11 +657,7 @@ def get_instruments(
     """
     _cur_module = importlib.import_module("data_collector.{}.collector".format(market_index))
     obj = getattr(_cur_module, f"{index_name.upper()}Index")(
-        qlib_dir=qlib_dir,
-        index_name=index_name,
-        freq=freq,
-        request_retry=request_retry,
-        retry_sleep=retry_sleep,
+        qlib_dir=qlib_dir, index_name=index_name, freq=freq, request_retry=request_retry, retry_sleep=retry_sleep
     )
     getattr(obj, method)()
 
@@ -737,10 +665,7 @@ def get_instruments(
 def _get_all_1d_data(_date_field_name: str, _symbol_field_name: str, _1d_data_all: pd.DataFrame):
     df = copy.deepcopy(_1d_data_all)
     df.reset_index(inplace=True)
-    df.rename(
-        columns={"datetime": _date_field_name, "instrument": _symbol_field_name},
-        inplace=True,
-    )
+    df.rename(columns={"datetime": _date_field_name, "instrument": _symbol_field_name}, inplace=True)
     df.columns = list(map(lambda x: x[1:] if x.startswith("$") else x, df.columns))
     return df
 

scripts/get_data.py

@@ -4,6 +4,5 @@
 import fire
 from qlib.tests.data import GetData
 
-
 if __name__ == "__main__":
     fire.Fire(GetData)

setup.py

@@ -3,7 +3,6 @@ import os
 import numpy
 from setuptools import Extension, setup
 
-
 NUMPY_INCLUDE = numpy.get_include()
 
 

tests/backtest/test_soft_topk_strategy.py

@@ -0,0 +1,56 @@
+import pandas as pd
+import pytest
+from qlib.contrib.strategy.cost_control import SoftTopkStrategy
+
+
+class MockPosition:
+    def __init__(self, weights):
+        self.weights = weights
+
+    def get_stock_weight_dict(self, only_stock=True):
+        return self.weights
+
+
+def test_soft_topk_logic():
+    # Initial: A=0.8, B=0.2 (Total=1.0). Target Risk=0.95.
+    # Scores: A and B are low, C and D are topk.
+    scores = pd.Series({"C": 0.9, "D": 0.8, "A": 0.1, "B": 0.1})
+    current_pos = MockPosition({"A": 0.8, "B": 0.2})
+
+    topk = 2
+    risk_degree = 0.95
+    impact_limit = 0.1  # Max change per step
+
+    def create_test_strategy(impact_limit_value):
+        strat = SoftTopkStrategy.__new__(SoftTopkStrategy)
+        strat.topk = topk
+        strat.risk_degree = risk_degree
+        strat.trade_impact_limit = impact_limit_value
+        return strat
+
+    # 1. With impact limit: Expect deterministic sell and limited buy
+    strat_i = create_test_strategy(impact_limit)
+    res_i = strat_i.generate_target_weight_position(scores, current_pos, None, None)
+
+    # A should be exactly 0.8 - 0.1 = 0.7
+    assert abs(res_i["A"] - 0.7) < 1e-8
+    # B should be exactly 0.2 - 0.1 = 0.1
+    assert abs(res_i["B"] - 0.1) < 1e-8
+    # Total sells = 0.2 released. New budget = 0.2 + (0.95 - 1.0) = 0.15.
+    # C and D share 0.15 -> 0.075 each.
+    assert abs(res_i["C"] - 0.075) < 1e-8
+    assert abs(res_i["D"] - 0.075) < 1e-8
+
+    # 2. Without impact limit: Expect full liquidation and full target fill
+    strat_c = create_test_strategy(1.0)
+    res_c = strat_c.generate_target_weight_position(scores, current_pos, None, None)
+
+    # A, B not in topk -> Liquidated
+    assert "A" not in res_c and "B" not in res_c
+    # C, D should reach ideal_per_stock (0.95/2 = 0.475)
+    assert abs(res_c["C"] - 0.475) < 1e-8
+    assert abs(res_c["D"] - 0.475) < 1e-8
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])

tests/backtest/test_soft_topk_strategy_cold_start.py

@@ -0,0 +1,38 @@
+import pandas as pd
+import pytest
+
+from qlib.contrib.strategy.cost_control import SoftTopkStrategy
+
+
+class MockPosition:
+    def __init__(self, weights):
+        self.weights = weights
+
+    def get_stock_weight_dict(self, only_stock=True):
+        return self.weights
+
+
+def create_test_strategy(topk, risk_degree, impact_limit):
+    strat = SoftTopkStrategy.__new__(SoftTopkStrategy)
+    strat.topk = topk
+    strat.risk_degree = risk_degree
+    strat.trade_impact_limit = impact_limit
+    return strat
+
+
+@pytest.mark.parametrize(
+    ("impact_limit", "expected_fill"),
+    [
+        (0.1, 0.1),
+        (1.0, 0.475),
+    ],
+)
+def test_soft_topk_cold_start_impact_limit(impact_limit, expected_fill):
+    scores = pd.Series({"C": 0.9, "D": 0.8, "A": 0.1, "B": 0.1})
+    current_pos = MockPosition({})
+
+    strat = create_test_strategy(topk=2, risk_degree=0.95, impact_limit=impact_limit)
+    res = strat.generate_target_weight_position(scores, current_pos, None, None)
+
+    assert abs(res["C"] - expected_fill) < 1e-8
+    assert abs(res["D"] - expected_fill) < 1e-8

tests/rl/test_finite_env.py

@@ -17,7 +17,6 @@ from qlib.rl.utils.finite_env import (
     generate_nan_observation,
 )
 
-
 _test_space = gym.spaces.Dict(
     {
         "sensors": gym.spaces.Dict(

tests/test_contrib_workflow.py

@@ -13,7 +13,6 @@ from qlib.workflow import R
 from qlib.tests import TestAutoData
 from qlib.tests.config import GBDT_MODEL, get_dataset_config, CSI300_MARKET
 
-
 CSI300_GBDT_TASK = {
     "model": GBDT_MODEL,
     "dataset": get_dataset_config(

tests/test_dump_data.py

@@ -16,7 +16,6 @@ sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts")))
 from get_data import GetData
 from dump_bin import DumpDataAll, DumpDataFix
 
-
 DATA_DIR = Path(__file__).parent.joinpath("test_dump_data")
 SOURCE_DIR = DATA_DIR.joinpath("source")
 SOURCE_DIR.mkdir(exist_ok=True, parents=True)

tests/test_pit.py

@@ -19,7 +19,6 @@ from dump_pit import DumpPitData
 sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts/data_collector/pit")))
 from collector import Run
 
-
 pd.set_option("display.width", 1000)
 pd.set_option("display.max_columns", None)