fix ci

.github/workflows/python-publish.yml

@@ -58,6 +58,8 @@ jobs:
 
   deploy_with_manylinux:
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write
     steps:
     - uses: actions/checkout@v2
     - name: Build wheel on Linux

.github/workflows/release-drafter.yml

@@ -6,14 +6,8 @@ on:
     branches:
       - main
 
-permissions:
-  contents: read
-
 jobs:
   update_release_draft:
-    permissions:
-      contents: write
-      pull-requests: read
     runs-on: ubuntu-latest
     steps:
       # Drafts your next Release notes as Pull Requests are merged into "master"

.github/workflows/test_qlib_from_pip.yml

@@ -13,9 +13,6 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        # Since macos-latest changed from 12.7.4 to 14.4.1,
-        # the minimum python version that matches a 14.4.1 version of macos is 3.10,
-        # so we limit the macos version to macos-12.
         os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-11, macos-12]
         # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
         python-version: [3.7, 3.8]

.github/workflows/test_qlib_from_source.yml

@@ -14,9 +14,6 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        # Since macos-latest changed from 12.7.4 to 14.4.1,
-        # the minimum python version that matches a 14.4.1 version of macos is 3.10,
-        # so we limit the macos version to macos-12.
         os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-11, macos-12]
         # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
         python-version: [3.7, 3.8]

.github/workflows/test_qlib_from_source_slow.yml

@@ -14,9 +14,6 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        # Since macos-latest changed from 12.7.4 to 14.4.1,
-        # the minimum python version that matches a 14.4.1 version of macos is 3.10,
-        # so we limit the macos version to macos-12.
         os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-11, macos-12]
         # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
         python-version: [3.7, 3.8]

.gitignore

@@ -48,4 +48,4 @@ tags
 *.swp
 
 ./pretrain
-.idea/
+.idea/

.readthedocs.yml

@@ -5,12 +5,6 @@
 # Required
 version: 2
 
-# Set the version of Python and other tools you might need
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.7"
-
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/conf.py
@@ -20,6 +14,7 @@ formats: all
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
+  version: 3.7
   install:
     - requirements: docs/requirements.txt
     - method: pip

docs/requirements.txt

@@ -5,4 +5,3 @@ scipy
 scikit-learn
 pandas
 tianshou
-sphinx_rtd_theme

qlib/__init__.py

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 from pathlib import Path
 
-__version__ = "0.9.4.99"
+__version__ = "0.9.4"
 __version__bak = __version__  # This version is backup for QlibConfig.reset_qlib_version
 import os
 from typing import Union

qlib/data/data.py

@@ -536,6 +536,7 @@ class DatasetProvider(abc.ABC):
         """
         if len(fields) == 0:
             raise ValueError("fields cannot be empty")
+        fields = fields.copy()
         column_names = [str(f) for f in fields]
         return column_names
 

qlib/utils/__init__.py

@@ -25,12 +25,7 @@ import pandas as pd
 from pathlib import Path
 from typing import List, Union, Optional, Callable
 from packaging import version
-from .file import (
+from .file import get_or_create_path, save_multiple_parts_file, unpack_archive_with_buffer, get_tmp_file_with_buffer
-    get_or_create_path,
-    save_multiple_parts_file,
-    unpack_archive_with_buffer,
-    get_tmp_file_with_buffer,
-)
 from ..config import C
 from ..log import get_module_logger, set_log_with_config
 
@@ -42,12 +37,7 @@ is_deprecated_lexsorted_pandas = version.parse(pd.__version__) > version.parse("
 #################### Server ####################
 def get_redis_connection():
     """get redis connection instance."""
-    return redis.StrictRedis(
+    return redis.StrictRedis(host=C.redis_host, port=C.redis_port, db=C.redis_task_db, password=C.redis_password)
-        host=C.redis_host,
-        port=C.redis_port,
-        db=C.redis_task_db,
-        password=C.redis_password,
-    )
 
 
 #################### Data ####################
@@ -106,14 +96,7 @@ def get_period_offset(first_year, period, quarterly):
     return offset
 
 
-def read_period_data(
+def read_period_data(index_path, data_path, period, cur_date_int: int, quarterly, last_period_index: int = None):
-    index_path,
-    data_path,
-    period,
-    cur_date_int: int,
-    quarterly,
-    last_period_index: int = None,
-):
     """
     At `cur_date`(e.g. 20190102), read the information at `period`(e.g. 201803).
     Only the updating info before cur_date or at cur_date will be used.
@@ -290,10 +273,7 @@ def parse_field(field):
     # \uff09 -> )
     chinese_punctuation_regex = r"\u3001\uff1a\uff08\uff09"
     for pattern, new in [
-        (
+        (rf"\$\$([\w{chinese_punctuation_regex}]+)", r'PFeature("\1")'),  # $$ must be before $
-            rf"\$\$([\w{chinese_punctuation_regex}]+)",
-            r'PFeature("\1")',
-        ),  # $$ must be before $
         (rf"\$([\w{chinese_punctuation_regex}]+)", r'Feature("\1")'),
         (r"(\w+\s*)\(", r"Operators.\1("),
     ]:  # Features  # Operators
@@ -403,14 +383,7 @@ def get_date_range(trading_date, left_shift=0, right_shift=0, future=False):
     return calendar
 
 
-def get_date_by_shift(
+def get_date_by_shift(trading_date, shift, future=False, clip_shift=True, freq="day", align: Optional[str] = None):
-    trading_date,
-    shift,
-    future=False,
-    clip_shift=True,
-    freq="day",
-    align: Optional[str] = None,
-):
     """get trading date with shift bias will cur_date
         e.g. : shift == 1,  return next trading date
                shift == -1, return previous trading date
@@ -596,38 +569,7 @@ def exists_qlib_data(qlib_dir):
     # check instruments
     code_names = set(map(lambda x: fname_to_code(x.name.lower()), features_dir.iterdir()))
     _instrument = instruments_dir.joinpath("all.txt")
-    # Removed two possible ticker names "NA" and "NULL" from the default na_values list for column 0
+    miss_code = set(pd.read_csv(_instrument, sep="\t", header=None).loc[:, 0].apply(str.lower)) - set(code_names)
-    miss_code = set(
-        pd.read_csv(
-            _instrument,
-            sep="\t",
-            header=None,
-            keep_default_na=False,
-            na_values={
-                0: [
-                    " ",
-                    "#N/A",
-                    "#N/A N/A",
-                    "#NA",
-                    "-1.#IND",
-                    "-1.#QNAN",
-                    "-NaN",
-                    "-nan",
-                    "1.#IND",
-                    "1.#QNAN",
-                    "<NA>",
-                    "N/A",
-                    "NaN",
-                    "None",
-                    "n/a",
-                    "nan",
-                    "null ",
-                ]
-            },
-        )
-        .loc[:, 0]
-        .apply(str.lower)
-    ) - set(code_names)
     if miss_code and any(map(lambda x: "sht" not in x, miss_code)):
         return False
 

scripts/data_collector/cn_index/collector.py

@@ -396,7 +396,14 @@ class CSI500Index(CSIIndex):
         today = pd.Timestamp.now()
         date_range = pd.DataFrame(pd.date_range(start="2007-01-15", end=today, freq="7D"))[0].dt.date
         ret_list = []
+        col = ["date", "symbol", "code_name"]
         for date in tqdm(date_range, desc="Download CSI500"):
+            rs = bs.query_zz500_stocks(date=str(date))
+            zz500_stocks = []
+            while (rs.error_code == "0") & rs.next():
+                zz500_stocks.append(rs.get_row_data())
+            result = pd.DataFrame(zz500_stocks, columns=col)
+            result["symbol"] = result["symbol"].apply(lambda x: x.replace(".", "").upper())
             result = self.get_data_from_baostock(date)
             ret_list.append(result[["date", "symbol"]])
         bs.logout()

scripts/data_collector/cn_index/requirements.txt

@@ -5,5 +5,3 @@ pandas
 lxml
 loguru
 tqdm
-yahooquery
-openpyxl

scripts/dump_pit.py

@@ -3,7 +3,7 @@
 """
 TODO:
 - A more well-designed PIT database is required.
-    - separated insert, delete, update, query operations are required.
+    - seperated insert, delete, update, query operations are required.
 """
 
 import shutil