liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-17 17:34:35 +08:00
Code Activity

refactor(data_collector): use akshare to build unified trade calendar (#2093)

Browse Source 
* refactor(data_collector): use akshare to build unified trade calendar

* fix: github action failure caused by black upgrade
This commit is contained in:
Linlang
2026-01-20 22:52:57 +08:00
committed by GitHub
parent 80982f8904
commit 50c32ac15f
3 changed files with 12 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pyproject.toml
View File

@@ -69,8 +69,10 @@ rl = [
"torch", "torch",
"numpy<2.0.0", "numpy<2.0.0",
] ]
# We exclude black version 26.1.0 due to known issues with nbqa when formatting Jupyter notebooks,
# which can cause false-positive --check results and inconsistent notebook formatting.
lint = [ lint = [
"black", "black!=26.1.0",
"pylint", "pylint",
"mypy<1.5.0", "mypy<1.5.0",
"flake8", "flake8",

29
scripts/data_collector/utils.py
View File

@@ -7,7 +7,6 @@ import importlib
import time import time
import bisect import bisect
import pickle import pickle
import random
import requests import requests
import functools import functools
from pathlib import Path from pathlib import Path
@@ -80,28 +79,14 @@ def get_calendar_list(bench_code="CSI300") -> List[pd.Timestamp]:
calendar = df.index.get_level_values(level="date").map(pd.Timestamp).unique().tolist() calendar = df.index.get_level_values(level="date").map(pd.Timestamp).unique().tolist()
else: else:
if bench_code.upper() == "ALL": if bench_code.upper() == "ALL":
import akshare as ak # pylint: disable=C0415
@deco_retry trade_date_df = ak.tool_trade_date_hist_sina()
def _get_calendar_from_month(month): trade_date_list = trade_date_df["trade_date"].tolist()
_cal = [] trade_date_list = [pd.Timestamp(d) for d in trade_date_list]
try: dates = pd.DatetimeIndex(trade_date_list)
resp = requests.get( filtered_dates = dates[(dates >= "2000-01-04") & (dates <= pd.Timestamp.today().normalize())]
SZSE_CALENDAR_URL.format(month=month, random=random.random), timeout=None calendar = filtered_dates.tolist()
).json()
for _r in resp["data"]:
if int(_r["jybz"]):
_cal.append(pd.Timestamp(_r["jyrq"]))
except Exception as e:
raise ValueError(f"{month}-->{e}") from e
return _cal
month_range = pd.date_range(start="2000-01", end=pd.Timestamp.now() + pd.Timedelta(days=31), freq="M")
calendar = []
for _m in month_range:
cal = _get_calendar_from_month(_m.strftime("%Y-%m"))
if cal:
calendar += cal
calendar = list(filter(lambda x: x <= pd.Timestamp.now(), calendar))
else: else:
calendar = _get_calendar(CALENDAR_BENCH_URL_MAP[bench_code]) calendar = _get_calendar(CALENDAR_BENCH_URL_MAP[bench_code])
_CALENDAR_MAP[bench_code] = calendar _CALENDAR_MAP[bench_code] = calendar

3
scripts/data_collector/yahoo/requirements.txt
View File

@@ -9,4 +9,5 @@ yahooquery
joblib joblib
beautifulsoup4 beautifulsoup4
bs4 bs4
soupsieve soupsieve
akshare