liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-18 01:44:34 +08:00
Code Activity

update-csi500

Browse Source
This commit is contained in:
Linlang Lv (iSoftStone)
2022-02-28 03:48:07 +08:00
parent 74cc21fc2c
commit 40dd84857c
2 changed files with 96 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scripts/data_collector/cn_index/collector.py
View File

@@ -5,7 +5,6 @@ import re
import abc import abc
import sys import sys
import importlib import importlib
from tqdm import tqdm
from io import BytesIO from io import BytesIO
from typing import List, Iterable from typing import List, Iterable
from pathlib import Path from pathlib import Path
@@ -14,7 +13,7 @@ import fire
import requests import requests
import pandas as pd import pandas as pd
import baostock as bs import baostock as bs
from lxml import etree from tqdm import tqdm
from loguru import logger from loguru import logger
CUR_DIR = Path(__file__).resolve().parent CUR_DIR = Path(__file__).resolve().parent
@@ -47,7 +46,6 @@ def retry_request(url: str, method: str = "get", exclude_status: List = None):
class CSIIndex(IndexBase): class CSIIndex(IndexBase):
@property @property
def calendar_list(self) -> List[pd.Timestamp]: def calendar_list(self) -> List[pd.Timestamp]:
"""get history trading date """get history trading date
@@ -318,7 +316,6 @@ class CSIIndex(IndexBase):
class CSI300(CSIIndex): class CSI300(CSIIndex):
@property @property
def index_code(self): def index_code(self):
return "000300" return "000300"
@@ -329,11 +326,10 @@ class CSI300(CSIIndex):
@property @property
def html_table_index(self): def html_table_index(self):
return 0 return 1
class CSI100(CSIIndex): class CSI100(CSIIndex):
@property @property
def index_code(self): def index_code(self):
return "000903" return "000903"
@@ -344,11 +340,10 @@ class CSI100(CSIIndex):
@property @property
def html_table_index(self): def html_table_index(self):
return 1 return 2
class CSI500(CSIIndex): class CSI500(CSIIndex):
@property @property
def index_code(self): def index_code(self):
return "000905" return "000905"
@@ -366,7 +361,7 @@ class CSI500(CSIIndex):
def get_history_companies(self): def get_history_companies(self):
""" """
Data source：http://baostock.com/baostock/index.php/%E4%B8%AD%E8%AF%81500%E6%88%90%E5%88%86%E8%82%A1 Data source: http://baostock.com/baostock/index.php/%E4%B8%AD%E8%AF%81500%E6%88%90%E5%88%86%E8%82%A1
Avoid a large number of parallel data acquisition, Avoid a large number of parallel data acquisition,
such as 1000 times of concurrent data acquisition, because IP will be blocked such as 1000 times of concurrent data acquisition, because IP will be blocked
Returns Returns
@@ -391,7 +386,12 @@ class CSI500(CSIIndex):
def get_instruments( def get_instruments(
qlib_dir: str, index_name: str, method: str = "parse_instruments", request_retry: int = 5, retry_sleep: int = 3 qlib_dir: str,
index_name: str,
method: str = "parse_instruments",
freq: str = "day",
request_retry: int = 5,
retry_sleep: int = 3,
): ):
""" """

3
scripts/data_collector/utils.py
View File

@@ -34,6 +34,7 @@ CALENDAR_BENCH_URL_MAP = {
"ALL": CALENDAR_URL_BASE.format(market=1, bench_code="000905"), "ALL": CALENDAR_URL_BASE.format(market=1, bench_code="000905"),
# NOTE: Use the time series of ^GSPC(SP500) as the sequence of all stocks # NOTE: Use the time series of ^GSPC(SP500) as the sequence of all stocks
"US_ALL": "^GSPC", "US_ALL": "^GSPC",
"IN_ALL": "^NSEI",
} }
_BENCH_CALENDAR_LIST = None _BENCH_CALENDAR_LIST = None
@@ -452,9 +453,7 @@ def symbol_prefix_to_sufix(symbol: str, capital: bool = True) -> str:
def deco_retry(retry: int = 5, retry_sleep: int = 3): def deco_retry(retry: int = 5, retry_sleep: int = 3):
def deco_func(func): def deco_func(func):
@functools.wraps(func) @functools.wraps(func)
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
_retry = 5 if callable(retry) else retry _retry = 5 if callable(retry) else retry