liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-16 09:11:00 +08:00
Code Activity

update-csi500

Browse Source
This commit is contained in:
Linlang Lv (iSoftStone)
2022-02-28 03:48:07 +08:00
parent 74cc21fc2c
commit 40dd84857c
2 changed files with 96 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
scripts/data_collector/cn_index/collector.py
View File

@@ -5,7 +5,6 @@ import re
import abc import abc
import sys import sys
import importlib import importlib
from tqdm import tqdm
from io import BytesIO from io import BytesIO
from typing import List, Iterable from typing import List, Iterable
from pathlib import Path from pathlib import Path
@@ -14,7 +13,7 @@ import fire
import requests import requests
import pandas as pd import pandas as pd
import baostock as bs import baostock as bs
from lxml import etree from tqdm import tqdm
from loguru import logger from loguru import logger
CUR_DIR = Path(__file__).resolve().parent CUR_DIR = Path(__file__).resolve().parent
@@ -47,7 +46,6 @@ def retry_request(url: str, method: str = "get", exclude_status: List = None):
class CSIIndex(IndexBase): class CSIIndex(IndexBase):
@property @property
def calendar_list(self) -> List[pd.Timestamp]: def calendar_list(self) -> List[pd.Timestamp]:
"""get history trading date """get history trading date
@@ -74,20 +72,20 @@ class CSIIndex(IndexBase):
@abc.abstractmethod @abc.abstractmethod
def bench_start_date(self) -> pd.Timestamp: def bench_start_date(self) -> pd.Timestamp:
""" """
Returns Returns
------- -------
index start date index start date
""" """
raise NotImplementedError("rewrite bench_start_date") raise NotImplementedError("rewrite bench_start_date")
@property @property
@abc.abstractmethod @abc.abstractmethod
def index_code(self) -> str: def index_code(self) -> str:
""" """
Returns Returns
------- -------
index code index code
""" """
raise NotImplementedError("rewrite index_code") raise NotImplementedError("rewrite index_code")
@property @property
@@ -95,10 +93,10 @@ class CSIIndex(IndexBase):
def html_table_index(self) -> int: def html_table_index(self) -> int:
"""Which table of changes in html """Which table of changes in html
CSI300: 0 CSI300: 0
CSI100: 1 CSI100: 1
:return: :return:
""" """
raise NotImplementedError() raise NotImplementedError()
def format_datetime(self, inst_df: pd.DataFrame) -> pd.DataFrame: def format_datetime(self, inst_df: pd.DataFrame) -> pd.DataFrame:
@@ -149,15 +147,15 @@ class CSIIndex(IndexBase):
def normalize_symbol(symbol: str) -> str: def normalize_symbol(symbol: str) -> str:
""" """
Parameters Parameters
---------- ----------
symbol: str symbol: str
symbol symbol
Returns Returns
------- -------
symbol symbol
""" """
symbol = f"{int(symbol):06}" symbol = f"{int(symbol):06}"
return f"SH{symbol}" if symbol.startswith("60") else f"SZ{symbol}" return f"SH{symbol}" if symbol.startswith("60") else f"SZ{symbol}"
@@ -214,10 +212,10 @@ class CSIIndex(IndexBase):
def _read_change_from_url(self, url: str) -> pd.DataFrame: def _read_change_from_url(self, url: str) -> pd.DataFrame:
"""read change from url """read change from url
Parameters Parameters
---------- ----------
url : str url : str
change url change url
Returns Returns
------- -------
@@ -288,12 +286,12 @@ class CSIIndex(IndexBase):
def get_new_companies(self) -> pd.DataFrame: def get_new_companies(self) -> pd.DataFrame:
""" """
Returns Returns
------- -------
pd.DataFrame: pd.DataFrame:
symbol start_date end_date symbol start_date end_date
SH600000 2000-01-01 2099-12-31 SH600000 2000-01-01 2099-12-31
dtypes: dtypes:
symbol: str symbol: str
@@ -318,7 +316,6 @@ class CSIIndex(IndexBase):
class CSI300(CSIIndex): class CSI300(CSIIndex):
@property @property
def index_code(self): def index_code(self):
return "000300" return "000300"
@@ -329,11 +326,10 @@ class CSI300(CSIIndex):
@property @property
def html_table_index(self): def html_table_index(self):
return 0 return 1
class CSI100(CSIIndex): class CSI100(CSIIndex):
@property @property
def index_code(self): def index_code(self):
return "000903" return "000903"
@@ -344,11 +340,10 @@ class CSI100(CSIIndex):
@property @property
def html_table_index(self): def html_table_index(self):
return 1 return 2
class CSI500(CSIIndex): class CSI500(CSIIndex):
@property @property
def index_code(self): def index_code(self):
return "000905" return "000905"
@@ -366,13 +361,13 @@ class CSI500(CSIIndex):
def get_history_companies(self): def get_history_companies(self):
""" """
Data source：http://baostock.com/baostock/index.php/%E4%B8%AD%E8%AF%81500%E6%88%90%E5%88%86%E8%82%A1 Data source: http://baostock.com/baostock/index.php/%E4%B8%AD%E8%AF%81500%E6%88%90%E5%88%86%E8%82%A1
Avoid a large number of parallel data acquisition, Avoid a large number of parallel data acquisition,
such as 1000 times of concurrent data acquisition, because IP will be blocked such as 1000 times of concurrent data acquisition, because IP will be blocked
Returns Returns
------- -------
""" """
lg = bs.login() lg = bs.login()
today = pd.datetime.now() today = pd.datetime.now()
date_range = pd.DataFrame(pd.date_range(start="2007-01-15", end=today, freq="7D"))[0].dt.date date_range = pd.DataFrame(pd.date_range(start="2007-01-15", end=today, freq="7D"))[0].dt.date
@@ -391,7 +386,12 @@ class CSI500(CSIIndex):
def get_instruments( def get_instruments(
qlib_dir: str, index_name: str, method: str = "parse_instruments", request_retry: int = 5, retry_sleep: int = 3 qlib_dir: str,
index_name: str,
method: str = "parse_instruments",
freq: str = "day",
request_retry: int = 5,
retry_sleep: int = 3,
): ):
""" """
@@ -410,13 +410,13 @@ def get_instruments(
retry_sleep: int retry_sleep: int
request sleep, by default 3 request sleep, by default 3
Examples Examples
------- -------
# parse instruments # parse instruments
$ python collector.py --index_name CSI300 --qlib_dir ~/.qlib/qlib_data/cn_data --method parse_instruments $ python collector.py --index_name CSI300 --qlib_dir ~/.qlib/qlib_data/cn_data --method parse_instruments
# parse new companies # parse new companies
$ python collector.py --index_name CSI300 --qlib_dir ~/.qlib/qlib_data/cn_data --method save_new_companies $ python collector.py --index_name CSI300 --qlib_dir ~/.qlib/qlib_data/cn_data --method save_new_companies
""" """
_cur_module = importlib.import_module("data_collector.cn_index.collector") _cur_module = importlib.import_module("data_collector.cn_index.collector")

93
scripts/data_collector/utils.py
View File

@@ -34,6 +34,7 @@ CALENDAR_BENCH_URL_MAP = {
"ALL": CALENDAR_URL_BASE.format(market=1, bench_code="000905"), "ALL": CALENDAR_URL_BASE.format(market=1, bench_code="000905"),
# NOTE: Use the time series of ^GSPC(SP500) as the sequence of all stocks # NOTE: Use the time series of ^GSPC(SP500) as the sequence of all stocks
"US_ALL": "^GSPC", "US_ALL": "^GSPC",
"IN_ALL": "^NSEI",
} }
_BENCH_CALENDAR_LIST = None _BENCH_CALENDAR_LIST = None
@@ -52,15 +53,15 @@ MINIMUM_SYMBOLS_NUM = 3900
def get_calendar_list(bench_code="CSI300") -> List[pd.Timestamp]: def get_calendar_list(bench_code="CSI300") -> List[pd.Timestamp]:
"""get SH/SZ history calendar list """get SH/SZ history calendar list
Parameters Parameters
---------- ----------
bench_code: str bench_code: str
value from ["CSI300", "CSI500", "ALL", "US_ALL"] value from ["CSI300", "CSI500", "ALL", "US_ALL"]
Returns Returns
------- -------
history calendar list history calendar list
""" """
logger.info(f"get calendar list: {bench_code}......") logger.info(f"get calendar list: {bench_code}......")
@@ -177,10 +178,10 @@ def get_calendar_list_by_ratio(
def get_hs_stock_symbols() -> list: def get_hs_stock_symbols() -> list:
"""get SH/SZ stock symbols """get SH/SZ stock symbols
Returns Returns
------- -------
stock symbols stock symbols
""" """
global _HS_SYMBOLS global _HS_SYMBOLS
def _get_symbol(): def _get_symbol():
@@ -221,10 +222,10 @@ def get_hs_stock_symbols() -> list:
def get_us_stock_symbols(qlib_data_path: [str, Path] = None) -> list: def get_us_stock_symbols(qlib_data_path: [str, Path] = None) -> list:
"""get US stock symbols """get US stock symbols
Returns Returns
------- -------
stock symbols stock symbols
""" """
global _US_SYMBOLS global _US_SYMBOLS
@deco_retry @deco_retry
@@ -416,16 +417,16 @@ def get_cg_crypto_symbols(qlib_data_path: [str, Path] = None) -> list:
def symbol_suffix_to_prefix(symbol: str, capital: bool = True) -> str: def symbol_suffix_to_prefix(symbol: str, capital: bool = True) -> str:
"""symbol suffix to prefix """symbol suffix to prefix
Parameters Parameters
---------- ----------
symbol: str symbol: str
symbol symbol
capital : bool capital : bool
by default True by default True
Returns Returns
------- -------
""" """
code, exchange = symbol.split(".") code, exchange = symbol.split(".")
if exchange.lower() in ["sh", "ss"]: if exchange.lower() in ["sh", "ss"]:
res = f"sh{code}" res = f"sh{code}"
@@ -437,24 +438,22 @@ def symbol_suffix_to_prefix(symbol: str, capital: bool = True) -> str:
def symbol_prefix_to_sufix(symbol: str, capital: bool = True) -> str: def symbol_prefix_to_sufix(symbol: str, capital: bool = True) -> str:
"""symbol prefix to sufix """symbol prefix to sufix
Parameters Parameters
---------- ----------
symbol: str symbol: str
symbol symbol
capital : bool capital : bool
by default True by default True
Returns Returns
------- -------
""" """
res = f"{symbol[:-2]}.{symbol[-2:]}" res = f"{symbol[:-2]}.{symbol[-2:]}"
return res.upper() if capital else res.lower() return res.upper() if capital else res.lower()
def deco_retry(retry: int = 5, retry_sleep: int = 3): def deco_retry(retry: int = 5, retry_sleep: int = 3):
def deco_func(func): def deco_func(func):
@functools.wraps(func) @functools.wraps(func)
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
_retry = 5 if callable(retry) else retry _retry = 5 if callable(retry) else retry
@@ -480,19 +479,19 @@ def deco_retry(retry: int = 5, retry_sleep: int = 3):
def get_trading_date_by_shift(trading_list: list, trading_date: pd.Timestamp, shift: int = 1): def get_trading_date_by_shift(trading_list: list, trading_date: pd.Timestamp, shift: int = 1):
"""get trading date by shift """get trading date by shift
Parameters Parameters
---------- ----------
trading_list: list trading_list: list
trading calendar list trading calendar list
shift : int shift : int
shift, default is 1 shift, default is 1
trading_date : pd.Timestamp trading_date : pd.Timestamp
trading date trading date
Returns Returns
------- -------
""" """
trading_date = pd.Timestamp(trading_date) trading_date = pd.Timestamp(trading_date)
left_index = bisect.bisect_left(trading_list, trading_date) left_index = bisect.bisect_left(trading_list, trading_date)
try: try: