diff --git a/scripts/data_collector/utils.py b/scripts/data_collector/utils.py index 1a8d479d9..883a1c551 100644 --- a/scripts/data_collector/utils.py +++ b/scripts/data_collector/utils.py @@ -32,6 +32,7 @@ CALENDAR_BENCH_URL_MAP = { "ALL": CALENDAR_URL_BASE.format(market=1, bench_code="000905"), # NOTE: Use the time series of ^GSPC(SP500) as the sequence of all stocks "US_ALL": "^GSPC", + "IN_ALL": "^NSEI", } @@ -39,6 +40,7 @@ _BENCH_CALENDAR_LIST = None _ALL_CALENDAR_LIST = None _HS_SYMBOLS = None _US_SYMBOLS = None +_IN_SYMBOLS = None _EN_FUND_SYMBOLS = None _CALENDAR_MAP = {} @@ -67,7 +69,7 @@ def get_calendar_list(bench_code="CSI300") -> List[pd.Timestamp]: calendar = _CALENDAR_MAP.get(bench_code, None) if calendar is None: - if bench_code.startswith("US_"): + if bench_code.startswith("US_") or bench_code.startswith("IN_"): df = Ticker(CALENDAR_BENCH_URL_MAP[bench_code]).history(interval="1d", period="max") calendar = df.index.get_level_values(level="date").map(pd.Timestamp).unique().tolist() else: @@ -298,6 +300,47 @@ def get_us_stock_symbols(qlib_data_path: [str, Path] = None) -> list: return _US_SYMBOLS +def get_in_stock_symbols(qlib_data_path: [str, Path] = None) -> list: + """get IN stock symbols + + Returns + ------- + stock symbols + """ + global _IN_SYMBOLS + + @deco_retry + def _get_nifty(): + url = f"https://www1.nseindia.com/content/equities/EQUITY_L.csv" + df = pd.read_csv(url) + df = df.rename(columns={"SYMBOL": "Symbol"}) + df["Symbol"] = df["Symbol"] + ".NS" + _symbols = df["Symbol"].dropna() + _symbols = _symbols.unique().tolist() + return _symbols + + if _IN_SYMBOLS is None: + _all_symbols = _get_nifty() + if qlib_data_path is not None: + for _index in ["nifty"]: + ins_df = pd.read_csv( + Path(qlib_data_path).joinpath(f"instruments/{_index}.txt"), + sep="\t", + names=["symbol", "start_date", "end_date"], + ) + _all_symbols += ins_df["symbol"].unique().tolist() + + def _format(s_): + s_ = s_.replace(".", "-") + s_ = s_.strip("$") + s_ = s_.strip("*") + return s_ + + _IN_SYMBOLS = sorted(set(_all_symbols)) + + return _IN_SYMBOLS + + def get_en_fund_symbols(qlib_data_path: [str, Path] = None) -> list: """get en fund symbols diff --git a/scripts/data_collector/yahoo/README.md b/scripts/data_collector/yahoo/README.md index 50f731e38..0deef9c95 100644 --- a/scripts/data_collector/yahoo/README.md +++ b/scripts/data_collector/yahoo/README.md @@ -37,7 +37,7 @@ pip install -r requirements.txt - user can append data to `v2`: [automatic update of daily frequency data](#automatic-update-of-daily-frequency-datafrom-yahoo-finance) - **the [benchmarks](https://github.com/microsoft/qlib/tree/main/examples/benchmarks) for qlib use `v1`**, *due to the unstable access to historical data by YahooFinance, there are some differences between `v2` and `v1`* - `interval`: `1d` or `1min`, by default `1d` - - `region`: `cn` or `us`, by default `cn` + - `region`: `cn` or `us` or `in`, by default `cn` - `delete_old`: delete existing data from `target_dir`(*features, calendars, instruments, dataset_cache, features_cache*), value from [`True`, `False`], by default `True` - `exists_skip`: traget_dir data already exists, skip `get_data`, value from [`True`, `False`], by default `False` - examples: @@ -50,6 +50,10 @@ pip install -r requirements.txt python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/qlib_us_1d --region us --interval 1d # us 1min python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/qlib_us_1min --region us --interval 1min + # in 1d + python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/qlib_in_1d --region in --interval 1d + # in 1min + python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/qlib_in_1min --region in --interval 1min ``` ### Collector *YahooFinance* data to qlib @@ -60,7 +64,7 @@ pip install -r requirements.txt - `source_dir`: save the directory - `interval`: `1d` or `1min`, by default `1d` > **due to the limitation of the *YahooFinance API*, only the last month's data is available in `1min`** - - `region`: `CN` or `US`, by default `CN` + - `region`: `CN` or `US` or `IN`, by default `CN` - `delay`: `time.sleep(delay)`, by default *0.5* - `start`: start datetime, by default *"2000-01-01"*; *closed interval(including start)* - `end`: end datetime, by default `pd.Timestamp(datetime.datetime.now() + pd.Timedelta(days=1))`; *open interval(excluding end)* @@ -78,6 +82,10 @@ pip install -r requirements.txt python collector.py download_data --source_dir ~/.qlib/stock_data/source/us_1d --start 2020-01-01 --end 2020-12-31 --delay 1 --interval 1d --region US # us 1min data python collector.py download_data --source_dir ~/.qlib/stock_data/source/us_1min --delay 1 --interval 1min --region US + # in 1d data + python collector.py download_data --source_dir ~/.qlib/stock_data/source/in_1d --start 2020-01-01 --end 2020-12-31 --delay 1 --interval 1d --region IN + # in 1min data + python collector.py download_data --source_dir ~/.qlib/stock_data/source/in_1min --delay 1 --interval 1min --region IN ``` 2. normalize data: `python scripts/data_collector/yahoo/collector.py normalize_data` @@ -87,7 +95,7 @@ pip install -r requirements.txt - `max_workers`: number of concurrent, by default *1* - `interval`: `1d` or `1min`, by default `1d` > if **`interval == 1min`**, `qlib_data_1d_dir` cannot be `None` - - `region`: `CN` or `US`, by default `CN` + - `region`: `CN` or `US` or `IN`, by default `CN` - `date_field_name`: column *name* identifying time in csv files, by default `date` - `symbol_field_name`: column *name* identifying symbol in csv files, by default `symbol` - `end_date`: if not `None`, normalize the last date saved (*including end_date*); if `None`, it will ignore this parameter; by default `None` diff --git a/scripts/data_collector/yahoo/collector.py b/scripts/data_collector/yahoo/collector.py index feb28a94f..e4422c4ce 100644 --- a/scripts/data_collector/yahoo/collector.py +++ b/scripts/data_collector/yahoo/collector.py @@ -34,6 +34,7 @@ from data_collector.utils import ( get_calendar_list, get_hs_stock_symbols, get_us_stock_symbols, + get_in_stock_symbols, generate_minutes_calendar_from_daily, ) @@ -279,6 +280,32 @@ class YahooCollectorUS1min(YahooCollectorUS): pass +class YahooCollectorIN(YahooCollector, ABC): + def get_instrument_list(self): + logger.info("get INDIA stock symbols......") + symbols = get_in_stock_symbols() + logger.info(f"get {len(symbols)} symbols.") + return symbols + + def download_index_data(self): + pass + + def normalize_symbol(self, symbol): + return code_to_fname(symbol).upper() + + @property + def _timezone(self): + return "Asia/Kolkata" + + +class YahooCollectorIN1d(YahooCollectorIN): + pass + + +class YahooCollectorIN1min(YahooCollectorIN): + pass + + class YahooNormalize(BaseNormalize): COLUMNS = ["open", "close", "high", "low", "volume"] DAILY_FORMAT = "%Y-%m-%d" @@ -738,6 +765,29 @@ class YahooNormalizeUS1min(YahooNormalizeUS, YahooNormalize1minOffline): return fname_to_code(symbol) +class YahooNormalizeIN: + def _get_calendar_list(self) -> Iterable[pd.Timestamp]: + return get_calendar_list("IN_ALL") + + +class YahooNormalizeIN1d(YahooNormalizeIN, YahooNormalize1d): + pass + + +class YahooNormalizeIN1min(YahooNormalizeIN, YahooNormalize1minOffline): + CALC_PAUSED_NUM = False + + def _get_calendar_list(self) -> Iterable[pd.Timestamp]: + # TODO: support 1min + raise ValueError("Does not support 1min") + + def _get_1d_calendar_list(self): + return get_calendar_list("IN_ALL") + + def symbol_to_yahoo(self, symbol): + return fname_to_code(symbol) + + class YahooNormalizeCN: def _get_calendar_list(self) -> Iterable[pd.Timestamp]: # TODO: from MSN