add dependencies for generate hdf5 files

* fix pkl file not loading in StaticDataLoader

* resolve hard code

* resolve hard code

pyproject.toml

@@ -44,6 +44,8 @@ dependencies = [
   "matplotlib",
   "jupyter",
   "nbconvert",
+  "pyarrow",
+  "tables",
 ]
 
 [project.optional-dependencies]
@@ -79,7 +81,6 @@ package = [
 test = [
   "yahooquery",
   "baostock",
-  "akshare",
 ]
 analysis = [
   "plotly",

qlib/data/dataset/loader.py

@@ -279,8 +279,11 @@ class StaticDataLoader(DataLoader, Serializable):
             )
             self._data.sort_index(inplace=True)
         elif isinstance(self._config, (str, Path)):
-            with Path(self._config).open("rb") as f:
-                self._data = pickle.load(f)
+            if str(self._config).strip().endswith(".parquet"):
+                self._data = pd.read_parquet(self._config, engine="pyarrow")
+            else:
+                with Path(self._config).open("rb") as f:
+                    self._data = pickle.load(f)
         elif isinstance(self._config, pd.DataFrame):
             self._data = self._config
 

scripts/data_collector/utils.py

@@ -13,7 +13,6 @@ import functools
 from pathlib import Path
 from typing import Iterable, Tuple, List
 
-import akshare as ak
 import numpy as np
 import pandas as pd
 from loguru import logger
@@ -203,9 +202,18 @@ def get_hs_stock_symbols() -> list:
         -------
             {600000.ss, 600001.ss, 600002.ss, 600003.ss, ...}
         """
-        stock_info_a_code_name_df = ak.stock_info_a_code_name()
-        stock_codes = stock_info_a_code_name_df["code"].tolist()
-        _symbols = [code for code in stock_codes if code and code.strip()]
+        url = "http://99.push2.eastmoney.com/api/qt/clist/get?pn=1&pz=10000&po=1&np=1&fs=m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23,m:0+t:81+s:2048&fields=f12"
+        try:
+            resp = requests.get(url, timeout=None)
+            resp.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            raise requests.exceptions.HTTPError(f"Request to {url} failed with status code {resp.status_code}") from e
+
+        try:
+            _symbols = [_v["f12"] for _v in resp.json()["data"]["diff"]]
+        except Exception as e:
+            logger.warning("An error occurred while extracting data from the response.")
+            raise
 
         if len(_symbols) < 3900:
             raise ValueError("The complete list of stocks is not available.")

scripts/data_collector/yahoo/README.md

@@ -50,6 +50,12 @@ pip install -r requirements.txt
     python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data_1min --region cn --interval 1min
     # us 1d
     python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/us_data --region us --interval 1d
+    # us 1min
+    python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/us_data_1min --region us --interval 1min
+    # in 1d
+    python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/in_data --region in --interval 1d
+    # in 1min
+    python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/in_data_1min --region in --interval 1min
     ```
 
 ### Collector *YahooFinance* data to qlib