fix us instruments

scripts/data_collector/index.py

@@ -24,6 +24,7 @@ class IndexBase:
     INSTRUMENTS_COLUMNS = [SYMBOL_FIELD_NAME, START_DATE_FIELD, END_DATE_FIELD]
     REMOVE = "remove"
     ADD = "add"
+    INST_PREFIX = ""
 
     def __init__(self, index_name: str, qlib_dir: [str, Path] = None, request_retry: int = 5, retry_sleep: int = 3):
         """
@@ -196,7 +197,11 @@ class IndexBase:
                 _tmp_df = pd.DataFrame([[_row.symbol, self.bench_start_date, _row.date]], columns=instruments_columns)
                 new_df = new_df.append(_tmp_df, sort=False)
 
-        new_df.loc[:, instruments_columns].to_csv(
+        inst_df = new_df.loc[:, instruments_columns]
+        _inst_prefix = self.INST_PREFIX.strip()
+        if _inst_prefix:
+            inst_df["save_inst"] = inst_df[self.SYMBOL_FIELD_NAME].apply(lambda x: f"{_inst_prefix}{x}")
+        inst_df.to_csv(
             self.instruments_dir.joinpath(f"{self.index_name.lower()}.txt"), sep="\t", index=False, header=None
         )
         logger.info(f"parse {self.index_name.lower()} companies finished.")

scripts/data_collector/us_index/collector.py

@@ -33,6 +33,10 @@ WIKI_INDEX_NAME_MAP = {
 
 
 class WIKIIndex(IndexBase):
+    # NOTE: The US stock code contains "PRN", and the directory cannot be created on Windows system, use the "_" prefix
+    # https://superuser.com/questions/613313/why-cant-we-make-con-prn-null-folder-in-windows
+    INST_PREFIX = "_"
+
     def __init__(self, index_name: str, qlib_dir: [str, Path] = None, request_retry: int = 5, retry_sleep: int = 3):
         super(WIKIIndex, self).__init__(
             index_name=index_name, qlib_dir=qlib_dir, request_retry=request_retry, retry_sleep=retry_sleep

scripts/data_collector/utils.py

@@ -184,9 +184,14 @@ def get_us_stock_symbols(qlib_data_path: [str, Path] = None) -> list:
                     names=["symbol", "start_date", "end_date"],
                 )
                 _all_symbols += ins_df["symbol"].unique().tolist()
-        _US_SYMBOLS = sorted(
-            set(map(lambda x: x.replace(".", "-"), filter(lambda x: len(x) < 8 and not x.endswith("WS"), _all_symbols)))
-        )
+
+        def _format(s_):
+            s_ = s_.replace(".", "-")
+            s_ = s_.strip("$")
+            s_ = s_.strip("*")
+            return s_
+
+        _US_SYMBOLS = sorted(set(map(_format, filter(lambda x: len(x) < 8 and not x.endswith("WS"), _all_symbols))))
 
     return _US_SYMBOLS