Resolve issues while running Automatic update of daily frequency data (from yahoo finance) for US region (#1358)

* Update YahooNormalizeUS1dExtend(#1196)

* Prevent pandas read_csv errors while running update_data_to_bin for US region

* Fix parse_index error while running update_data_to_bin for US region

* prevent pandas.read_csv error on specific symbol names

* Reordering parameters for better rendering

* removes prefix during feature_dir existence checking

* add explanation comments

scripts/data_collector/base.py

@@ -289,7 +289,20 @@ class Normalize:
 
     def _executor(self, file_path: Path):
         file_path = Path(file_path)
-        df = pd.read_csv(file_path)
+
+        # some symbol_field values such as TRUE, NA are decoded as True(bool), NaN(np.float) by pandas default csv parsing.
+        # manually defines dtype and na_values of the symbol_field.
+        default_na = pd._libs.parsers.STR_NA_VALUES
+        symbol_na = default_na.copy()
+        symbol_na.remove("NA")
+        columns = pd.read_csv(file_path, nrows=0).columns
+        df = pd.read_csv(
+            file_path,
+            dtype={self._symbol_field_name: str},
+            keep_default_na=False,
+            na_values={col: symbol_na if col == self._symbol_field_name else default_na for col in columns},
+        )
+
         df = self._normalize_obj.normalize(df)
         if df is not None and not df.empty:
             if self._end_date is not None:

scripts/data_collector/yahoo/collector.py

@@ -817,6 +817,10 @@ class YahooNormalizeUS1d(YahooNormalizeUS, YahooNormalize1d):
     pass
 
 
+class YahooNormalizeUS1dExtend(YahooNormalizeUS, YahooNormalize1dExtend):
+    pass
+
+
 class YahooNormalizeUS1min(YahooNormalizeUS, YahooNormalize1minOffline):
     CALC_PAUSED_NUM = False
 
@@ -1196,7 +1200,7 @@ class Run(BaseRun):
             importlib.import_module(f"data_collector.{_region}_index.collector"), "get_instruments"
         )
         for _index in index_list:
-            get_instruments(str(qlib_data_1d_dir), _index)
+            get_instruments(str(qlib_data_1d_dir), _index, market_index=f"{_region}_index")
 
 
 if __name__ == "__main__":