liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-07 04:50:56 +08:00
Code Activity

fix dump_bin:DumpDataUpdate (#783)

Browse Source
This commit is contained in:
Pengrong Zhu
2021-12-29 09:29:08 +08:00
committed by GitHub
parent 622303b83a
commit 1ecdfd45fe
1 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
scripts/dump_bin.py
View File

@@ -214,6 +214,9 @@ class DumpDataBase:
if df.empty: if df.empty:
logger.warning(f"{features_dir.name} data is None or empty") logger.warning(f"{features_dir.name} data is None or empty")
return return
if not calendar_list:
logger.warning("calendar_list is empty")
return
# align index # align index
_df = self.data_merge_calendar(df, calendar_list) _df = self.data_merge_calendar(df, calendar_list)
# used when creating a bin file # used when creating a bin file
@@ -231,6 +234,9 @@ class DumpDataBase:
np.hstack([date_index, _df[field]]).astype("<f").tofile(str(bin_path.resolve())) np.hstack([date_index, _df[field]]).astype("<f").tofile(str(bin_path.resolve()))
def _dump_bin(self, file_or_data: [Path, pd.DataFrame], calendar_list: List[pd.Timestamp]): def _dump_bin(self, file_or_data: [Path, pd.DataFrame], calendar_list: List[pd.Timestamp]):
if not calendar_list:
logger.warning("calendar_list is empty")
return
if isinstance(file_or_data, pd.DataFrame): if isinstance(file_or_data, pd.DataFrame):
if file_or_data.empty: if file_or_data.empty:
return return
@@ -459,14 +465,15 @@ class DumpDataUpdate(DumpDataBase):
if _code in self._update_instruments: if _code in self._update_instruments:
# exists stock, will append data # exists stock, will append data
_update_calendars = ( _update_calendars = (
_df[_df[self.date_field_name] > self._update_instruments[_code][self.INSTRUMENTS_START_FIELD]][ _df[_df[self.date_field_name] > self._update_instruments[_code][self.INSTRUMENTS_END_FIELD]][
self.date_field_name self.date_field_name
] ]
.sort_values() .sort_values()
.to_list() .to_list()
) )
self._update_instruments[_code][self.INSTRUMENTS_END_FIELD] = self._format_datetime(_end) if _update_calendars:
futures[executor.submit(self._dump_bin, _df, _update_calendars)] = _code self._update_instruments[_code][self.INSTRUMENTS_END_FIELD] = self._format_datetime(_end)
futures[executor.submit(self._dump_bin, _df, _update_calendars)] = _code
else: else:
# new stock # new stock
_dt_range = self._update_instruments.setdefault(_code, dict()) _dt_range = self._update_instruments.setdefault(_code, dict())