1
0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-17 09:24:34 +08:00

more general exception

This commit is contained in:
lzh222333
2021-06-28 05:23:45 +00:00
committed by you-n-g
parent d96f7a67c6
commit 0b83fb3564
4 changed files with 26 additions and 29 deletions

View File

@@ -12,7 +12,7 @@ from typing import Tuple, Union
from qlib.data import D from qlib.data import D
from qlib.data import filter as filter_module from qlib.data import filter as filter_module
from qlib.data.filter import BaseDFilter from qlib.data.filter import BaseDFilter
from qlib.utils import load_dataset, init_instance_by_config from qlib.utils import load_dataset, init_instance_by_config, time_to_slc_point
from qlib.log import get_module_logger from qlib.log import get_module_logger
@@ -208,10 +208,8 @@ class StaticDataLoader(DataLoader):
if start_time is None and end_time is None: if start_time is None and end_time is None:
return df # NOTE: avoid copy by loc return df # NOTE: avoid copy by loc
# pd.Timestamp(None) == NaT, use NaT as index can not fetch correct thing, so do not change None. # pd.Timestamp(None) == NaT, use NaT as index can not fetch correct thing, so do not change None.
if start_time is not None: start_time = time_to_slc_point(start_time)
start_time = pd.Timestamp(start_time) end_time = time_to_slc_point(end_time)
if end_time is not None:
end_time = pd.Timestamp(end_time)
return df.loc[start_time:end_time] return df.loc[start_time:end_time]
def _maybe_load_raw_data(self): def _maybe_load_raw_data(self):

View File

@@ -10,3 +10,8 @@ class QlibException(Exception):
# Error type for reinitialization when starting an experiment # Error type for reinitialization when starting an experiment
class RecorderInitializationError(QlibException): class RecorderInitializationError(QlibException):
pass pass
# Error type for Recorder when can not load object
class LoadObjectError(QlibException):
pass

View File

@@ -12,7 +12,7 @@ from qlib.data.dataset import TSDatasetH
from qlib.log import get_module_logger from qlib.log import get_module_logger
from qlib.utils import get_cls_kwargs from qlib.utils import get_cls_kwargs
from qlib.utils.exceptions import QlibException from qlib.utils.exceptions import LoadObjectError
from qlib.workflow.online.update import PredUpdater from qlib.workflow.online.update import PredUpdater
from qlib.workflow.recorder import Recorder from qlib.workflow.recorder import Recorder
from qlib.workflow.task.utils import list_recorders from qlib.workflow.task.utils import list_recorders
@@ -138,12 +138,7 @@ class OnlineToolR(OnlineTool):
exp_name (str): the experiment name. If None, then use default_exp_name. exp_name (str): the experiment name. If None, then use default_exp_name.
""" """
if exp_name is None: exp_name = self._get_exp_name(exp_name)
if self.default_exp_name is None:
raise ValueError(
"Both default_exp_name and exp_name are None. OnlineToolR needs a specific experiment."
)
exp_name = self.default_exp_name
if isinstance(recorder, Recorder): if isinstance(recorder, Recorder):
recorder = [recorder] recorder = [recorder]
recs = list_recorders(exp_name) recs = list_recorders(exp_name)
@@ -160,12 +155,7 @@ class OnlineToolR(OnlineTool):
Returns: Returns:
list: a list of `online` models. list: a list of `online` models.
""" """
if exp_name is None: exp_name = self._get_exp_name(exp_name)
if self.default_exp_name is None:
raise ValueError(
"Both default_exp_name and exp_name are None. OnlineToolR needs a specific experiment."
)
exp_name = self.default_exp_name
return list(list_recorders(exp_name, lambda rec: self.get_online_tag(rec) == self.ONLINE_TAG).values()) return list(list_recorders(exp_name, lambda rec: self.get_online_tag(rec) == self.ONLINE_TAG).values())
def update_online_pred(self, to_date=None, exp_name: str = None): def update_online_pred(self, to_date=None, exp_name: str = None):
@@ -176,12 +166,7 @@ class OnlineToolR(OnlineTool):
to_date (pd.Timestamp): the pred before this date will be updated. None for updating to latest time in Calendar. to_date (pd.Timestamp): the pred before this date will be updated. None for updating to latest time in Calendar.
exp_name (str): the experiment name. If None, then use default_exp_name. exp_name (str): the experiment name. If None, then use default_exp_name.
""" """
if exp_name is None: exp_name = self._get_exp_name(exp_name)
if self.default_exp_name is None:
raise ValueError(
"Both default_exp_name and exp_name are None. OnlineToolR needs a specific experiment."
)
exp_name = self.default_exp_name
online_models = self.online_models(exp_name=exp_name) online_models = self.online_models(exp_name=exp_name)
for rec in online_models: for rec in online_models:
hist_ref = 0 hist_ref = 0
@@ -192,10 +177,19 @@ class OnlineToolR(OnlineTool):
hist_ref = kwargs.get("step_len", TSDatasetH.DEFAULT_STEP_LEN) hist_ref = kwargs.get("step_len", TSDatasetH.DEFAULT_STEP_LEN)
try: try:
updater = PredUpdater(rec, to_date=to_date, hist_ref=hist_ref) updater = PredUpdater(rec, to_date=to_date, hist_ref=hist_ref)
except QlibException as e: except LoadObjectError as e:
# skip the recorder without pred # skip the recorder without pred
self.logger.warn(f"An exception `{str(e)}` happened when load `pred.pkl`, skip it.") self.logger.warn(f"An exception `{str(e)}` happened when load `pred.pkl`, skip it.")
continue continue
updater.update() updater.update()
self.logger.info(f"Finished updating {len(online_models)} online model predictions of {exp_name}.") self.logger.info(f"Finished updating {len(online_models)} online model predictions of {exp_name}.")
def _get_exp_name(self, exp_name):
if exp_name is None:
if self.default_exp_name is None:
raise ValueError(
"Both default_exp_name and exp_name are None. OnlineToolR needs a specific experiment."
)
exp_name = self.default_exp_name
return exp_name

View File

@@ -6,8 +6,7 @@ import shutil, os, pickle, tempfile, codecs, pickle
from pathlib import Path from pathlib import Path
from datetime import datetime from datetime import datetime
from mlflow.exceptions import MlflowException from qlib.utils.exceptions import LoadObjectError
from qlib.utils.exceptions import QlibException
from ..utils.objm import FileManager from ..utils.objm import FileManager
from ..log import get_module_logger from ..log import get_module_logger
@@ -311,12 +310,13 @@ class MLflowRecorder(Recorder):
def load_object(self, name): def load_object(self, name):
assert self.uri is not None, "Please start the experiment and recorder first before using recorder directly." assert self.uri is not None, "Please start the experiment and recorder first before using recorder directly."
try: try:
path = self.client.download_artifacts(self.id, name) path = self.client.download_artifacts(self.id, name)
with Path(path).open("rb") as f: with Path(path).open("rb") as f:
return pickle.load(f) return pickle.load(f)
except OSError as e: except Exception as e:
raise QlibException(message=str(e)) raise LoadObjectError(message=str(e))
def log_params(self, **kwargs): def log_params(self, **kwargs):
for name, data in kwargs.items(): for name, data in kwargs.items():