From 0b83fb3564b31e6988b8d8cde62104823b055c3d Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Mon, 28 Jun 2021 05:23:45 +0000 Subject: [PATCH] more general exception --- qlib/data/dataset/loader.py | 8 +++----- qlib/utils/exceptions.py | 5 +++++ qlib/workflow/online/utils.py | 34 ++++++++++++++-------------------- qlib/workflow/recorder.py | 8 ++++---- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/qlib/data/dataset/loader.py b/qlib/data/dataset/loader.py index b4c75e104..54c00dc7e 100644 --- a/qlib/data/dataset/loader.py +++ b/qlib/data/dataset/loader.py @@ -12,7 +12,7 @@ from typing import Tuple, Union from qlib.data import D from qlib.data import filter as filter_module from qlib.data.filter import BaseDFilter -from qlib.utils import load_dataset, init_instance_by_config +from qlib.utils import load_dataset, init_instance_by_config, time_to_slc_point from qlib.log import get_module_logger @@ -208,10 +208,8 @@ class StaticDataLoader(DataLoader): if start_time is None and end_time is None: return df # NOTE: avoid copy by loc # pd.Timestamp(None) == NaT, use NaT as index can not fetch correct thing, so do not change None. - if start_time is not None: - start_time = pd.Timestamp(start_time) - if end_time is not None: - end_time = pd.Timestamp(end_time) + start_time = time_to_slc_point(start_time) + end_time = time_to_slc_point(end_time) return df.loc[start_time:end_time] def _maybe_load_raw_data(self): diff --git a/qlib/utils/exceptions.py b/qlib/utils/exceptions.py index 69712172b..dad12506b 100644 --- a/qlib/utils/exceptions.py +++ b/qlib/utils/exceptions.py @@ -10,3 +10,8 @@ class QlibException(Exception): # Error type for reinitialization when starting an experiment class RecorderInitializationError(QlibException): pass + + +# Error type for Recorder when can not load object +class LoadObjectError(QlibException): + pass diff --git a/qlib/workflow/online/utils.py b/qlib/workflow/online/utils.py index 83cacbb88..2cd972494 100644 --- a/qlib/workflow/online/utils.py +++ b/qlib/workflow/online/utils.py @@ -12,7 +12,7 @@ from qlib.data.dataset import TSDatasetH from qlib.log import get_module_logger from qlib.utils import get_cls_kwargs -from qlib.utils.exceptions import QlibException +from qlib.utils.exceptions import LoadObjectError from qlib.workflow.online.update import PredUpdater from qlib.workflow.recorder import Recorder from qlib.workflow.task.utils import list_recorders @@ -138,12 +138,7 @@ class OnlineToolR(OnlineTool): exp_name (str): the experiment name. If None, then use default_exp_name. """ - if exp_name is None: - if self.default_exp_name is None: - raise ValueError( - "Both default_exp_name and exp_name are None. OnlineToolR needs a specific experiment." - ) - exp_name = self.default_exp_name + exp_name = self._get_exp_name(exp_name) if isinstance(recorder, Recorder): recorder = [recorder] recs = list_recorders(exp_name) @@ -160,12 +155,7 @@ class OnlineToolR(OnlineTool): Returns: list: a list of `online` models. """ - if exp_name is None: - if self.default_exp_name is None: - raise ValueError( - "Both default_exp_name and exp_name are None. OnlineToolR needs a specific experiment." - ) - exp_name = self.default_exp_name + exp_name = self._get_exp_name(exp_name) return list(list_recorders(exp_name, lambda rec: self.get_online_tag(rec) == self.ONLINE_TAG).values()) def update_online_pred(self, to_date=None, exp_name: str = None): @@ -176,12 +166,7 @@ class OnlineToolR(OnlineTool): to_date (pd.Timestamp): the pred before this date will be updated. None for updating to latest time in Calendar. exp_name (str): the experiment name. If None, then use default_exp_name. """ - if exp_name is None: - if self.default_exp_name is None: - raise ValueError( - "Both default_exp_name and exp_name are None. OnlineToolR needs a specific experiment." - ) - exp_name = self.default_exp_name + exp_name = self._get_exp_name(exp_name) online_models = self.online_models(exp_name=exp_name) for rec in online_models: hist_ref = 0 @@ -192,10 +177,19 @@ class OnlineToolR(OnlineTool): hist_ref = kwargs.get("step_len", TSDatasetH.DEFAULT_STEP_LEN) try: updater = PredUpdater(rec, to_date=to_date, hist_ref=hist_ref) - except QlibException as e: + except LoadObjectError as e: # skip the recorder without pred self.logger.warn(f"An exception `{str(e)}` happened when load `pred.pkl`, skip it.") continue updater.update() self.logger.info(f"Finished updating {len(online_models)} online model predictions of {exp_name}.") + + def _get_exp_name(self, exp_name): + if exp_name is None: + if self.default_exp_name is None: + raise ValueError( + "Both default_exp_name and exp_name are None. OnlineToolR needs a specific experiment." + ) + exp_name = self.default_exp_name + return exp_name diff --git a/qlib/workflow/recorder.py b/qlib/workflow/recorder.py index e11287ca1..1bb8bea6e 100644 --- a/qlib/workflow/recorder.py +++ b/qlib/workflow/recorder.py @@ -6,8 +6,7 @@ import shutil, os, pickle, tempfile, codecs, pickle from pathlib import Path from datetime import datetime -from mlflow.exceptions import MlflowException -from qlib.utils.exceptions import QlibException +from qlib.utils.exceptions import LoadObjectError from ..utils.objm import FileManager from ..log import get_module_logger @@ -311,12 +310,13 @@ class MLflowRecorder(Recorder): def load_object(self, name): assert self.uri is not None, "Please start the experiment and recorder first before using recorder directly." + try: path = self.client.download_artifacts(self.id, name) with Path(path).open("rb") as f: return pickle.load(f) - except OSError as e: - raise QlibException(message=str(e)) + except Exception as e: + raise LoadObjectError(message=str(e)) def log_params(self, **kwargs): for name, data in kwargs.items():