diff --git a/qlib/config.py b/qlib/config.py index 52b05568d..75ab0fa3e 100644 --- a/qlib/config.py +++ b/qlib/config.py @@ -105,7 +105,7 @@ _default_config = { "redis_port": 6379, "redis_task_db": 1, # This value can be reset via qlib.init - "logging_level": "INFO", + "logging_level": logging.INFO, # Global configuration of qlib log # logging_level can control the logging level more finely "logging_config": { @@ -124,12 +124,12 @@ _default_config = { "handlers": { "console": { "class": "logging.StreamHandler", - "level": "DEBUG", + "level": logging.DEBUG, "formatter": "logger_format", "filters": ["field_not_found"], } }, - "loggers": {"qlib": {"level": "DEBUG", "handlers": ["console"]}}, + "loggers": {"qlib": {"level": logging.DEBUG, "handlers": ["console"]}}, }, # Defatult config for experiment manager "exp_manager": { @@ -185,7 +185,7 @@ MODE_CONF = { # The nfs should be auto-mounted by qlib on other # serversS(such as PAI) [auto_mount:True] "timeout": 100, - "logging_level": "INFO", + "logging_level": logging.INFO, "region": REG_CN, ## Custom Operator "custom_ops": [], diff --git a/qlib/log.py b/qlib/log.py index 6553dcb11..78f12eb09 100644 --- a/qlib/log.py +++ b/qlib/log.py @@ -3,8 +3,7 @@ import logging -import logging.handlers -import os +from typing import Optional, Text, Dict, Any import re from logging import config as logging_config from time import time @@ -13,16 +12,13 @@ from contextlib import contextmanager from .config import C -def get_module_logger(module_name, level=None): +def get_module_logger(module_name, level: Optional[int] = None): """ Get a logger for a specific module. :param module_name: str Logic module name. :param level: int - :param sh_level: int - Stream handler log level. - :param log_format: str :return: Logger Logger object. """ @@ -103,7 +99,7 @@ class TimeInspector: cls.log_cost_time(info=f"{name} Done") -def set_log_with_config(log_config: dict): +def set_log_with_config(log_config: Dict[Text, Any]): """set log with config :param log_config: @@ -112,6 +108,27 @@ def set_log_with_config(log_config: dict): logging_config.dictConfig(log_config) +def set_log_basic_config(filename: Optional[Text] = None, format: Optional[Text] = None, level: Optional[int] = None): + """ + Set the basic configuration for the logging system. + See details at https://docs.python.org/3/library/logging.html#logging.basicConfig + + :param filename: str or None + The path to save the logs. + :param format: the logging format + :param level: int + :return: Logger + Logger object. + """ + if level is None: + level = C.logging_level + + if format is None: + format = C.logging_config["formatters"]["logger_format"]["format"] + + logging.basicConfig(filename=filename, format=format, level=level) + + class LogFilter(logging.Filter): def __init__(self, param=None): self.param = param diff --git a/qlib/workflow/recorder.py b/qlib/workflow/recorder.py index 31077176d..519b69710 100644 --- a/qlib/workflow/recorder.py +++ b/qlib/workflow/recorder.py @@ -11,7 +11,7 @@ from ..log import get_module_logger logger = get_module_logger("workflow", "INFO") -class Recorder(object): +class Recorder: """ This is the `Recorder` class for logging the experiments. The API is designed similar to mlflow. (The link: https://mlflow.org/docs/latest/python_api/mlflow.html) @@ -201,7 +201,7 @@ class MLflowRecorder(Recorder): def __init__(self, experiment_id, uri, name=None, mlflow_run=None): super(MLflowRecorder, self).__init__(experiment_id, name) self._uri = uri - self.artifact_uri = None + self._artifact_uri = None self.client = mlflow.tracking.MlflowClient(tracking_uri=self._uri) # construct from mlflow run if mlflow_run is not None: @@ -220,14 +220,45 @@ class MLflowRecorder(Recorder): else None ) + def __repr__(self): + name = self.__class__.__name__ + space_length = len(name) + 1 + return "{name}(info={info},\n{space}uri={uri},\n{space}artifact_uri={artifact_uri},\n{space}client={client})".format( + name=name, + space=" " * space_length, + info=self.info, + uri=self.uri, + artifact_uri=self.artifact_uri, + client=self.client, + ) + + @property + def uri(self): + return self._uri + + @property + def artifact_uri(self): + return self._artifact_uri + + @property + def root_uri(self): + start_str = "file:" + if self.artifact_uri is not None: + xpath = self.artifact_uri.strip(start_str) + return (Path(xpath) / "..").resolve() + else: + raise Exception( + "Please make sure the recorder has been created and started properly before getting artifact uri." + ) + def start_run(self): # set the tracking uri - mlflow.set_tracking_uri(self._uri) + mlflow.set_tracking_uri(self.uri) # start the run run = mlflow.start_run(self.id, self.experiment_id, self.name) # save the run id and artifact_uri self.id = run.info.run_id - self.artifact_uri = run.info.artifact_uri + self._artifact_uri = run.info.artifact_uri self.start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.status = Recorder.STATUS_R logger.info(f"Recorder {self.id} starts running under Experiment {self.experiment_id} ...") @@ -247,7 +278,7 @@ class MLflowRecorder(Recorder): self.status = status def save_objects(self, local_path=None, artifact_path=None, **kwargs): - assert self._uri is not None, "Please start the experiment and recorder first before using recorder directly." + assert self.uri is not None, "Please start the experiment and recorder first before using recorder directly." if local_path is not None: self.client.log_artifacts(self.id, local_path, artifact_path) else: @@ -259,7 +290,7 @@ class MLflowRecorder(Recorder): shutil.rmtree(temp_dir) def load_object(self, name): - assert self._uri is not None, "Please start the experiment and recorder first before using recorder directly." + assert self.uri is not None, "Please start the experiment and recorder first before using recorder directly." path = self.client.download_artifacts(self.id, name) with Path(path).open("rb") as f: return pickle.load(f) @@ -289,7 +320,7 @@ class MLflowRecorder(Recorder): ) def list_artifacts(self, artifact_path=None): - assert self._uri is not None, "Please start the experiment and recorder first before using recorder directly." + assert self.uri is not None, "Please start the experiment and recorder first before using recorder directly." artifacts = self.client.list_artifacts(self.id, artifact_path) return [art.path for art in artifacts]