Add set_log_basic_config function support re-directing log stream

qlib/config.py

@@ -105,7 +105,7 @@ _default_config = {
     "redis_port": 6379,
     "redis_task_db": 1,
     # This value can be reset via qlib.init
-    "logging_level": "INFO",
+    "logging_level": logging.INFO,
     # Global configuration of qlib log
     # logging_level can control the logging level more finely
     "logging_config": {
@@ -124,12 +124,12 @@ _default_config = {
         "handlers": {
             "console": {
                 "class": "logging.StreamHandler",
-                "level": "DEBUG",
+                "level": logging.DEBUG,
                 "formatter": "logger_format",
                 "filters": ["field_not_found"],
             }
         },
-        "loggers": {"qlib": {"level": "DEBUG", "handlers": ["console"]}},
+        "loggers": {"qlib": {"level": logging.DEBUG, "handlers": ["console"]}},
     },
     # Defatult config for experiment manager
     "exp_manager": {
@@ -185,7 +185,7 @@ MODE_CONF = {
         # The nfs should be auto-mounted by qlib on other
         # serversS(such as PAI) [auto_mount:True]
         "timeout": 100,
-        "logging_level": "INFO",
+        "logging_level": logging.INFO,
         "region": REG_CN,
         ## Custom Operator
         "custom_ops": [],

qlib/log.py

@@ -3,8 +3,7 @@
 
 
 import logging
-import logging.handlers
+from typing import Optional, Text, Dict, Any
-import os
 import re
 from logging import config as logging_config
 from time import time
@@ -13,16 +12,13 @@ from contextlib import contextmanager
 from .config import C
 
 
-def get_module_logger(module_name, level=None):
+def get_module_logger(module_name, level: Optional[int] = None):
     """
     Get a logger for a specific module.
 
     :param module_name: str
         Logic module name.
     :param level: int
-    :param sh_level: int
-        Stream handler log level.
-    :param log_format: str
     :return: Logger
         Logger object.
     """
@@ -103,7 +99,7 @@ class TimeInspector:
         cls.log_cost_time(info=f"{name} Done")
 
 
-def set_log_with_config(log_config: dict):
+def set_log_with_config(log_config: Dict[Text, Any]):
     """set log with config
 
     :param log_config:
@@ -112,6 +108,27 @@ def set_log_with_config(log_config: dict):
     logging_config.dictConfig(log_config)
 
 
+def set_log_basic_config(filename: Optional[Text] = None, format: Optional[Text] = None, level: Optional[int] = None):
+    """
+    Set the basic configuration for the logging system.
+    See details at https://docs.python.org/3/library/logging.html#logging.basicConfig
+
+    :param filename: str or None
+        The path to save the logs.
+    :param format: the logging format
+    :param level: int
+    :return: Logger
+        Logger object.
+    """
+    if level is None:
+        level = C.logging_level
+
+    if format is None:
+        format = C.logging_config["formatters"]["logger_format"]["format"]
+
+    logging.basicConfig(filename=filename, format=format, level=level)
+
+
 class LogFilter(logging.Filter):
     def __init__(self, param=None):
         self.param = param

qlib/workflow/recorder.py

@@ -11,7 +11,7 @@ from ..log import get_module_logger
 logger = get_module_logger("workflow", "INFO")
 
 
-class Recorder(object):
+class Recorder:
     """
     This is the `Recorder` class for logging the experiments. The API is designed similar to mlflow.
     (The link: https://mlflow.org/docs/latest/python_api/mlflow.html)
@@ -201,7 +201,7 @@ class MLflowRecorder(Recorder):
     def __init__(self, experiment_id, uri, name=None, mlflow_run=None):
         super(MLflowRecorder, self).__init__(experiment_id, name)
         self._uri = uri
-        self.artifact_uri = None
+        self._artifact_uri = None
         self.client = mlflow.tracking.MlflowClient(tracking_uri=self._uri)
         # construct from mlflow run
         if mlflow_run is not None:
@@ -220,14 +220,45 @@ class MLflowRecorder(Recorder):
                 else None
             )
 
+    def __repr__(self):
+        name = self.__class__.__name__
+        space_length = len(name) + 1
+        return "{name}(info={info},\n{space}uri={uri},\n{space}artifact_uri={artifact_uri},\n{space}client={client})".format(
+            name=name,
+            space=" " * space_length,
+            info=self.info,
+            uri=self.uri,
+            artifact_uri=self.artifact_uri,
+            client=self.client,
+        )
+
+    @property
+    def uri(self):
+        return self._uri
+
+    @property
+    def artifact_uri(self):
+        return self._artifact_uri
+
+    @property
+    def root_uri(self):
+        start_str = "file:"
+        if self.artifact_uri is not None:
+            xpath = self.artifact_uri.strip(start_str)
+            return (Path(xpath) / "..").resolve()
+        else:
+            raise Exception(
+                "Please make sure the recorder has been created and started properly before getting artifact uri."
+            )
+
     def start_run(self):
         # set the tracking uri
-        mlflow.set_tracking_uri(self._uri)
+        mlflow.set_tracking_uri(self.uri)
         # start the run
         run = mlflow.start_run(self.id, self.experiment_id, self.name)
         # save the run id and artifact_uri
         self.id = run.info.run_id
-        self.artifact_uri = run.info.artifact_uri
+        self._artifact_uri = run.info.artifact_uri
         self.start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         self.status = Recorder.STATUS_R
         logger.info(f"Recorder {self.id} starts running under Experiment {self.experiment_id} ...")
@@ -247,7 +278,7 @@ class MLflowRecorder(Recorder):
             self.status = status
 
     def save_objects(self, local_path=None, artifact_path=None, **kwargs):
-        assert self._uri is not None, "Please start the experiment and recorder first before using recorder directly."
+        assert self.uri is not None, "Please start the experiment and recorder first before using recorder directly."
         if local_path is not None:
             self.client.log_artifacts(self.id, local_path, artifact_path)
         else:
@@ -259,7 +290,7 @@ class MLflowRecorder(Recorder):
             shutil.rmtree(temp_dir)
 
     def load_object(self, name):
-        assert self._uri is not None, "Please start the experiment and recorder first before using recorder directly."
+        assert self.uri is not None, "Please start the experiment and recorder first before using recorder directly."
         path = self.client.download_artifacts(self.id, name)
         with Path(path).open("rb") as f:
             return pickle.load(f)
@@ -289,7 +320,7 @@ class MLflowRecorder(Recorder):
             )
 
     def list_artifacts(self, artifact_path=None):
-        assert self._uri is not None, "Please start the experiment and recorder first before using recorder directly."
+        assert self.uri is not None, "Please start the experiment and recorder first before using recorder directly."
         artifacts = self.client.list_artifacts(self.id, artifact_path)
         return [art.path for art in artifacts]