diff --git a/qlib/__init__.py b/qlib/__init__.py index 9f3bc693a..f82848b90 100644 --- a/qlib/__init__.py +++ b/qlib/__init__.py @@ -11,8 +11,6 @@ import logging import platform import subprocess -from .utils import set_config, config_based_on_c - # init qlib def init(default_conf="client", **kwargs): @@ -25,7 +23,7 @@ def init(default_conf="client", **kwargs): # FIXME: this logger ignored the level in config logger = get_module_logger("Initialization", level=logging.INFO) - set_config(C, default_conf, **kwargs) + C.set(default_conf, **kwargs) # check path if server/local if C.get_uri_type() == C.LOCAL_URI: @@ -44,7 +42,7 @@ def init(default_conf="client", **kwargs): if "flask_server" in C: logger.info(f"flask_server={C['flask_server']}, flask_port={C['flask_port']}") - config_based_on_c(C) + C.register() logger.info("qlib successfully initialized based on %s settings." % default_conf) logger.info(f"data_path={C.get_data_path()}") diff --git a/qlib/config.py b/qlib/config.py index eb68a504d..a65d41041 100644 --- a/qlib/config.py +++ b/qlib/config.py @@ -11,11 +11,12 @@ Two modes are supported """ -import copy -from pathlib import Path -import re import os +import re +import copy +import logging import multiprocessing +from pathlib import Path class Config: @@ -212,6 +213,10 @@ class QlibConfig(Config): LOCAL_URI = "local" NFS_URI = "nfs" + def __init__(self, default_conf): + super().__init__(default_conf) + self._registered = False + def set_mode(self, mode): # raise KeyError self.update(MODE_CONF[mode]) @@ -248,6 +253,64 @@ class QlibConfig(Config): else: raise NotImplementedError(f"This type of uri is not supported") + def set(self, default_conf="client", **kwargs): + from .utils import set_log_with_config, get_module_logger, can_use_cache + + self.reset() + + _logging_config = self.logging_config + if "logging_config" in kwargs: + _logging_config = kwargs["logging_config"] + + # set global config + if _logging_config: + set_log_with_config(_logging_config) + + # FIXME: this logger ignored the level in config + logger = get_module_logger("Initialization", level=logging.INFO) + logger.info(f"default_conf: {default_conf}.") + + self.set_mode(default_conf) + self.set_region(kwargs.get("region", self["region"] if "region" in self else REG_CN)) + + for k, v in kwargs.items(): + if k not in self: + logger.warning("Unrecognized config %s" % k) + self[k] = v + + self.resolve_path() + + if not (self["expression_cache"] is None and self["dataset_cache"] is None): + # check redis + if not can_use_cache(): + logger.warning( + f"redis connection failed(host={self['redis_host']} port={self['redis_port']}), cache will not be used!" + ) + self["expression_cache"] = None + self["dataset_cache"] = None + + def register(self): + from .utils import init_instance_by_config + from .data.ops import register_custom_ops + from .data.data import register_all_wrappers + from .workflow import R, QlibRecorder + from .workflow.utils import experiment_exit_handler + + register_custom_ops(self) + register_all_wrappers(self) + # set up QlibRecorder + exp_manager = init_instance_by_config(self["exp_manager"]) + qr = QlibRecorder(exp_manager) + R.register(qr) + # clean up experiment when python program ends + experiment_exit_handler() + + self._registered = True + + @property + def registered(self): + return self._registered + # global config C = QlibConfig(_default_config) diff --git a/qlib/data/data.py b/qlib/data/data.py index cfb6db1dc..d95728199 100644 --- a/qlib/data/data.py +++ b/qlib/data/data.py @@ -25,7 +25,7 @@ from ..log import get_module_logger from ..utils import parse_field, read_bin, hash_args, normalize_cache_fields from .base import Feature from .cache import DiskDatasetCache, DiskExpressionCache -from ..utils import Wrapper, init_instance_by_config, register_wrapper, get_module_by_module_path, config_based_on_c +from ..utils import Wrapper, init_instance_by_config, register_wrapper, get_module_by_module_path class CalendarProvider(abc.ABC): @@ -486,9 +486,9 @@ class DatasetProvider(abc.ABC): """ # FIXME: Windows OS or MacOS using spawn: https://docs.python.org/3.8/library/multiprocessing.html?highlight=spawn#contexts-and-start-methods # NOTE: This place is compatible with windows, windows multi-process is spawn - if getattr(ExpressionD, "_provider", None) is None: + if not C.registered: C.set_conf_from_C(g_config) - config_based_on_c(g_config) + C.register() obj = dict() for field in column_names: diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 9e2e92a2b..2d9d11de2 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -744,55 +744,3 @@ def load_dataset(path_or_obj): elif extension == ".csv": return pd.read_csv(path_or_obj, parse_dates=True, index_col=[0, 1]) raise ValueError(f"unsupported file type `{extension}`") - - -def set_config(config_c, default_conf="client", **kwargs): - - config_c.reset() - - _logging_config = config_c.logging_config - if "logging_config" in kwargs: - _logging_config = kwargs["logging_config"] - - # set global config - if _logging_config: - set_log_with_config(_logging_config) - - # FIXME: this logger ignored the level in config - logger = get_module_logger("Initialization", level=logging.INFO) - logger.info(f"default_conf: {default_conf}.") - - config_c.set_mode(default_conf) - config_c.set_region(kwargs.get("region", config_c["region"] if "region" in config_c else REG_CN)) - - for k, v in kwargs.items(): - if k not in config_c: - logger.warning("Unrecognized config %s" % k) - config_c[k] = v - - config_c.resolve_path() - - if not (config_c["expression_cache"] is None and config_c["dataset_cache"] is None): - # check redis - if not can_use_cache(): - logger.warning( - f"redis connection failed(host={config_c['redis_host']} port={config_c['redis_port']}), cache will not be used!" - ) - config_c["expression_cache"] = None - config_c["dataset_cache"] = None - - -def config_based_on_c(config_c): - from ..data.ops import register_custom_ops - from ..data.data import register_all_wrappers - from ..workflow import R, QlibRecorder - from ..workflow.utils import experiment_exit_handler - - register_custom_ops(config_c) - register_all_wrappers(config_c) - # set up QlibRecorder - exp_manager = init_instance_by_config(config_c["exp_manager"]) - qr = QlibRecorder(exp_manager) - R.register(qr) - # clean up experiment when python program ends - experiment_exit_handler()