refactor qlib conf&init. Fix test bug

qlib/__init__.py

@@ -10,6 +10,7 @@ import logging
 import re
 import subprocess
 import platform
+import yaml
 from pathlib import Path
 
 from .utils import can_use_cache
@@ -17,15 +18,13 @@ from .utils import can_use_cache
 
 # init qlib
 def init(default_conf="client", **kwargs):
-    from .config import (
+    from .config import C, REG_CN, REG_US, QlibConfig
-        C,
-        _default_client_config,
-        _default_server_config,
-        _default_region_config,
-        REG_CN,
-    )
     from .data.data import register_all_wrappers
     from .log import get_module_logger, set_log_with_config
+    from .data.cache import H
+
+    C.reset()
+    H.clear()
 
     _logging_config = C.logging_config
     if "logging_config" in kwargs:
@@ -37,24 +36,17 @@ def init(default_conf="client", **kwargs):
 
     LOG = get_module_logger("Initialization", level=logging.INFO)
     LOG.info(f"default_conf: {default_conf}.")
-    if default_conf == "server":
+
-        base_config = copy.deepcopy(_default_server_config)
+    C.set_mode(default_conf)
-    elif default_conf == "client":
-        base_config = copy.deepcopy(_default_client_config)
-    else:
-        raise ValueError("Unknown system type")
-    if base_config:
-        base_config.update(_default_region_config[kwargs.get("region", REG_CN)])
-        for k, v in base_config.items():
-            C[k] = v
 
     for k, v in kwargs.items():
         C[k] = v
         if k not in C:
             LOG.warning("Unrecognized config %s" % k)
 
-    if default_conf == "client":
+    C.set_region(kwargs.get('region', REG_CN))
-        C["mount_path"] = str(Path(C["mount_path"]).expanduser().resolve())
+    C.resolve_path()
+
     if not (C["expression_cache"] is None and C["dataset_cache"] is None):
         # check redis
         if not can_use_cache():
@@ -65,8 +57,7 @@ def init(default_conf="client", **kwargs):
             C["dataset_cache"] = None
 
     # check path if server/local
-    if re.match("^[^/ ]+:.+", C["provider_uri"]) is None:
+    if C.get_uri_type() == QlibConfig.LOCAL_URI:
-        C["provider_uri"] = str(Path(C["provider_uri"]).expanduser().resolve())
         if not os.path.exists(C["provider_uri"]):
             if C["auto_mount"]:
                 LOG.error(
@@ -76,7 +67,26 @@ def init(default_conf="client", **kwargs):
                 )
             else:
                 LOG.warning("auto_path is False, please make sure {} is mounted".format(C["mount_path"]))
+    elif C.get_uri_type() == QlibConfig.NFS_URI:
+        _mount_nfs_uri(C)
     else:
+        raise NotImplementedError(f"This type of URI is not supported")
+
+    LOG.info("qlib successfully initialized based on %s settings." % default_conf)
+    register_all_wrappers()
+
+    LOG.info(f"data_path={C.get_data_path()}")
+
+    if "flask_server" in C:
+        LOG.info(f"flask_server={C['flask_server']}, flask_port={C['flask_port']}")
+
+
+def _mount_nfs_uri(C):
+    from .log import get_module_logger
+    LOG = get_module_logger("mount nfs", level=logging.INFO)
+
+    # FIXME: the C["provider_uri"] is modified in this function
+    # If it is not modified, we can pass only  provider_uri or mount_path instead of C
     mount_command = "sudo mount.nfs %s %s" % (C["provider_uri"], C["mount_path"])
     # If the provider uri looks like this 172.23.233.89//data/csdesign'
     # It will be a nfs path. The client provider will be used
@@ -170,26 +180,12 @@ def init(default_conf="client", **kwargs):
             else:
                 LOG.warning("{} on {} is already mounted".format(_remote_uri, _mount_path))
 
-    LOG.info("qlib successfully initialized based on %s settings." % default_conf)
-    register_all_wrappers()
-    try:
-        if C["auto_mount"]:
-            LOG.info(f"provider_uri={C['provider_uri']}")
-        else:
-            LOG.info(f"mount_path={C['mount_path']}")
-    except KeyError:
-        LOG.info(f"provider_uri={C['provider_uri']}")
-
-    if "flask_server" in C:
-        LOG.info(f"flask_server={C['flask_server']}, flask_port={C['flask_port']}")
-
 
 def init_from_yaml_conf(conf_path):
     """init_from_yaml_conf
 
     :param conf_path: A path to the qlib config in yml format
     """
-    import yaml
 
     with open(conf_path) as f:
         config = yaml.load(f, Loader=yaml.FullLoader)

qlib/config.py

@@ -1,5 +1,62 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
+"""
+About the configs
+=================
+
+The config will based on _default_config.
+Two modes are supported
+- client
+- server
+
+"""
+
+import copy
+from pathlib import Path
+import re
+
+
+class Config:
+
+    def __init__(self, default_conf):
+        self.__dict__["_default_config"] = default_conf  # avoiding conflictions with __getattr__
+        self.reset()
+
+    def __getitem__(self, key):
+        return self.__dict__["_config"][key]
+
+    def __getattr__(self, attr):
+        try:
+            return self.__dict__["_config"][attr]
+        except KeyError:
+            return AttributeError(f"No such {attr} in self._config")
+
+    def __setitem__(self, key, value):
+        self.__dict__["_config"][key] = value
+
+    def __setattr__(self, attr, value):
+        self.__dict__["_config"][attr] = value
+
+    def __contains__(self, item):
+        return item in self.__dict__["_config"]
+
+    def __getstate__(self):
+        return self.__dict__
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+
+    def __str__(self):
+        return str(self.__dict__["_config"])
+
+    def __repr__(self):
+        return str(self.__dict__["_config"])
+
+    def reset(self):
+        self.__dict__["_config"] = copy.deepcopy(self._default_config)
+
+    def update(self, *args, **kwargs):
+        self.__dict__["_config"].update(*args, **kwargs)
 
 
 # REGION CONST
@@ -70,7 +127,8 @@ _default_config = {
     },
 }
 
-_default_server_config = {
+MODE_CONF = {
+    'server': {
         # data provider config
         "calendar_provider": "LocalCalendarProvider",
         "instrument_provider": "LocalInstrumentProvider",
@@ -88,9 +146,9 @@ _default_server_config = {
         # cache
         "expression_cache": "DiskExpressionCache",
         "dataset_cache": "DiskDatasetCache",
-}
+    },
 
-_default_client_config = {
+    'client': {
         # data provider config
         "calendar_provider": "LocalCalendarProvider",
         "instrument_provider": "LocalInstrumentProvider",
@@ -107,13 +165,14 @@ _default_client_config = {
         "calendar_cache": None,
         # client config
         "kernels": 16,
-    "mount_path": "~/.qlib/qlib_data/cn_data",
+        "mount_path": None,
         "auto_mount": False,  # The nfs is already mounted on our server[auto_mount: False].
         # The nfs should be auto-mounted by qlib on other
         # serversS(such as PAI) [auto_mount:True]
         "timeout": 100,
         "logging_level": "INFO",
         "region": REG_CN,
+    }
 }
 
 
@@ -131,37 +190,43 @@ _default_region_config = {
 }
 
 
-class Config:
+class QlibConfig(Config):
-    def __getitem__(self, key):
+    # URI_TYPE
-        return _default_config[key]
+    LOCAL_URI = 'local'
+    NFS_URI = 'nfs'
 
-    def __getattr__(self, attr):
+    def set_mode(self, mode):
-        try:
+        # raise KeyError
-            return _default_config[attr]
+        self.update(MODE_CONF[mode])
-        except KeyError:
+        # TODO: update region based on kwargs
-            return AttributeError(f"No such {attr} in _default_config")
 
-    def __setitem__(self, key, value):
+    def set_region(self, region):
-        _default_config[key] = value
+        # raise KeyError
+        self.update(_default_region_config[region])
 
-    def __setattr__(self, attr, value):
+    def resolve_path(self):
-        _default_config[attr] = value
+        # resolve path
+        if self["mount_path"] is not None:
+            self["mount_path"]= str(Path(self["mount_path"]).expanduser().resolve())
 
-    def __contains__(self, item):
+        if self.get_uri_type() == QlibConfig.LOCAL_URI:
-        return item in _default_config
+            self["provider_uri"] = str(Path(self["provider_uri"]).expanduser().resolve())
 
-    def __getstate__(self):
+    def get_uri_type(self):
-        return _default_config
+        rm = re.match("^[^/ ]+:.+", self["provider_uri"])
+        if rm is None:
+            return QlibConfig.LOCAL_URI
+        else:
+            return QlibConfig.NFS_URI
 
-    def __setstate__(self, state):
+    def get_data_path(self):
-        _default_config.update(state)
+        if self.get_uri_type() == QlibConfig.LOCAL_URI:
-
+            return self['provider_uri']
-    def __str__(self):
+        elif self.get_uri_type() == QlibConfig.NFS_URI:
-        return str(_default_config)
+            return self['mount_path']
-
+        else:
-    def __repr__(self):
+            raise NotImplementedError(f"This type of uri is not supported")
-        return str(_default_config)
 
 
 # global config
-C = Config()
+C = QlibConfig(_default_config)

qlib/contrib/backtest/profit_attribution.py

@@ -33,7 +33,7 @@ def get_benchmark_weight(
 
     """
     if not path:
-        path = Path(C.mount_path).expanduser() / "raw" / "AIndexMembers" / "weights.csv"
+        path = Path(C.get_data_path()).expanduser() / "raw" / "AIndexMembers" / "weights.csv"
     # TODO: the storage of weights should be implemented in a more elegent way
     # TODO: The benchmark is not consistant with the filename in instruments.
     bench_weight_df = pd.read_csv(path, usecols=["code", "date", "index", "weight"])

qlib/data/cache.py

@@ -388,10 +388,7 @@ class DiskExpressionCache(ExpressionCache):
         self.r = get_redis_connection()
         # remote==True means client is using this module, writing behaviour will not be allowed.
         self.remote = kwargs.get("remote", False)
-        if self.remote:
+        self.expr_cache_path = os.path.join(C.get_data_path(), C.features_cache_dir_name)
-            self.expr_cache_path = os.path.join(C.mount_path, C.features_cache_dir_name)
-        else:
-            self.expr_cache_path = os.path.join(C.provider_uri, C.features_cache_dir_name)
         os.makedirs(self.expr_cache_path, exist_ok=True)
 
     def _uri(self, instrument, field, start_time, end_time, freq):
@@ -562,10 +559,7 @@ class DiskDatasetCache(DatasetCache):
         super(DiskDatasetCache, self).__init__(provider)
         self.r = get_redis_connection()
         self.remote = kwargs.get("remote", False)
-        if self.remote:
+        self.dtst_cache_path = os.path.join(C.get_data_path(), C.dataset_cache_dir_name)
-            self.dtst_cache_path = os.path.join(C.mount_path, C.dataset_cache_dir_name)
-        else:
-            self.dtst_cache_path = os.path.join(C.provider_uri, C.dataset_cache_dir_name)
         os.makedirs(self.dtst_cache_path, exist_ok=True)
 
     @staticmethod
@@ -1003,7 +997,7 @@ class DatasetURICache(DatasetCache):
         # use ClientDatasetProvider
         feature_uri = self._uri(instruments, fields, None, None, freq, disk_cache=disk_cache)
         value, expire = MemCacheExpire.get_cache(H["f"], feature_uri)
-        mnt_feature_uri = os.path.join(C.mount_path, C.dataset_cache_dir_name, feature_uri)
+        mnt_feature_uri = os.path.join(C.get_data_path(), C.dataset_cache_dir_name, feature_uri)
         if value is None or expire or not os.path.exists(mnt_feature_uri):
             df, uri = self.provider.dataset(
                 instruments, fields, start_time, end_time, freq, disk_cache, return_uri=True
@@ -1014,7 +1008,7 @@ class DatasetURICache(DatasetCache):
             # HZ['f'][uri] = df.copy()
             get_module_logger("cache").debug(f"get feature from {C.dataset_provider}")
         else:
-            mnt_feature_uri = os.path.join(C.mount_path, C.dataset_cache_dir_name, feature_uri)
+            mnt_feature_uri = os.path.join(C.get_data_path(), C.dataset_cache_dir_name, feature_uri)
             df = DiskDatasetCache.read_data_from_cache(mnt_feature_uri, start_time, end_time, fields)
             get_module_logger("cache").debug("get feature from uri cache")
 

qlib/data/data.py

@@ -502,10 +502,7 @@ class LocalCalendarProvider(CalendarProvider):
     @property
     def _uri_cal(self):
         """Calendar file uri."""
-        if self.remote:
+        return os.path.join(C.get_data_path(), "calendars", "{}.txt")
-            return os.path.join(C.mount_path, "calendars", "{}.txt")
-        else:
-            return os.path.join(C.provider_uri, "calendars", "{}.txt")
 
     def _load_calendar(self, freq, future):
         """Load original calendar timestamp from file.
@@ -568,7 +565,7 @@ class LocalInstrumentProvider(InstrumentProvider):
     @property
     def _uri_inst(self):
         """Instrument file uri."""
-        return os.path.join(C.provider_uri, "instruments", "{}.txt")
+        return os.path.join(C.get_data_path(), "instruments", "{}.txt")
 
     def _load_instruments(self, market):
         fname = self._uri_inst.format(market)
@@ -637,10 +634,7 @@ class LocalFeatureProvider(FeatureProvider):
     @property
     def _uri_data(self):
         """Static feature file uri."""
-        if self.remote:
+        return os.path.join(C.get_data_path(), "features", "{}", "{}.{}.bin")
-            return os.path.join(C.mount_path, "features", "{}", "{}.{}.bin")
-        else:
-            return os.path.join(C.provider_uri, "features", "{}", "{}.{}.bin")
 
     def feature(self, instrument, field, start_index, end_index, freq):
         # validate
@@ -914,7 +908,7 @@ class ClientDatasetProvider(DatasetProvider):
             get_module_logger("data").debug("get result")
             try:
                 # pre-mound nfs, used for demo
-                mnt_feature_uri = os.path.join(C.mount_path, C.dataset_cache_dir_name, feature_uri)
+                mnt_feature_uri = os.path.join(C.get_data_path(), C.dataset_cache_dir_name, feature_uri)
                 df = DiskDatasetCache.read_data_from_cache(mnt_feature_uri, start_time, end_time, fields)
                 get_module_logger("data").debug("finish slicing data")
                 if return_uri: