Online serving V11

qlib/utils/__init__.py

@@ -716,23 +716,33 @@ def lazy_sort_index(df: pd.DataFrame, axis=0) -> pd.DataFrame:
         return df.sort_index(axis=axis)
 
 
+FLATTEN_TUPLE = "_FLATTEN_TUPLE"
+
+
 def flatten_dict(d, parent_key="", sep="."):
-    """flatten_dict.
+    """
+    Flatten a nested dict.
+
         >>> flatten_dict({'a': 1, 'c': {'a': 2, 'b': {'x': 5, 'y' : 10}}, 'd': [1, 2, 3]})
         >>> {'a': 1, 'c.a': 2, 'c.b.x': 5, 'd': [1, 2, 3], 'c.b.y': 10}
 
-    Parameters
-    ----------
-    d :
-        d
-    parent_key :
-        parent_key
-    sep :
-        sep
+        >>> flatten_dict({'a': 1, 'c': {'a': 2, 'b': {'x': 5, 'y' : 10}}, 'd': [1, 2, 3]}, sep=FLATTEN_TUPLE)
+        >>> {'a': 1, ('c','a'): 2, ('c','b','x'): 5, 'd': [1, 2, 3], ('c','b','y'): 10}
+
+    Args:
+        d (dict): the dict waiting for flatting
+        parent_key (str, optional): the parent key, will be a prefix in new key. Defaults to "".
+        sep (str, optional): the separator for string connecting. FLATTEN_TUPLE for tuple connecting.
+
+    Returns:
+        dict: flatten dict
     """
     items = []
     for k, v in d.items():
-        new_key = parent_key + sep + str(k) if parent_key else k
+        if sep == FLATTEN_TUPLE:
+            new_key = (parent_key, k) if parent_key else k
+        else:
+            new_key = parent_key + sep + k if parent_key else k
         if isinstance(v, collections.abc.MutableMapping):
             items.extend(flatten_dict(v, new_key, sep=sep).items())
         else:

qlib/utils/serial.py

@@ -16,9 +16,10 @@ class Serializable:
     """
 
     pickle_backend = "pickle"  # another optional value is "dill" which can pickle more things of python.
+    default_dump_all = False # if dump all things
 
     def __init__(self):
-        self._dump_all = False
+        self._dump_all = self.default_dump_all
         self._exclude = []
 
     def __getstate__(self) -> dict:
@@ -77,12 +78,7 @@ class Serializable:
     def to_pickle(self, path: Union[Path, str], dump_all: bool = None, exclude: list = None):
         self.config(dump_all=dump_all, exclude=exclude)
         with Path(path).open("wb") as f:
-            if self.pickle_backend == "pickle":
-                pickle.dump(self, f)
-            elif self.pickle_backend == "dill":
-                dill.dump(self, f)
-            else:
-                raise ValueError("Unknown pickle backend, please use 'pickle' or 'dill'.")
+            self.get_backend().dump(self, f)
 
     @classmethod
     def load(cls, filepath):
@@ -99,13 +95,24 @@ class Serializable:
             Collector: the instance of Collector
         """
         with open(filepath, "rb") as f:
-            if cls.pickle_backend == "pickle":
-                object = pickle.load(f)
-            elif cls.pickle_backend == "dill":
-                object = dill.load(f)
-            else:
-                raise ValueError("Unknown pickle backend, please use 'pickle' or 'dill'.")
+            object = cls.get_backend().load(f)
         if isinstance(object, cls):
             return object
         else:
             raise TypeError(f"The instance of {type(object)} is not a valid `{type(cls)}`!")
+
+    @classmethod
+    def get_backend(cls):
+        """
+        Return the backend of a Serializable class. The value will be "pickle" or "dill".
+
+        Returns:
+            str: The value of "pickle" or "dill"
+        """
+        if cls.pickle_backend == "pickle":
+            return pickle
+        elif cls.pickle_backend == "dill":
+            return dill
+        else:
+            raise ValueError("Unknown pickle backend, please use 'pickle' or 'dill'.")
+