DDG-DA paper code (#743)

* Merge data selection to main

* Update trainer for reweighter

* Typos fixed.

* update data selection interface

* successfully run exp after refactor some interface

* data selection share handler &  trainer

* fix meta model time series bug

* fix online workflow set_uri bug

* fix set_uri bug

* updawte ds docs and delay trainer bug

* docs

* resume reweighter

* add reweighting result

* fix qlib model import

* make recorder more friendly

* fix experiment workflow bug

* commit for merging master incase of conflictions

* Successful run DDG-DA with a single command

* remove unused code

* asdd more docs

* Update README.md

* Update & fix some bugs.

* Update configuration & remove debug functions

* Update README.md

* Modfify horizon from code rather than yaml

* Update performance in README.md

* fix part comments

* Remove unfinished TCTS.

* Fix some details.

* Update meta docs

* Update README.md of the benchmarks_dynamic

* Update README.md files

* Add README.md to the rolling_benchmark baseline.

* Refine the docs and link

* Rename README.md in benchmarks_dynamic.

* Remove comments.

* auto download data

Co-authored-by: wendili-cs <wendili.academic@qq.com>
Co-authored-by: demon143 <785696300@qq.com>

qlib/data/dataset/__init__.py

@@ -1,5 +1,5 @@
 from ...utils.serial import Serializable
-from typing import Union, List, Tuple, Dict, Text, Optional
+from typing import Callable, Union, List, Tuple, Dict, Text, Optional
 from ...utils import init_instance_by_config, np_ffill, time_to_slc_point
 from ...log import get_module_logger
 from .handler import DataHandler, DataHandlerLP
@@ -235,6 +235,28 @@ class DatasetH(Dataset):
         else:
             raise NotImplementedError(f"This type of input is not supported")
 
+    # helper functions
+    @staticmethod
+    def get_min_time(segments):
+        return DatasetH._get_extrema(segments, 0, (lambda a, b: a > b))
+
+    @staticmethod
+    def get_max_time(segments):
+        return DatasetH._get_extrema(segments, 1, (lambda a, b: a < b))
+
+    @staticmethod
+    def _get_extrema(segments, idx: int, cmp: Callable, key_func=pd.Timestamp):
+        """it will act like sort and return the max value or None"""
+        candidate = None
+        for k, seg in segments.items():
+            point = seg[idx]
+            if point is None:
+                # None indicates unbounded, return directly
+                return None
+            elif candidate is None or cmp(key_func(candidate), key_func(point)):
+                candidate = point
+        return candidate
+
 
 class TSDataSampler:
     """

qlib/data/dataset/loader.py

@@ -2,6 +2,8 @@
 # Licensed under the MIT License.
 
 import abc
+import pickle
+from pathlib import Path
 import warnings
 import pandas as pd
 
@@ -10,6 +12,7 @@ from typing import Tuple, Union, List
 from qlib.data import D
 from qlib.utils import load_dataset, init_instance_by_config, time_to_slc_point
 from qlib.log import get_module_logger
+from qlib.utils.serial import Serializable
 
 
 class DataLoader(abc.ABC):
@@ -216,12 +219,14 @@ class QlibDataLoader(DLWParser):
         return df
 
 
-class StaticDataLoader(DataLoader):
+class StaticDataLoader(DataLoader, Serializable):
     """
     DataLoader that supports loading data from file or as provided.
     """
 
-    def __init__(self, config: dict, join="outer"):
+    include_attr = ["_config"]
+
+    def __init__(self, config: Union[dict, str], join="outer"):
         """
         Parameters
         ----------
@@ -230,7 +235,7 @@ class StaticDataLoader(DataLoader):
         join : str
             How to align different dataframes
         """
-        self.config = config
+        self._config = config  # using "_" to avoid confliction with the method `config` of Serializable
         self.join = join
         self._data = None
 
@@ -254,12 +259,16 @@ class StaticDataLoader(DataLoader):
     def _maybe_load_raw_data(self):
         if self._data is not None:
             return
-        self._data = pd.concat(
-            {fields_group: load_dataset(path_or_obj) for fields_group, path_or_obj in self.config.items()},
-            axis=1,
-            join=self.join,
-        )
-        self._data.sort_index(inplace=True)
+        if isinstance(self._config, dict):
+            self._data = pd.concat(
+                {fields_group: load_dataset(path_or_obj) for fields_group, path_or_obj in self._config.items()},
+                axis=1,
+                join=self.join,
+            )
+            self._data.sort_index(inplace=True)
+        elif isinstance(self._config, (str, Path)):
+            with Path(self._config).open("rb") as f:
+                self._data = pickle.load(f)
 
 
 class DataLoaderDH(DataLoader):

qlib/data/dataset/processor.py

@@ -6,6 +6,7 @@ from typing import Union, Text
 import numpy as np
 import pandas as pd
 
+from qlib.utils.data import robust_zscore, zscore
 from ...constant import EPS
 from .utils import fetch_df_by_index
 from ...utils.serial import Serializable
@@ -293,14 +294,22 @@ class RobustZScoreNorm(Processor):
 class CSZScoreNorm(Processor):
     """Cross Sectional ZScore Normalization"""
 
-    def __init__(self, fields_group=None):
+    def __init__(self, fields_group=None, method="zscore"):
         self.fields_group = fields_group
+        if method == "zscore":
+            self.zscore_func = zscore
+        elif method == "robust":
+            self.zscore_func = robust_zscore
+        else:
+            raise NotImplementedError(f"This type of input is not supported")
 
     def __call__(self, df):
         # try not modify original dataframe
-        cols = get_group_columns(df, self.fields_group)
-        df[cols] = df[cols].groupby("datetime").apply(lambda x: (x - x.mean()).div(x.std()))
-
+        if not isinstance(self.fields_group, list):
+            self.fields_group = [self.fields_group]
+        for g in self.fields_group:
+            cols = get_group_columns(df, g)
+            df[cols] = df[cols].groupby("datetime").apply(self.zscore_func)
         return df
 
 

qlib/data/dataset/utils.py

@@ -1,8 +1,13 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
-
+from __future__ import annotations
 import pandas as pd
 from typing import Union, List
+from qlib.utils import init_instance_by_config
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from qlib.data.dataset import DataHandler
 
 
 def get_level_index(df: pd.DataFrame, level=Union[str, int]) -> int:
@@ -111,3 +116,28 @@ def convert_index_format(df: Union[pd.DataFrame, pd.Series], level: str = "datet
     if get_level_index(df, level=level) == 1:
         df = df.swaplevel().sort_index()
     return df
+
+
+def init_task_handler(task: dict) -> Union[DataHandler, None]:
+    """
+    initialize the handler part of the task **inplace**
+
+    Parameters
+    ----------
+    task : dict
+        the task to be handled
+
+    Returns
+    -------
+    Union[DataHandler, None]:
+        returns
+    """
+    # avoid recursive import
+    from .handler import DataHandler
+
+    h_conf = task["dataset"]["kwargs"].get("handler")
+    if h_conf is not None:
+        handler = init_instance_by_config(h_conf, accept_types=DataHandler)
+        task["dataset"]["kwargs"]["handler"] = handler
+
+        return handler

qlib/data/dataset/weight.py

@@ -0,0 +1,34 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import pandas as pd
+import numpy as np
+from typing import Union, List, Tuple
+from ...data.dataset import TSDataSampler
+from ...data.dataset.utils import get_level_index
+from ...utils import lazy_sort_index
+
+
+class Reweighter:
+    def __init__(self, *args, **kwargs):
+        """
+        To initialize the Reweighter, users should provide specific methods to let reweighter do the reweighting (such as sample-wise, rule-based).
+        """
+        raise NotImplementedError()
+
+    def reweight(self, data: object) -> object:
+        """
+        Get weights for data
+
+        Parameters
+        ----------
+        data : object
+            The input data.
+            The first dimension is the index of samples
+
+        Returns
+        -------
+        object:
+            the weights info for the data
+        """
+        raise NotImplementedError(f"This type of input is not supported")