DDG-DA paper code (#743)

* Merge data selection to main

* Update trainer for reweighter

* Typos fixed.

* update data selection interface

* successfully run exp after refactor some interface

* data selection share handler &  trainer

* fix meta model time series bug

* fix online workflow set_uri bug

* fix set_uri bug

* updawte ds docs and delay trainer bug

* docs

* resume reweighter

* add reweighting result

* fix qlib model import

* make recorder more friendly

* fix experiment workflow bug

* commit for merging master incase of conflictions

* Successful run DDG-DA with a single command

* remove unused code

* asdd more docs

* Update README.md

* Update & fix some bugs.

* Update configuration & remove debug functions

* Update README.md

* Modfify horizon from code rather than yaml

* Update performance in README.md

* fix part comments

* Remove unfinished TCTS.

* Fix some details.

* Update meta docs

* Update README.md of the benchmarks_dynamic

* Update README.md files

* Add README.md to the rolling_benchmark baseline.

* Refine the docs and link

* Rename README.md in benchmarks_dynamic.

* Remove comments.

* auto download data

Co-authored-by: wendili-cs <wendili.academic@qq.com>
Co-authored-by: demon143 <785696300@qq.com>

qlib/contrib/model/gbdt.py

@@ -4,59 +4,73 @@
 import numpy as np
 import pandas as pd
 import lightgbm as lgb
-from typing import Text, Union
+from typing import List, Text, Tuple, Union
 from ...model.base import ModelFT
 from ...data.dataset import DatasetH
 from ...data.dataset.handler import DataHandlerLP
 from ...model.interpret.base import LightGBMFInt
+from ...data.dataset.weight import Reweighter
 
 
 class LGBModel(ModelFT, LightGBMFInt):
     """LightGBM Model"""
 
-    def __init__(self, loss="mse", early_stopping_rounds=50, **kwargs):
+    def __init__(self, loss="mse", early_stopping_rounds=50, num_boost_round=1000, **kwargs):
         if loss not in {"mse", "binary"}:
             raise NotImplementedError
         self.params = {"objective": loss, "verbosity": -1}
         self.params.update(kwargs)
         self.early_stopping_rounds = early_stopping_rounds
+        self.num_boost_round = num_boost_round
         self.model = None
 
-    def _prepare_data(self, dataset: DatasetH):
-        df_train, df_valid = dataset.prepare(
-            ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
-        )
-        if df_train.empty or df_valid.empty:
-            raise ValueError("Empty data from dataset, please check your dataset config.")
-        x_train, y_train = df_train["feature"], df_train["label"]
-        x_valid, y_valid = df_valid["feature"], df_valid["label"]
+    def _prepare_data(self, dataset: DatasetH, reweighter=None) -> List[Tuple[lgb.Dataset, str]]:
+        """
+        The motivation of current version is to make validation optional
+        - train segment is necessary;
+        """
+        ds_l = []
+        assert "train" in dataset.segments
+        for key in ["train", "valid"]:
+            if key in dataset.segments:
+                df = dataset.prepare(key, col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
+                if df.empty:
+                    raise ValueError("Empty data from dataset, please check your dataset config.")
+                x, y = df["feature"], df["label"]
 
-        # Lightgbm need 1D array as its label
-        if y_train.values.ndim == 2 and y_train.values.shape[1] == 1:
-            y_train, y_valid = np.squeeze(y_train.values), np.squeeze(y_valid.values)
-        else:
-            raise ValueError("LightGBM doesn't support multi-label training")
+                # Lightgbm need 1D array as its label
+                if y.values.ndim == 2 and y.values.shape[1] == 1:
+                    y = np.squeeze(y.values)
+                else:
+                    raise ValueError("LightGBM doesn't support multi-label training")
 
-        dtrain = lgb.Dataset(x_train, label=y_train)
-        dvalid = lgb.Dataset(x_valid, label=y_valid)
-        return dtrain, dvalid
+                if reweighter is None:
+                    w = None
+                elif isinstance(reweighter, Reweighter):
+                    w = reweighter.reweight(df)
+                else:
+                    raise ValueError("Unsupported reweighter type.")
+                ds_l.append((lgb.Dataset(x.values, label=y, weight=w), key))
+        return ds_l
 
     def fit(
         self,
         dataset: DatasetH,
-        num_boost_round=1000,
+        num_boost_round=None,
         early_stopping_rounds=None,
         verbose_eval=20,
         evals_result=dict(),
+        reweighter=None,
         **kwargs
     ):
-        dtrain, dvalid = self._prepare_data(dataset)
+        ds_l = self._prepare_data(dataset, reweighter)
+        ds, names = list(zip(*ds_l))
         self.model = lgb.train(
             self.params,
-            dtrain,
-            num_boost_round=num_boost_round,
-            valid_sets=[dtrain, dvalid],
-            valid_names=["train", "valid"],
+            ds[0],  # training dataset
+            num_boost_round=self.num_boost_round if num_boost_round is None else num_boost_round,
+            valid_sets=ds,
+            valid_names=names,
             early_stopping_rounds=(
                 self.early_stopping_rounds if early_stopping_rounds is None else early_stopping_rounds
             ),
@@ -64,8 +78,8 @@ class LGBModel(ModelFT, LightGBMFInt):
             evals_result=evals_result,
             **kwargs
         )
-        evals_result["train"] = list(evals_result["train"].values())[0]
-        evals_result["valid"] = list(evals_result["valid"].values())[0]
+        for k in names:
+            evals_result[k] = list(evals_result[k].values())[0]
 
     def predict(self, dataset: DatasetH, segment: Union[Text, slice] = "test"):
         if self.model is None:
@@ -73,7 +87,7 @@ class LGBModel(ModelFT, LightGBMFInt):
         x_test = dataset.prepare(segment, col_set="feature", data_key=DataHandlerLP.DK_I)
         return pd.Series(self.model.predict(x_test.values), index=x_test.index)
 
-    def finetune(self, dataset: DatasetH, num_boost_round=10, verbose_eval=20):
+    def finetune(self, dataset: DatasetH, num_boost_round=10, verbose_eval=20, reweighter=None):
         """
         finetune model
 
@@ -87,7 +101,7 @@ class LGBModel(ModelFT, LightGBMFInt):
             verbose level
         """
         # Based on existing model and finetune by train more rounds
-        dtrain, _ = self._prepare_data(dataset)
+        dtrain, _ = self._prepare_data(dataset, reweighter)
         if dtrain.empty:
             raise ValueError("Empty data from dataset, please check your dataset config.")
         self.model = lgb.train(