Merge

examples/benchmarks/CatBoost/README.md

@@ -0,0 +1,3 @@
+# CatBoost
+* Code: [https://github.com/catboost/catboost](https://github.com/catboost/catboost)
+* Paper: CatBoost: unbiased boosting with categorical features. [https://proceedings.neurips.cc/paper/2018/file/14491b756b3a51daac41c24863285549-Paper.pdf](https://proceedings.neurips.cc/paper/2018/file/14491b756b3a51daac41c24863285549-Paper.pdf).

examples/benchmarks/GATs/workflow_config_gats.yaml

@@ -37,9 +37,10 @@ task:
             lr: 1e-3
             early_stop: 20
             batch_size: 800
-            metric: IC
+            metric: loss
             loss: mse
-            base_model: GRU
+            base_model: LSTM
+            with_pretrain: True
             seed: 0
             GPU: 0
     dataset:

examples/benchmarks/LightGBM/README.md

@@ -0,0 +1,4 @@
+# LightGBM
+* Code: [https://github.com/microsoft/LightGBM](https://github.com/microsoft/LightGBM)
+* Paper: LightGBM: A Highly Efficient Gradient Boosting
+Decision Tree. [https://proceedings.neurips.cc/paper/2017/file/6449f44a102fde848669bdd9eb6b76fa-Paper.pdf](https://proceedings.neurips.cc/paper/2017/file/6449f44a102fde848669bdd9eb6b76fa-Paper.pdf).

examples/benchmarks/XGBoost/README.md

@@ -0,0 +1,3 @@
+# XGBoost
+* Code: [https://github.com/dmlc/xgboost](https://github.com/dmlc/xgboost)
+* Paper: XGBoost: A Scalable Tree Boosting System. [https://dl.acm.org/doi/pdf/10.1145/2939672.2939785](https://dl.acm.org/doi/pdf/10.1145/2939672.2939785).

examples/workflow_by_code_gats.py

@@ -70,9 +70,10 @@ if __name__ == "__main__":
                 "lr": 1e-3,
                 "early_stop": 20,
                 "batch_size": 800,
-                "metric": "IC",
+                "metric": "loss",
                 "loss": "mse",
-                "base_model": "GRU",
+                "base_model": "LSTM",
+                "with_pretrain": True,
                 "seed": 0,
                 "GPU": 0,
             },

qlib/contrib/model/pytorch_gats.py

@@ -55,6 +55,7 @@ class GAT(Model):
         early_stop=20,
         loss="mse",
         base_model="GRU",
+        with_pretrain=True,
         optimizer="adam",
         GPU="0",
         seed=0,
@@ -77,6 +78,7 @@ class GAT(Model):
         self.optimizer = optimizer.lower()
         self.loss = loss
         self.base_model = base_model
+        self.with_pretrain = with_pretrain
         self.visible_GPU = GPU
         self.use_gpu = torch.cuda.is_available()
         self.seed = seed
@@ -95,6 +97,7 @@ class GAT(Model):
             "\noptimizer : {}"
             "\nloss_type : {}"
             "\nbase_model : {}"
+            "\nwith_pretrain : {}"
             "\nvisible_GPU : {}"
             "\nuse_GPU : {}"
             "\nseed : {}".format(
@@ -110,6 +113,7 @@ class GAT(Model):
                 optimizer.lower(),
                 loss,
                 base_model,
+                with_pretrain,
                 GPU,
                 self.use_gpu,
                 seed,
@@ -256,6 +260,23 @@ class GAT(Model):
         evals_result["train"] = []
         evals_result["valid"] = []
 
+        # load pretrained base_model
+        if self.with_pretrain:
+            self.logger.info("Loading pretrained model...")
+            if self.base_model == "LSTM":
+                from ...contrib.model.pytorch_lstm import LSTMModel
+                pretrained_model = LSTMModel()
+                pretrained_model.load_state_dict(torch.load('benchmarks/LSTM/model_lstm_csi300.pkl'))
+            elif self.base_model == "GRU":
+                from ...contrib.model.pytorch_gru import GRUModel
+                pretrained_model = GRUModel()
+                pretrained_model.load_state_dict(torch.load('benchmarks/GRU/model_gru_csi300.pkl'))
+            model_dict = self.GAT_model.state_dict()
+            pretrained_dict = {k: v for k, v in pretrained_model.state_dict().items() if k in model_dict}
+            model_dict.update(pretrained_dict)
+            self.GAT_model.load_state_dict(model_dict)
+            self.logger.info("Loading pretrained model Done...")
+
         # train
         self.logger.info("training...")
         self._fitted = True

qlib/contrib/strategy/strategy.py

@@ -46,7 +46,6 @@ class BaseStrategy:
 
     def update(self, score_series, pred_date, trade_date):
         """User can use this method to update strategy state each trade date.
-
         Parameters
         -----------
         score_series : pd.Series
@@ -191,7 +190,18 @@ class WeightStrategyBase(BaseStrategy, AdjustTimer):
 
 
 class TopkDropoutStrategy(BaseStrategy, ListAdjustTimer):
-    def __init__(self, topk, n_drop, method="bottom", risk_degree=0.95, thresh=1, hold_thresh=1, **kwargs):
+    def __init__(
+        self,
+        topk,
+        n_drop,
+        method_sell="bottom",
+        method_buy="top",
+        risk_degree=0.95,
+        thresh=1,
+        hold_thresh=1,
+        only_tradable=False,
+        **kwargs,
+    ):
         """
         Parameters
         -----------
@@ -199,8 +209,10 @@ class TopkDropoutStrategy(BaseStrategy, ListAdjustTimer):
             The number of stocks in the portfolio
         n_drop : int
             number of stocks to be replaced in each trading date
-        method : str
+        method_sell : str
-            dropout method, random/bottom
+            dropout method_sell, random/bottom
+        method_buy : str
+            dropout method_buy, random/top
         risk_degree : float
             position percentage of total value
         thresh : int
@@ -208,12 +220,19 @@ class TopkDropoutStrategy(BaseStrategy, ListAdjustTimer):
         hold_thresh : int
             minimum holding days
             before sell stock , will check current.get_stock_count(order.stock_id) >= self.thresh
+        only_tradable : bool
+            will the strategy only consider the tradable stock when buying and selling.
+            if only_tradable:
+                strategy will make buy sell decision without checking the tradable state of the stock
+            else:
+                strategy will make decision with the tradable state of the stock info and avoid buy and sell them
         """
         super(TopkDropoutStrategy, self).__init__()
         ListAdjustTimer.__init__(self, kwargs.get("adjust_dates", None))
         self.topk = topk
         self.n_drop = n_drop
-        self.method = method
+        self.method_sell = method_sell
+        self.method_buy = method_buy
         self.risk_degree = risk_degree
         self.thresh = thresh
         # self.stock_count['code'] will be the days the stock has been hold
@@ -221,6 +240,7 @@ class TopkDropoutStrategy(BaseStrategy, ListAdjustTimer):
         self.stock_count = {}
 
         self.hold_thresh = hold_thresh
+        self.only_tradable = only_tradable
 
     def get_risk_degree(self, date):
         """get_risk_degree
@@ -249,24 +269,85 @@ class TopkDropoutStrategy(BaseStrategy, ListAdjustTimer):
         """
         if not self.is_adjust(trade_date):
             return []
+
+        if self.only_tradable:
+            # If The strategy only consider tradable stock when make decision
+            # It needs following actions to filter stocks
+            def get_first_n(l, n, reverse=False):
+                cur_n = 0
+                res = []
+                for si in reversed(l) if reverse else l:
+                    if trade_exchange.is_stock_tradable(stock_id=si, trade_date=trade_date):
+                        res.append(si)
+                        cur_n += 1
+                        if cur_n >= n:
+                            break
+                return res[::-1] if reverse else res
+
+            def get_last_n(l, n):
+                return get_first_n(l, n, reverse=True)
+
+            def filter_stock(l):
+                return [si for si in l if trade_exchange.is_stock_tradable(stock_id=si, trade_date=trade_date)]
+
+        else:
+            # Otherwise, the stock will make decision with out the stock tradable info
+            def get_first_n(l, n):
+                return list(l)[:n]
+
+            def get_last_n(l, n):
+                return list(l)[-n:]
+
+            def filter_stock(l):
+                return l
+
         current_temp = copy.deepcopy(current)
         # generate order list for this adjust date
         sell_order_list = []
         buy_order_list = []
         # load score
+        cash = current_temp.get_cash()
         current_stock_list = current_temp.get_stock_list()
+        # last position (sorted by score)
         last = score_series.reindex(current_stock_list).sort_values(ascending=False).index
-        today = (
+        # The new stocks today want to buy **at most**
-            score_series[~score_series.index.isin(last)]
+        if self.method_buy == "top":
-            .sort_values(ascending=False)
+            today = get_first_n(
-            .index[: self.n_drop + self.topk - len(last)]
+                score_series[~score_series.index.isin(last)].sort_values(ascending=False).index,
-        )
+                self.n_drop + self.topk - len(last),
-        comb = score_series.reindex(last.union(today)).sort_values(ascending=False).index
+            )
-        if self.method == "bottom":
+        elif self.method_buy == "random":
-            sell = last[last.isin(comb[-self.n_drop :])]
+            topk_candi = get_first_n(score_series.sort_values(ascending=False).index, self.topk)
-        elif self.method == "random":
+            candi = list(filter(lambda x: x not in last, topk_candi))
-            sell = pd.Index(np.random.choice(last, self.n_drop) if len(last) else [])
+            n = self.n_drop + self.topk - len(last)
+            try:
+                today = np.random.choice(candi, n, replace=False)
+            except ValueError:
+                today = candi
+        else:
+            raise NotImplementedError(f"This type of input is not supported")
+        # combine(new stocks + last stocks),  we will drop stocks from this list
+        # In case of dropping higher score stock and buying lower score stock.
+        comb = score_series.reindex(last.union(pd.Index(today))).sort_values(ascending=False).index
+
+        # Get the stock list we really want to sell (After filtering the case that we sell high and buy low)
+        if self.method_sell == "bottom":
+            sell = last[last.isin(get_last_n(comb, self.n_drop))]
+        elif self.method_sell == "random":
+            candi = filter_stock(last)
+            try:
+                sell = pd.Index(np.random.choice(candi, self.n_drop, replace=False) if len(last) else [])
+            except ValueError:  #  No enough candidates
+                sell = candi
+        else:
+            raise NotImplementedError(f"This type of input is not supported")
+
+        # Get the stock list we really want to buy
         buy = today[: len(sell) + self.topk - len(last)]
+
+        # buy singal: if a stock falls into topk, it appear in the buy_sinal
+        buy_signal = score_series.sort_values(ascending=False).iloc[: self.topk].index
+
         for code in current_stock_list:
             if not trade_exchange.is_stock_tradable(stock_id=code, trade_date=trade_date):
                 continue
@@ -290,12 +371,14 @@ class TopkDropoutStrategy(BaseStrategy, ListAdjustTimer):
                 if trade_exchange.check_order(sell_order):
                     sell_order_list.append(sell_order)
                     trade_val, trade_cost, trade_price = trade_exchange.deal_order(sell_order, position=current_temp)
+                    # update cash
+                    cash += trade_val - trade_cost
                     # sold
                     del self.stock_count[code]
                 else:
                     # no buy signal, but the stock is kept
                     self.stock_count[code] += 1
-            elif code in buy:
+            elif code in buy_signal:
                 # NOTE: This is different from the original version
                 # get new buy signal
                 # Only the stock fall in to topk will produce buy signal
@@ -305,7 +388,7 @@ class TopkDropoutStrategy(BaseStrategy, ListAdjustTimer):
         # buy new stock
         # note the current has been changed
         current_stock_list = current_temp.get_stock_list()
-        value = current_temp.get_cash() * self.risk_degree / len(buy) if len(buy) > 0 else 0
+        value = cash * self.risk_degree / len(buy) if len(buy) > 0 else 0
 
         # open_cost should be considered in the real trading environment, while the backtest in evaluate.py does not consider it
         # as the aim of demo is to accomplish same strategy as evaluate.py, so comment out this line

scripts/README.md

@@ -43,6 +43,8 @@ python get_data.py qlib_data --help
 
 ### US data
 
+> Need to download data first: [Downlaod US Data](#Downlaod-US-Data)
+
 ```python
 import qlib
 from qlib.config import REG_US
@@ -52,6 +54,8 @@ qlib.init(provider_uri=provider_uri, region=REG_US)
 
 ### CN data
 
+> Need to download data first: [Download CN Data](#Download-CN-Data)
+
 ```python
 import qlib
 from qlib.config import REG_CN

scripts/dump_bin.py

@@ -140,7 +140,7 @@ class DumpDataBase:
 
     def _get_source_data(self, file_path: Path) -> pd.DataFrame:
         df = pd.read_csv(str(file_path.resolve()), low_memory=False)
-        df[self.date_field_name] = df[self.date_field_name].astype(np.datetime64)
+        df[self.date_field_name] = df[self.date_field_name].astype(str).astype(np.datetime64)
         # df.drop_duplicates([self.date_field_name], inplace=True)
         return df
 
@@ -339,10 +339,10 @@ class DumpDataFix(DumpDataAll):
     def dump(self):
         self._calendars_list = self._read_calendars(self._calendars_dir.joinpath(f"{self.freq}.txt"))
         # noinspection PyAttributeOutsideInit
-        self._old_instruments = self._read_instruments(
+        self._old_instruments = (
-            self._instruments_dir.joinpath(self.INSTRUMENTS_FILE_NAME)
+            self._read_instruments(self._instruments_dir.joinpath(self.INSTRUMENTS_FILE_NAME))
-        ).to_dict(
+            .set_index([self.symbol_field_name])
-            orient="index"
+            .to_dict(orient="index")
         )  # type: dict
         self._dump_instruments()
         self._dump_features()