diff --git a/examples/benchmarks/ALSTM/workflow_config_alstm.yaml b/examples/benchmarks/ALSTM/workflow_config_alstm.yaml index 66367034d..37dceaacb 100644 --- a/examples/benchmarks/ALSTM/workflow_config_alstm.yaml +++ b/examples/benchmarks/ALSTM/workflow_config_alstm.yaml @@ -54,7 +54,6 @@ task: batch_size: 800 metric: loss loss: mse - seed: 0 GPU: 0 rnn_type: GRU dataset: diff --git a/examples/benchmarks/GATs/workflow_config_gats.yaml b/examples/benchmarks/GATs/workflow_config_gats.yaml index a608a7e49..d4f2c3106 100644 --- a/examples/benchmarks/GATs/workflow_config_gats.yaml +++ b/examples/benchmarks/GATs/workflow_config_gats.yaml @@ -56,7 +56,6 @@ task: base_model: LSTM with_pretrain: True model_path: "benchmarks/LSTM/model_lstm_csi300.pkl" - seed: 0 GPU: 0 dataset: class: DatasetH diff --git a/examples/benchmarks/GRU/workflow_config_gru.yaml b/examples/benchmarks/GRU/workflow_config_gru.yaml index 381581a77..64ae3437c 100644 --- a/examples/benchmarks/GRU/workflow_config_gru.yaml +++ b/examples/benchmarks/GRU/workflow_config_gru.yaml @@ -54,7 +54,6 @@ task: batch_size: 800 metric: loss loss: mse - seed: 0 GPU: 0 dataset: class: DatasetH diff --git a/examples/benchmarks/LSTM/workflow_config_lstm.yaml b/examples/benchmarks/LSTM/workflow_config_lstm.yaml index cb3b2a789..b857d97d4 100644 --- a/examples/benchmarks/LSTM/workflow_config_lstm.yaml +++ b/examples/benchmarks/LSTM/workflow_config_lstm.yaml @@ -54,7 +54,6 @@ task: batch_size: 800 metric: loss loss: mse - seed: 0 GPU: 0 dataset: class: DatasetH diff --git a/examples/benchmarks/SFM/workflow_config_sfm.yaml b/examples/benchmarks/SFM/workflow_config_sfm.yaml index edf176e62..d0a95bfa5 100644 --- a/examples/benchmarks/SFM/workflow_config_sfm.yaml +++ b/examples/benchmarks/SFM/workflow_config_sfm.yaml @@ -58,7 +58,6 @@ task: loss: mse optimizer: adam GPU: 1 - seed: 710 dataset: class: DatasetH module_path: qlib.data.dataset diff --git a/examples/run_all_model.py b/examples/run_all_model.py index 3d8628c4a..6df0ff4bb 100644 --- a/examples/run_all_model.py +++ b/examples/run_all_model.py @@ -152,6 +152,10 @@ def get_all_results(folders) -> dict: result["annualized_return_with_cost"] = list() result["information_ratio_with_cost"] = list() result["max_drawdown_with_cost"] = list() + result["ic"] = list() + result["icir"] = list() + result["rank_ic"] = list() + result["rank_icir"] = list() for recorder_id in recorders: if recorders[recorder_id].status == "FINISHED": recorder = R.get_recorder(recorder_id=recorder_id, experiment_name=fn) @@ -159,19 +163,27 @@ def get_all_results(folders) -> dict: result["annualized_return_with_cost"].append(metrics["excess_return_with_cost.annualized_return"]) result["information_ratio_with_cost"].append(metrics["excess_return_with_cost.information_ratio"]) result["max_drawdown_with_cost"].append(metrics["excess_return_with_cost.max_drawdown"]) + result["ic"].append(metrics["IC"]) + result["icir"].append(metrics["ICIR"]) + result["rank_ic"].append(metrics["Rank IC"]) + result["rank_icir"].append(metrics["Rank ICIR"]) results[fn] = result return results # function to generate and save markdown table def gen_and_save_md_table(metrics): - table = "| Model Name | Annualized Return | Information Ratio | Max Drawdown |\n" - table += "|---|---|---|---|\n" + table = "| Model Name | Annualized Return | Information Ratio | Max Drawdown | IC | ICIR | Rank IC | Rank ICIR |\n" + table += "|---|---|---|---|---|---|---|---|\n" for fn in metrics: ar = metrics[fn]["annualized_return_with_cost"] ir = metrics[fn]["information_ratio_with_cost"] md = metrics[fn]["max_drawdown_with_cost"] - table += f"| {fn} | {ar[0]:9.4f}±{ar[1]:9.2f} | {ir[0]:9.4f}±{ir[1]:9.2f}| {md[0]:9.4f}±{md[1]:9.2f} |\n" + ic = metrics[fn]["ic"] + icir = metrics[fn]["icir"] + ric = metrics[fn]["rank_ic"] + ricir = metrics[fn]["rank_icir"] + table += f"| {fn} | {ar[0]:5.4f}±{ar[1]:2.2f} | {ir[0]:5.4f}±{ir[1]:2.2f}| {md[0]:5.4f}±{md[1]:2.2f} | {ic[0]:5.4f}±{ic[1]:2.2f} | {icir[0]:5.4f}±{icir[1]:2.2f}| {ric[0]:5.4f}±{ric[1]:2.2f} | {ricir[0]:5.4f}±{ricir[1]:2.2f} |\n" pprint(table) with open("table.md", "w") as f: f.write(table) @@ -240,6 +252,7 @@ def run(times=1, models=None, exclude=False): sys.stderr.write("\n") # install qlib sys.stderr.write("Installing qlib...\n") + execute(f"{python_path} -m pip install --upgrade pip") # TODO: FIX ME! execute(f"{python_path} -m pip install --upgrade cython") # TODO: FIX ME! if fn == "TFT": execute( diff --git a/qlib/contrib/model/pytorch_alstm.py b/qlib/contrib/model/pytorch_alstm.py index ba75c70ca..d992ec52f 100644 --- a/qlib/contrib/model/pytorch_alstm.py +++ b/qlib/contrib/model/pytorch_alstm.py @@ -57,7 +57,7 @@ class ALSTM(Model): loss="mse", optimizer="adam", GPU="0", - seed=0, + seed=None, **kwargs ): # Set logger. @@ -113,8 +113,9 @@ class ALSTM(Model): ) ) - np.random.seed(self.seed) - torch.manual_seed(self.seed) + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) self.ALSTM_model = ALSTMModel( d_feat=self.d_feat, diff --git a/qlib/contrib/model/pytorch_gats.py b/qlib/contrib/model/pytorch_gats.py index 2e19c0542..83f2633c5 100644 --- a/qlib/contrib/model/pytorch_gats.py +++ b/qlib/contrib/model/pytorch_gats.py @@ -62,7 +62,7 @@ class GATs(Model): model_path=None, optimizer="adam", GPU="0", - seed=0, + seed=None, **kwargs ): # Set logger. @@ -123,8 +123,11 @@ class GATs(Model): seed, ) ) - np.random.seed(self.seed) - torch.manual_seed(self.seed) + + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) + self.GAT_model = GATModel( d_feat=self.d_feat, hidden_size=self.hidden_size, diff --git a/qlib/contrib/model/pytorch_gru.py b/qlib/contrib/model/pytorch_gru.py index 4ef4e2553..1c24017fc 100755 --- a/qlib/contrib/model/pytorch_gru.py +++ b/qlib/contrib/model/pytorch_gru.py @@ -57,7 +57,7 @@ class GRU(Model): loss="mse", optimizer="adam", GPU="0", - seed=0, + seed=None, **kwargs ): # Set logger. @@ -113,8 +113,9 @@ class GRU(Model): ) ) - np.random.seed(self.seed) - torch.manual_seed(self.seed) + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) self.gru_model = GRUModel( d_feat=self.d_feat, diff --git a/qlib/contrib/model/pytorch_lstm.py b/qlib/contrib/model/pytorch_lstm.py index 59df51e42..81e803e96 100755 --- a/qlib/contrib/model/pytorch_lstm.py +++ b/qlib/contrib/model/pytorch_lstm.py @@ -57,7 +57,7 @@ class LSTM(Model): loss="mse", optimizer="adam", GPU="0", - seed=0, + seed=None, **kwargs ): # Set logger. @@ -113,8 +113,9 @@ class LSTM(Model): ) ) - np.random.seed(self.seed) - torch.manual_seed(self.seed) + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) self.lstm_model = LSTMModel( d_feat=self.d_feat, diff --git a/qlib/contrib/model/pytorch_nn.py b/qlib/contrib/model/pytorch_nn.py index a60227ff0..fac9d4607 100644 --- a/qlib/contrib/model/pytorch_nn.py +++ b/qlib/contrib/model/pytorch_nn.py @@ -61,7 +61,7 @@ class DNNModelPytorch(Model): optimizer="gd", loss="mse", GPU="0", - seed=0, + seed=None, **kwargs ): # Set logger. @@ -116,8 +116,9 @@ class DNNModelPytorch(Model): ) ) - np.random.seed(self.seed) - torch.manual_seed(self.seed) + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) if loss not in {"mse", "binary"}: raise NotImplementedError("loss {} is not supported!".format(loss)) diff --git a/qlib/contrib/model/pytorch_sfm.py b/qlib/contrib/model/pytorch_sfm.py index d845f6245..da75b1bca 100644 --- a/qlib/contrib/model/pytorch_sfm.py +++ b/qlib/contrib/model/pytorch_sfm.py @@ -217,7 +217,7 @@ class SFM(Model): loss="mse", optimizer="gd", GPU="0", - seed=0, + seed=None, **kwargs ): # Set logger. @@ -282,8 +282,9 @@ class SFM(Model): ) ) - np.random.seed(self.seed) - torch.manual_seed(self.seed) + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) self.sfm_model = SFM_Model( d_feat=self.d_feat,