diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 033d31536..261599401 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,6 +27,7 @@ jobs: run: | pip install --upgrade cython pip install numpy jupyter jupyter_contrib_nbextensions + pip install -U scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't python setup.py install - name: Install Lightgbm for MacOS @@ -56,4 +57,4 @@ jobs: - name: Test workflow by config run: | - qrun examples/benchmarks/LightGBM/workflow_config_lightgbm.yaml + qrun examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml diff --git a/CHANGES.rst b/CHANGES.rst index 1b0982242..114d577f3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -114,7 +114,7 @@ Version 0.4.1 Version 0.4.2 -------------------- - Refactor DataHandler -- Add ``ALPHA360`` DataHandler +- Add ``Alpha360`` DataHandler Version 0.4.3 diff --git a/README.md b/README.md index b1a5f7092..de7dd1cf0 100644 --- a/README.md +++ b/README.md @@ -153,9 +153,6 @@ Qlib provides a tool named `qrun` to run the whole workflow automatically (inclu annualized_return 0.128982 information_ratio 1.444287 max_drawdown -0.091078 - - - ``` Here are detailed documents for `qrun` and [workflow](https://qlib.readthedocs.io/en/latest/component/workflow.html). @@ -218,7 +215,7 @@ All the models listed above are runnable with ``Qlib``. Users can find the confi ## Run multiple models `Qlib` also provides a script [`run_all_model.py`](examples/run_all_model.py) which can run multiple models for several iterations. (**Note**: the script only supprots *Linux* now. Other OS will be supported in the future.) -The script will create a unique virtual environment for each model, and delete the environments after training. Thus, only experiment results such as `IC` and `backtest` results will be generated and stored. (**Note**: the script will erase your previous experiment records created by running itself.) +The script will create a unique virtual environment for each model, and delete the environments after training. Thus, only experiment results such as `IC` and `backtest` results will be generated and stored. Here is an example of running all the models for 10 iterations: ```python @@ -234,7 +231,7 @@ Dataset plays a very important role in Quant. Here is a list of the datasets bui | Dataset | US Market | China Market | | -- | -- | -- | | [Alpha360](./qlib/contrib/data/handler.py) | √ | √ | -| [Alpha158](./qlib/contrib/data/handler.py) | √ | √ | +| [Alpha158](./qlib/contrib/data/handler.py) | √ | √ | [Here](https://qlib.readthedocs.io/en/latest/advanced/alpha.html) is a tutorial to build dataset with `Qlib`. Your PR to build new Quant dataset is highly welcomed. diff --git a/examples/benchmarks/ALSTM/workflow_config_alstm.yaml b/examples/benchmarks/ALSTM/workflow_config_alstm_Alpha360.yaml similarity index 97% rename from examples/benchmarks/ALSTM/workflow_config_alstm.yaml rename to examples/benchmarks/ALSTM/workflow_config_alstm_Alpha360.yaml index 66367034d..6226cdaf2 100644 --- a/examples/benchmarks/ALSTM/workflow_config_alstm.yaml +++ b/examples/benchmarks/ALSTM/workflow_config_alstm_Alpha360.yaml @@ -54,7 +54,6 @@ task: batch_size: 800 metric: loss loss: mse - seed: 0 GPU: 0 rnn_type: GRU dataset: @@ -62,7 +61,7 @@ task: module_path: qlib.data.dataset kwargs: handler: - class: ALPHA360 + class: Alpha360 module_path: qlib.contrib.data.handler kwargs: *data_handler_config segments: diff --git a/examples/benchmarks/CatBoost/workflow_config_catboost.yaml b/examples/benchmarks/CatBoost/workflow_config_catboost_Alpha158.yaml similarity index 100% rename from examples/benchmarks/CatBoost/workflow_config_catboost.yaml rename to examples/benchmarks/CatBoost/workflow_config_catboost_Alpha158.yaml diff --git a/examples/benchmarks/GATs/workflow_config_gats.yaml b/examples/benchmarks/GATs/workflow_config_gats_Alpha360.yaml similarity index 92% rename from examples/benchmarks/GATs/workflow_config_gats.yaml rename to examples/benchmarks/GATs/workflow_config_gats_Alpha360.yaml index a608a7e49..d778c9b1b 100644 --- a/examples/benchmarks/GATs/workflow_config_gats.yaml +++ b/examples/benchmarks/GATs/workflow_config_gats_Alpha360.yaml @@ -56,14 +56,13 @@ task: base_model: LSTM with_pretrain: True model_path: "benchmarks/LSTM/model_lstm_csi300.pkl" - seed: 0 GPU: 0 dataset: class: DatasetH module_path: qlib.data.dataset kwargs: handler: - class: ALPHA360 + class: Alpha360 module_path: qlib.contrib.data.handler kwargs: *data_handler_config segments: @@ -74,6 +73,11 @@ task: - class: SignalRecord module_path: qlib.workflow.record_temp kwargs: {} + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 - class: PortAnaRecord module_path: qlib.workflow.record_temp kwargs: diff --git a/examples/benchmarks/GRU/workflow_config_gru.yaml b/examples/benchmarks/GRU/workflow_config_gru_Alpha360.yaml similarity index 97% rename from examples/benchmarks/GRU/workflow_config_gru.yaml rename to examples/benchmarks/GRU/workflow_config_gru_Alpha360.yaml index 381581a77..2494d40f0 100644 --- a/examples/benchmarks/GRU/workflow_config_gru.yaml +++ b/examples/benchmarks/GRU/workflow_config_gru_Alpha360.yaml @@ -54,14 +54,13 @@ task: batch_size: 800 metric: loss loss: mse - seed: 0 GPU: 0 dataset: class: DatasetH module_path: qlib.data.dataset kwargs: handler: - class: ALPHA360 + class: Alpha360 module_path: qlib.contrib.data.handler kwargs: *data_handler_config segments: diff --git a/examples/benchmarks/LSTM/workflow_config_lstm.yaml b/examples/benchmarks/LSTM/workflow_config_lstm_Alpha360.yaml similarity index 97% rename from examples/benchmarks/LSTM/workflow_config_lstm.yaml rename to examples/benchmarks/LSTM/workflow_config_lstm_Alpha360.yaml index cb3b2a789..2aa5fd061 100644 --- a/examples/benchmarks/LSTM/workflow_config_lstm.yaml +++ b/examples/benchmarks/LSTM/workflow_config_lstm_Alpha360.yaml @@ -54,14 +54,13 @@ task: batch_size: 800 metric: loss loss: mse - seed: 0 GPU: 0 dataset: class: DatasetH module_path: qlib.data.dataset kwargs: handler: - class: ALPHA360 + class: Alpha360 module_path: qlib.contrib.data.handler kwargs: *data_handler_config segments: diff --git a/examples/benchmarks/LightGBM/workflow_config_lightgbm.yaml b/examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml similarity index 98% rename from examples/benchmarks/LightGBM/workflow_config_lightgbm.yaml rename to examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml index 76a6347db..88202be1b 100644 --- a/examples/benchmarks/LightGBM/workflow_config_lightgbm.yaml +++ b/examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml @@ -32,7 +32,7 @@ task: kwargs: loss: mse colsample_bytree: 0.8879 - learning_rate: 0.0421 + learning_rate: 0.2 subsample: 0.8789 lambda_l1: 205.6999 lambda_l2: 580.9768 diff --git a/examples/benchmarks/Linear/workflow_config_linear.yaml b/examples/benchmarks/Linear/workflow_config_linear_Alpha158.yaml similarity index 100% rename from examples/benchmarks/Linear/workflow_config_linear.yaml rename to examples/benchmarks/Linear/workflow_config_linear_Alpha158.yaml diff --git a/examples/benchmarks/MLP/workflow_config_mlp.yaml b/examples/benchmarks/MLP/workflow_config_mlp_Alpha158.yaml similarity index 100% rename from examples/benchmarks/MLP/workflow_config_mlp.yaml rename to examples/benchmarks/MLP/workflow_config_mlp_Alpha158.yaml diff --git a/examples/benchmarks/SFM/workflow_config_sfm.yaml b/examples/benchmarks/SFM/workflow_config_sfm_Alpha360.yaml similarity index 97% rename from examples/benchmarks/SFM/workflow_config_sfm.yaml rename to examples/benchmarks/SFM/workflow_config_sfm_Alpha360.yaml index edf176e62..a23fe3854 100644 --- a/examples/benchmarks/SFM/workflow_config_sfm.yaml +++ b/examples/benchmarks/SFM/workflow_config_sfm_Alpha360.yaml @@ -57,14 +57,13 @@ task: eval_steps: 5 loss: mse optimizer: adam - GPU: 1 - seed: 710 + GPU: 0 dataset: class: DatasetH module_path: qlib.data.dataset kwargs: handler: - class: ALPHA360 + class: Alpha360 module_path: qlib.contrib.data.handler kwargs: *data_handler_config segments: diff --git a/examples/benchmarks/TFT/data_formatters/qlib_Alpha158.py b/examples/benchmarks/TFT/data_formatters/qlib_Alpha158.py index 44a9284f7..03c169b9b 100644 --- a/examples/benchmarks/TFT/data_formatters/qlib_Alpha158.py +++ b/examples/benchmarks/TFT/data_formatters/qlib_Alpha158.py @@ -208,7 +208,7 @@ class Alpha158Formatter(GenericDataFormatter): model_params = { "dropout_rate": 0.4, - "hidden_layer_size": 16, + "hidden_layer_size": 160, "learning_rate": 0.0001, "minibatch_size": 128, "max_gradient_norm": 0.0135, diff --git a/examples/benchmarks/TFT/workflow_config_tft.yaml b/examples/benchmarks/TFT/workflow_config_tft_Alpha158.yaml similarity index 100% rename from examples/benchmarks/TFT/workflow_config_tft.yaml rename to examples/benchmarks/TFT/workflow_config_tft_Alpha158.yaml diff --git a/examples/benchmarks/XGBoost/workflow_config_xgboost.yaml b/examples/benchmarks/XGBoost/workflow_config_xgboost_Alpha158.yaml similarity index 100% rename from examples/benchmarks/XGBoost/workflow_config_xgboost.yaml rename to examples/benchmarks/XGBoost/workflow_config_xgboost_Alpha158.yaml diff --git a/examples/run_all_model.py b/examples/run_all_model.py index 3d8628c4a..c8fdebd84 100644 --- a/examples/run_all_model.py +++ b/examples/run_all_model.py @@ -15,6 +15,7 @@ import traceback import functools import statistics import subprocess +from datetime import datetime from pathlib import Path from operator import xor from pprint import pprint @@ -45,8 +46,6 @@ if not exists_qlib_data(provider_uri): GetData().qlib_data(target_dir=provider_uri, region=REG_CN) qlib.init(provider_uri=provider_uri, region=REG_CN, exp_manager=exp_manager) -if os.path.isdir(exp_path): - shutil.rmtree(exp_path) # decorator to check the arguments def only_allow_defined_args(function_to_decorate): @@ -136,9 +135,9 @@ def get_all_folders(models, exclude) -> dict: # function to get all the files under the model folder -def get_all_files(folder_path) -> (str, str): - yaml_path = str(Path(f"{folder_path}") / "*.yaml") - req_path = str(Path(f"{folder_path}") / "*.txt") +def get_all_files(folder_path, dataset) -> (str, str): + yaml_path = str(Path(f"{folder_path}") / f"*{dataset}*.yaml") + req_path = str(Path(f"{folder_path}") / f"*.txt") return glob.glob(yaml_path)[0], glob.glob(req_path)[0] @@ -152,6 +151,10 @@ def get_all_results(folders) -> dict: result["annualized_return_with_cost"] = list() result["information_ratio_with_cost"] = list() result["max_drawdown_with_cost"] = list() + result["ic"] = list() + result["icir"] = list() + result["rank_ic"] = list() + result["rank_icir"] = list() for recorder_id in recorders: if recorders[recorder_id].status == "FINISHED": recorder = R.get_recorder(recorder_id=recorder_id, experiment_name=fn) @@ -159,19 +162,27 @@ def get_all_results(folders) -> dict: result["annualized_return_with_cost"].append(metrics["excess_return_with_cost.annualized_return"]) result["information_ratio_with_cost"].append(metrics["excess_return_with_cost.information_ratio"]) result["max_drawdown_with_cost"].append(metrics["excess_return_with_cost.max_drawdown"]) + result["ic"].append(metrics["IC"]) + result["icir"].append(metrics["ICIR"]) + result["rank_ic"].append(metrics["Rank IC"]) + result["rank_icir"].append(metrics["Rank ICIR"]) results[fn] = result return results # function to generate and save markdown table -def gen_and_save_md_table(metrics): - table = "| Model Name | Annualized Return | Information Ratio | Max Drawdown |\n" - table += "|---|---|---|---|\n" +def gen_and_save_md_table(metrics, dataset): + table = "| Model Name | Dataset | IC | ICIR | Rank IC | Rank ICIR | Annualized Return | Information Ratio | Max Drawdown |\n" + table += "|---|---|---|---|---|---|---|---|---|\n" for fn in metrics: + ic = metrics[fn]["ic"] + icir = metrics[fn]["icir"] + ric = metrics[fn]["rank_ic"] + ricir = metrics[fn]["rank_icir"] ar = metrics[fn]["annualized_return_with_cost"] ir = metrics[fn]["information_ratio_with_cost"] md = metrics[fn]["max_drawdown_with_cost"] - table += f"| {fn} | {ar[0]:9.4f}±{ar[1]:9.2f} | {ir[0]:9.4f}±{ir[1]:9.2f}| {md[0]:9.4f}±{md[1]:9.2f} |\n" + table += f"| {fn} | {dataset} | {ic[0]:5.4f}±{ic[1]:2.2f} | {icir[0]:5.4f}±{icir[1]:2.2f}| {ric[0]:5.4f}±{ric[1]:2.2f} | {ricir[0]:5.4f}±{ricir[1]:2.2f} | {ar[0]:5.4f}±{ar[1]:2.2f} | {ir[0]:5.4f}±{ir[1]:2.2f}| {md[0]:5.4f}±{md[1]:2.2f} |\n" pprint(table) with open("table.md", "w") as f: f.write(table) @@ -180,7 +191,7 @@ def gen_and_save_md_table(metrics): # function to run the all the models @only_allow_defined_args -def run(times=1, models=None, exclude=False): +def run(times=1, models=None, dataset="Alpha360", exclude=False): """ Please be aware that this function can only work under Linux. MacOS and Windows will be supported in the future. Any PR to enhance this method is highly welcomed. @@ -193,6 +204,8 @@ def run(times=1, models=None, exclude=False): determines the specific model or list of models to run or exclude. exclude : boolean determines whether the model being used is excluded or included. + dataset : str + determines the dataset to be used for each model. Usage: ------- @@ -206,13 +219,16 @@ def run(times=1, models=None, exclude=False): # Case 2 - run specific models multiple times python run_all_model.py 3 mlp - # Case 3 - run other models except those are given as arguments for multiple times - python run_all_model.py 3 [mlp,tft,lstm] True + # Case 3 - run specific models multiple times with specific dataset + python run_all_model.py 3 mlp Alpha158 - # Case 4 - run specific models for one time + # Case 4 - run other models except those are given as arguments for multiple times + python run_all_model.py 3 [mlp,tft,lstm] --exclude=True + + # Case 5 - run specific models for one time python run_all_model.py --models=[mlp,lightgbm] - # Case 5 - run other models except those are given as aruments for one time + # Case 6 - run other models except those are given as aruments for one time python run_all_model.py --models=[mlp,tft,sfm] --exclude=True """ @@ -226,7 +242,7 @@ def run(times=1, models=None, exclude=False): env_path, python_path, conda_activate = create_env() # get all files sys.stderr.write("Retrieving files...\n") - yaml_path, req_path = get_all_files(folders[fn]) + yaml_path, req_path = get_all_files(folders[fn], dataset) sys.stderr.write("\n") # install requirements.txt sys.stderr.write("Installing requirements.txt...\n") @@ -240,6 +256,7 @@ def run(times=1, models=None, exclude=False): sys.stderr.write("\n") # install qlib sys.stderr.write("Installing qlib...\n") + execute(f"{python_path} -m pip install --upgrade pip") # TODO: FIX ME! execute(f"{python_path} -m pip install --upgrade cython") # TODO: FIX ME! if fn == "TFT": execute( @@ -272,12 +289,15 @@ def run(times=1, models=None, exclude=False): results = cal_mean_std(results) # generating md table sys.stderr.write(f"Generating markdown table...\n") - gen_and_save_md_table(results) + gen_and_save_md_table(results, dataset) sys.stderr.write("\n") # print erros sys.stderr.write(f"Here are some of the errors of the models...\n") pprint(errors) sys.stderr.write("\n") + # move results folder + shutil.move(exp_path, exp_path + f"_{dataset}_{datetime.now().strftime('%Y-%m-%d_%H:%M:%S')}") + shutil.move("table.md", f"table_{dataset}_{datetime.now().strftime('%Y-%m-%d_%H:%M:%S')}.md") if __name__ == "__main__": diff --git a/qlib/__init__.py b/qlib/__init__.py index 080fa5329..f79b8c4f5 100644 --- a/qlib/__init__.py +++ b/qlib/__init__.py @@ -4,6 +4,7 @@ __version__ = "0.6.0.dev" + import os import re import sys diff --git a/qlib/contrib/data/handler.py b/qlib/contrib/data/handler.py index 6b3de336d..88a1f0680 100644 --- a/qlib/contrib/data/handler.py +++ b/qlib/contrib/data/handler.py @@ -43,7 +43,7 @@ _DEFAULT_INFER_PROCESSORS = [ ] -class ALPHA360(DataHandlerLP): +class Alpha360(DataHandlerLP): def __init__( self, instruments="csi500", @@ -119,7 +119,7 @@ class ALPHA360(DataHandlerLP): return fields, names -class ALPHA360vwap(ALPHA360): +class Alpha360vwap(Alpha360): def get_label_config(self): return (["Ref($vwap, -2)/Ref($vwap, -1) - 1"], ["LABEL0"]) diff --git a/qlib/contrib/model/pytorch_alstm.py b/qlib/contrib/model/pytorch_alstm.py index ba75c70ca..d85fef628 100644 --- a/qlib/contrib/model/pytorch_alstm.py +++ b/qlib/contrib/model/pytorch_alstm.py @@ -57,7 +57,7 @@ class ALSTM(Model): loss="mse", optimizer="adam", GPU="0", - seed=0, + seed=None, **kwargs ): # Set logger. @@ -76,7 +76,7 @@ class ALSTM(Model): self.early_stop = early_stop self.optimizer = optimizer.lower() self.loss = loss - self.visible_GPU = GPU + self.device = torch.device("cuda:%d" % (GPU) if torch.cuda.is_available() else "cpu") self.use_gpu = torch.cuda.is_available() self.seed = seed @@ -113,8 +113,9 @@ class ALSTM(Model): ) ) - np.random.seed(self.seed) - torch.manual_seed(self.seed) + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) self.ALSTM_model = ALSTMModel( d_feat=self.d_feat, @@ -130,11 +131,7 @@ class ALSTM(Model): raise NotImplementedError("optimizer {} is not supported!".format(optimizer)) self._fitted = False - if self.use_gpu: - self.ALSTM_model.cuda() - # set the visible GPU - if self.visible_GPU: - os.environ["CUDA_VISIBLE_DEVICES"] = self.visible_GPU + self.ALSTM_model.to(self.device) def mse(self, pred, label): loss = (pred - label) ** 2 @@ -172,12 +169,8 @@ class ALSTM(Model): if len(indices) - i < self.batch_size: break - feature = torch.from_numpy(x_train_values[indices[i : i + self.batch_size]]).float() - label = torch.from_numpy(y_train_values[indices[i : i + self.batch_size]]).float() - - if self.use_gpu: - feature = feature.cuda() - label = label.cuda() + feature = torch.from_numpy(x_train_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_train_values[indices[i : i + self.batch_size]]).float().to(self.device) pred = self.ALSTM_model(feature) loss = self.loss_fn(pred, label) @@ -205,12 +198,8 @@ class ALSTM(Model): if len(indices) - i < self.batch_size: break - feature = torch.from_numpy(x_values[indices[i : i + self.batch_size]]).float() - label = torch.from_numpy(y_values[indices[i : i + self.batch_size]]).float() - - if self.use_gpu: - feature = feature.cuda() - label = label.cuda() + feature = torch.from_numpy(x_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_values[indices[i : i + self.batch_size]]).float().to(self.device) pred = self.ALSTM_model(feature) loss = self.loss_fn(pred, label) @@ -298,10 +287,7 @@ class ALSTM(Model): else: end = begin + self.batch_size - x_batch = torch.from_numpy(x_values[begin:end]).float() - - if self.use_gpu: - x_batch = x_batch.cuda() + x_batch = torch.from_numpy(x_values[begin:end]).float().to(self.device) with torch.no_grad(): if self.use_gpu: diff --git a/qlib/contrib/model/pytorch_gats.py b/qlib/contrib/model/pytorch_gats.py index 2e19c0542..c22e48204 100644 --- a/qlib/contrib/model/pytorch_gats.py +++ b/qlib/contrib/model/pytorch_gats.py @@ -62,7 +62,7 @@ class GATs(Model): model_path=None, optimizer="adam", GPU="0", - seed=0, + seed=None, **kwargs ): # Set logger. @@ -83,7 +83,7 @@ class GATs(Model): self.base_model = base_model self.with_pretrain = with_pretrain self.model_path = model_path - self.visible_GPU = GPU + self.device = torch.device("cuda:%d" % (GPU) if torch.cuda.is_available() else "cpu") self.use_gpu = torch.cuda.is_available() self.seed = seed @@ -123,8 +123,11 @@ class GATs(Model): seed, ) ) - np.random.seed(self.seed) - torch.manual_seed(self.seed) + + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) + self.GAT_model = GATModel( d_feat=self.d_feat, hidden_size=self.hidden_size, @@ -140,11 +143,7 @@ class GATs(Model): raise NotImplementedError("optimizer {} is not supported!".format(optimizer)) self._fitted = False - if self.use_gpu: - self.GAT_model.cuda() - # set the visible GPU - if self.visible_GPU: - os.environ["CUDA_VISIBLE_DEVICES"] = self.visible_GPU + self.GAT_model.to(self.device) def mse(self, pred, label): loss = (pred - label) ** 2 @@ -190,12 +189,8 @@ class GATs(Model): for idx, count in zip(daily_index, daily_count): batch = slice(idx, idx + count) - feature = torch.from_numpy(x_train_values[batch]).float() - label = torch.from_numpy(y_train_values[batch]).float() - - if self.use_gpu: - feature = feature.cuda() - label = label.cuda() + feature = torch.from_numpy(x_train_values[batch]).float().to(self.device) + label = torch.from_numpy(y_train_values[batch]).float().to(self.device) pred = self.GAT_model(feature) loss = self.loss_fn(pred, label) @@ -221,12 +216,8 @@ class GATs(Model): for idx, count in zip(daily_index, daily_count): batch = slice(idx, idx + count) - feature = torch.from_numpy(x_values[batch]).float() - label = torch.from_numpy(y_values[batch]).float() - - if self.use_gpu: - feature = feature.cuda() - label = label.cuda() + feature = torch.from_numpy(x_values[batch]).float().to(self.device) + label = torch.from_numpy(y_values[batch]).float().to(self.device) pred = self.GAT_model(feature) loss = self.loss_fn(pred, label) @@ -330,10 +321,7 @@ class GATs(Model): for idx, count in zip(daily_index, daily_count): batch = slice(idx, idx + count) - x_batch = torch.from_numpy(x_values[batch]).float() - - if self.use_gpu: - x_batch = x_batch.cuda() + x_batch = torch.from_numpy(x_values[batch]).float().to(self.device) with torch.no_grad(): if self.use_gpu: diff --git a/qlib/contrib/model/pytorch_gru.py b/qlib/contrib/model/pytorch_gru.py index 4ef4e2553..22c63c539 100755 --- a/qlib/contrib/model/pytorch_gru.py +++ b/qlib/contrib/model/pytorch_gru.py @@ -57,7 +57,7 @@ class GRU(Model): loss="mse", optimizer="adam", GPU="0", - seed=0, + seed=None, **kwargs ): # Set logger. @@ -76,7 +76,7 @@ class GRU(Model): self.early_stop = early_stop self.optimizer = optimizer.lower() self.loss = loss - self.visible_GPU = GPU + self.device = torch.device("cuda:%d" % (GPU) if torch.cuda.is_available() else "cpu") self.use_gpu = torch.cuda.is_available() self.seed = seed @@ -113,8 +113,9 @@ class GRU(Model): ) ) - np.random.seed(self.seed) - torch.manual_seed(self.seed) + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) self.gru_model = GRUModel( d_feat=self.d_feat, @@ -130,11 +131,7 @@ class GRU(Model): raise NotImplementedError("optimizer {} is not supported!".format(optimizer)) self._fitted = False - if self.use_gpu: - self.gru_model.cuda() - # set the visible GPU - if self.visible_GPU: - os.environ["CUDA_VISIBLE_DEVICES"] = self.visible_GPU + self.gru_model.to(self.device) def mse(self, pred, label): loss = (pred - label) ** 2 @@ -172,12 +169,8 @@ class GRU(Model): if len(indices) - i < self.batch_size: break - feature = torch.from_numpy(x_train_values[indices[i : i + self.batch_size]]).float() - label = torch.from_numpy(y_train_values[indices[i : i + self.batch_size]]).float() - - if self.use_gpu: - feature = feature.cuda() - label = label.cuda() + feature = torch.from_numpy(x_train_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_train_values[indices[i : i + self.batch_size]]).float().to(self.device) pred = self.gru_model(feature) loss = self.loss_fn(pred, label) @@ -205,12 +198,8 @@ class GRU(Model): if len(indices) - i < self.batch_size: break - feature = torch.from_numpy(x_values[indices[i : i + self.batch_size]]).float() - label = torch.from_numpy(y_values[indices[i : i + self.batch_size]]).float() - - if self.use_gpu: - feature = feature.cuda() - label = label.cuda() + feature = torch.from_numpy(x_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_values[indices[i : i + self.batch_size]]).float().to(self.device) pred = self.gru_model(feature) loss = self.loss_fn(pred, label) @@ -298,10 +287,7 @@ class GRU(Model): else: end = begin + self.batch_size - x_batch = torch.from_numpy(x_values[begin:end]).float() - - if self.use_gpu: - x_batch = x_batch.cuda() + x_batch = torch.from_numpy(x_values[begin:end]).float().to(self.device) with torch.no_grad(): if self.use_gpu: diff --git a/qlib/contrib/model/pytorch_lstm.py b/qlib/contrib/model/pytorch_lstm.py index 59df51e42..1c24246f8 100755 --- a/qlib/contrib/model/pytorch_lstm.py +++ b/qlib/contrib/model/pytorch_lstm.py @@ -57,7 +57,7 @@ class LSTM(Model): loss="mse", optimizer="adam", GPU="0", - seed=0, + seed=None, **kwargs ): # Set logger. @@ -76,7 +76,7 @@ class LSTM(Model): self.early_stop = early_stop self.optimizer = optimizer.lower() self.loss = loss - self.visible_GPU = GPU + self.device = torch.device("cuda:%d" % (GPU) if torch.cuda.is_available() else "cpu") self.use_gpu = torch.cuda.is_available() self.seed = seed @@ -113,8 +113,9 @@ class LSTM(Model): ) ) - np.random.seed(self.seed) - torch.manual_seed(self.seed) + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) self.lstm_model = LSTMModel( d_feat=self.d_feat, @@ -130,11 +131,7 @@ class LSTM(Model): raise NotImplementedError("optimizer {} is not supported!".format(optimizer)) self._fitted = False - if self.use_gpu: - self.lstm_model.cuda() - # set the visible GPU - if self.visible_GPU: - os.environ["CUDA_VISIBLE_DEVICES"] = self.visible_GPU + self.lstm_model.to(self.device) def mse(self, pred, label): loss = (pred - label) ** 2 @@ -172,12 +169,8 @@ class LSTM(Model): if len(indices) - i < self.batch_size: break - feature = torch.from_numpy(x_train_values[indices[i : i + self.batch_size]]).float() - label = torch.from_numpy(y_train_values[indices[i : i + self.batch_size]]).float() - - if self.use_gpu: - feature = feature.cuda() - label = label.cuda() + feature = torch.from_numpy(x_train_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_train_values[indices[i : i + self.batch_size]]).float().to(self.device) pred = self.lstm_model(feature) loss = self.loss_fn(pred, label) @@ -205,12 +198,8 @@ class LSTM(Model): if len(indices) - i < self.batch_size: break - feature = torch.from_numpy(x_values[indices[i : i + self.batch_size]]).float() - label = torch.from_numpy(y_values[indices[i : i + self.batch_size]]).float() - - if self.use_gpu: - feature = feature.cuda() - label = label.cuda() + feature = torch.from_numpy(x_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_values[indices[i : i + self.batch_size]]).float().to(self.device) pred = self.lstm_model(feature) loss = self.loss_fn(pred, label) @@ -298,10 +287,7 @@ class LSTM(Model): else: end = begin + self.batch_size - x_batch = torch.from_numpy(x_values[begin:end]).float() - - if self.use_gpu: - x_batch = x_batch.cuda() + x_batch = torch.from_numpy(x_values[begin:end]).float().to(self.device) with torch.no_grad(): if self.use_gpu: diff --git a/qlib/contrib/model/pytorch_nn.py b/qlib/contrib/model/pytorch_nn.py index a60227ff0..2e2a3e885 100644 --- a/qlib/contrib/model/pytorch_nn.py +++ b/qlib/contrib/model/pytorch_nn.py @@ -61,7 +61,7 @@ class DNNModelPytorch(Model): optimizer="gd", loss="mse", GPU="0", - seed=0, + seed=None, **kwargs ): # Set logger. @@ -79,7 +79,7 @@ class DNNModelPytorch(Model): self.lr_decay_steps = lr_decay_steps self.optimizer = optimizer.lower() self.loss_type = loss - self.visible_GPU = GPU + self.device = torch.device("cuda:%d" % (GPU) if torch.cuda.is_available() else "cpu") self.use_GPU = torch.cuda.is_available() self.seed = seed @@ -116,8 +116,9 @@ class DNNModelPytorch(Model): ) ) - np.random.seed(self.seed) - torch.manual_seed(self.seed) + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) if loss not in {"mse", "binary"}: raise NotImplementedError("loss {} is not supported!".format(loss)) @@ -146,11 +147,7 @@ class DNNModelPytorch(Model): ) self._fitted = False - if self.use_GPU: - self.dnn_model.cuda() - # set the visible GPU - if self.visible_GPU: - os.environ["CUDA_VISIBLE_DEVICES"] = self.visible_GPU + self.dnn_model.to(self.device) def fit( self, @@ -187,13 +184,9 @@ class DNNModelPytorch(Model): w_train_values = torch.from_numpy(w_train.values).float() train_num = y_train_values.shape[0] # prepare validation data - x_val_auto = torch.from_numpy(x_valid.values).float() - y_val_auto = torch.from_numpy(y_valid.values).float() - w_val_auto = torch.from_numpy(w_valid.values).float() - if self.use_GPU: - x_val_auto = x_val_auto.cuda() - y_val_auto = y_val_auto.cuda() - w_val_auto = w_val_auto.cuda() + x_val_auto = torch.from_numpy(x_valid.values).float().to(self.device) + y_val_auto = torch.from_numpy(y_valid.values).float().to(self.device) + w_val_auto = torch.from_numpy(w_valid.values).float().to(self.device) for step in range(self.max_steps): if stop_steps >= self.early_stop_rounds: @@ -204,14 +197,9 @@ class DNNModelPytorch(Model): self.dnn_model.train() self.train_optimizer.zero_grad() choice = np.random.choice(train_num, self.batch_size) - x_batch_auto = x_train_values[choice] - y_batch_auto = y_train_values[choice] - w_batch_auto = w_train_values[choice] - - if self.use_GPU: - x_batch_auto = x_batch_auto.cuda() - y_batch_auto = y_batch_auto.cuda() - w_batch_auto = w_batch_auto.cuda() + x_batch_auto = x_train_values[choice].to(self.device) + y_batch_auto = y_train_values[choice].to(self.device) + w_batch_auto = w_train_values[choice].to(self.device) # forward preds = self.dnn_model(x_batch_auto) @@ -276,9 +264,7 @@ class DNNModelPytorch(Model): if not self._fitted: raise ValueError("model is not fitted yet!") x_test_pd = dataset.prepare("test", col_set="feature") - x_test = torch.from_numpy(x_test_pd.values).float() - if self.use_GPU: - x_test = x_test.cuda() + x_test = torch.from_numpy(x_test_pd.values).float().to(self.device) self.dnn_model.eval() with torch.no_grad(): diff --git a/qlib/contrib/model/pytorch_sfm.py b/qlib/contrib/model/pytorch_sfm.py index d845f6245..15d945c89 100644 --- a/qlib/contrib/model/pytorch_sfm.py +++ b/qlib/contrib/model/pytorch_sfm.py @@ -217,7 +217,7 @@ class SFM(Model): loss="mse", optimizer="gd", GPU="0", - seed=0, + seed=None, **kwargs ): # Set logger. @@ -239,7 +239,7 @@ class SFM(Model): self.eval_steps = eval_steps self.optimizer = optimizer.lower() self.loss = loss - self.device = "cuda:%d" % (GPU) if torch.cuda.is_available() else "cpu" + self.device = torch.device("cuda:%d" % (GPU) if torch.cuda.is_available() else "cpu") self.use_gpu = torch.cuda.is_available() self.seed = seed @@ -282,8 +282,9 @@ class SFM(Model): ) ) - np.random.seed(self.seed) - torch.manual_seed(self.seed) + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) self.sfm_model = SFM_Model( d_feat=self.d_feat, diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 638ce22f4..000000000 --- a/requirements.txt +++ /dev/null @@ -1,25 +0,0 @@ -requests==2.22.0 -six==1.14.0 -lxml==4.5.0 -statsmodels==0.12.0 -pandas==1.1.2 -matplotlib==3.3.2 -scipy==1.3.3 -numpy==1.17.4 -Cython==0.29.21 -fire==0.3.1 -gevent_socketio==0.3.6 -hyperopt==0.2.4 -lightgbm==3.0.0 -loguru==0.5.3 -plotly==4.10.0 -pymongo==3.11.0 -PyYAML==5.3.1 -redis==3.5.3 -redis_lock==0.2.0 -sacred==0.8.1 -scikit_learn==0.23.2 -torch==1.6.0 -tqdm==4.49.0 -yahooquery==2.2.7 -mlflow==1.12.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 86efd1de4..cc495824e 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ from setuptools import find_packages, setup, Extension NAME = "pyqlib" DESCRIPTION = "A Quantitative-research Platform" REQUIRES_PYTHON = ">=3.5.0" + VERSION = "0.6.0.dev" # Detect Cython