diff --git a/README.md b/README.md
index 4f2509188..89d14e9eb 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,7 @@ For more details, please refer to our paper ["Qlib: An AI-oriented Quantitative
# Framework of Qlib
-
+
![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\")
"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -10,14 +17,43 @@
"# Licensed under the MIT License."
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "import sys, site\n",
+ "from pathlib import Path\n",
+ "\n",
+ "\n",
+ "try:\n",
+ " import qlib\n",
+ "except ImportError:\n",
+ " # install qlib\n",
+ " ! pip install pyqlib\n",
+ " # reload\n",
+ " site.main()\n",
+ "\n",
+ "scripts_dir = Path.cwd().parent.joinpath(\"scripts\")\n",
+ "if not scripts_dir.joinpath(\"get_data.py\").exists():\n",
+ " # download get_data.py script\n",
+ " scripts_dir = Path(\"~/tmp/qlib_code/scripts\").expanduser().resolve()\n",
+ " scripts_dir.mkdir(parents=True, exist_ok=True)\n",
+ " import requests\n",
+ " with requests.get(\"https://raw.githubusercontent.com/microsoft/qlib/main/scripts/get_data.py\") as resp:\n",
+ " with open(scripts_dir.joinpath(\"get_data.py\"), \"wb\") as fp:\n",
+ " fp.write(resp.content)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
- "import sys\n",
- "from pathlib import Path\n",
"\n",
"import qlib\n",
"import pandas as pd\n",
@@ -32,7 +68,7 @@
"from qlib.utils import exists_qlib_data, init_instance_by_config\n",
"from qlib.workflow import R\n",
"from qlib.workflow.record_temp import SignalRecord, PortAnaRecord\n",
- "from qlib.utils import flatten_dict"
+ "from qlib.utils import flatten_dict\n"
]
},
{
@@ -48,7 +84,7 @@
"provider_uri = \"~/.qlib/qlib_data/cn_data\" # target_dir\n",
"if not exists_qlib_data(provider_uri):\n",
" print(f\"Qlib data is not found in {provider_uri}\")\n",
- " sys.path.append(str(Path.cwd().parent.joinpath(\"scripts\")))\n",
+ " sys.path.append(str(scripts_dir))\n",
" from get_data import GetData\n",
" GetData().qlib_data(target_dir=provider_uri, region=REG_CN)\n",
"qlib.init(provider_uri=provider_uri, region=REG_CN)"
@@ -202,7 +238,9 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "scrolled": false
+ },
"outputs": [],
"source": [
"from qlib.contrib.report import analysis_model, analysis_position\n",
@@ -320,7 +358,8 @@
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
- "pygments_lexer": "ipython3"
+ "pygments_lexer": "ipython3",
+ "version": "3.7.9"
},
"toc": {
"base_numbering": 1,
diff --git a/examples/workflow_by_code_alstm.py b/examples/workflow_by_code_alstm.py
deleted file mode 100644
index 8fd9e3565..000000000
--- a/examples/workflow_by_code_alstm.py
+++ /dev/null
@@ -1,138 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-import sys
-from pathlib import Path
-
-import qlib
-import pandas as pd
-from qlib.config import REG_CN
-from qlib.contrib.strategy.strategy import TopkDropoutStrategy
-from qlib.contrib.evaluate import (
- backtest as normal_backtest,
- risk_analysis,
-)
-from qlib.utils import exists_qlib_data
-from qlib.utils import init_instance_by_config
-
-if __name__ == "__main__":
-
- # use default data
- provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
- if not exists_qlib_data(provider_uri):
- print(f"Qlib data is not found in {provider_uri}")
- sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts")))
- from get_data import GetData
-
- GetData().qlib_data(target_dir=provider_uri, region=REG_CN)
-
- qlib.init(provider_uri=provider_uri, region=REG_CN)
-
- MARKET = "csi300"
- BENCHMARK = "SH000300"
-
- ###################################
- # train model
- ###################################
- DATA_HANDLER_CONFIG = {
- "start_time": "2008-01-01",
- "end_time": "2020-08-01",
- "fit_start_time": "2008-01-01",
- "fit_end_time": "2014-12-31",
- "instruments": MARKET,
- }
-
- TRAINER_CONFIG = {
- "train_start_time": "2008-01-01",
- "train_end_time": "2014-12-31",
- "validate_start_time": "2015-01-01",
- "validate_end_time": "2016-12-31",
- "test_start_time": "2017-01-01",
- "test_end_time": "2020-08-01",
- }
-
- task = {
- "model": {
- "class": "ALSTM",
- "module_path": "qlib.contrib.model.pytorch_alstm",
- "kwargs": {
- "d_feat": 6,
- "hidden_size": 64,
- "num_layers": 2,
- "dropout": 0.0,
- "n_epochs": 200,
- "lr": 1e-3,
- "early_stop": 20,
- "batch_size": 800,
- "metric": "IC",
- "loss": "mse",
- "seed": 0,
- "GPU": "0",
- "rnn_type": "GRU",
- },
- },
- "dataset": {
- "class": "DatasetH",
- "module_path": "qlib.data.dataset",
- "kwargs": {
- "handler": {
- "class": "ALPHA360_Denoise",
- "module_path": "qlib.contrib.data.handler",
- "kwargs": DATA_HANDLER_CONFIG,
- },
- "segments": {
- "train": ("2008-01-01", "2014-12-31"),
- "valid": ("2015-01-01", "2016-12-31"),
- "test": ("2017-01-01", "2020-08-01"),
- },
- },
- }
- # You shoud record the data in specific sequence
- # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
- }
-
- model = init_instance_by_config(task["model"])
- dataset = init_instance_by_config(task["dataset"])
- model.fit(dataset)
-
- pred_score = model.predict(dataset)
-
- # save pred_score to file
- pred_score_path = Path("~/tmp/qlib/pred_score.pkl").expanduser()
- pred_score_path.parent.mkdir(exist_ok=True, parents=True)
- pred_score.to_pickle(pred_score_path)
-
- ###################################
- # backtest
- ###################################
- STRATEGY_CONFIG = {
- "topk": 50,
- "n_drop": 5,
- }
- BACKTEST_CONFIG = {
- "verbose": False,
- "limit_threshold": 0.095,
- "account": 100000000,
- "benchmark": BENCHMARK,
- "deal_price": "close",
- "open_cost": 0.0005,
- "close_cost": 0.0015,
- "min_cost": 5,
- }
-
- # use default strategy
- # custom Strategy, refer to: TODO: Strategy API url
- strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)
- report_normal, positions_normal = normal_backtest(pred_score, strategy=strategy, **BACKTEST_CONFIG)
-
- ###################################
- # analyze
- # If need a more detailed analysis, refer to: examples/train_and_bakctest.ipynb
- ###################################
- analysis = dict()
- analysis["excess_return_without_cost"] = risk_analysis(report_normal["return"] - report_normal["bench"])
- analysis["excess_return_with_cost"] = risk_analysis(
- report_normal["return"] - report_normal["bench"] - report_normal["cost"]
- )
- analysis_df = pd.concat(analysis) # type: pd.DataFrame
- print(analysis_df)
diff --git a/examples/workflow_by_code_gats.py b/examples/workflow_by_code_gats.py
deleted file mode 100644
index 20f3ae552..000000000
--- a/examples/workflow_by_code_gats.py
+++ /dev/null
@@ -1,140 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-import sys
-from pathlib import Path
-
-import qlib
-import pandas as pd
-from qlib.config import REG_CN
-
-from qlib.contrib.strategy.strategy import TopkDropoutStrategy
-from qlib.contrib.evaluate import (
- backtest as normal_backtest,
- risk_analysis,
-)
-from qlib.utils import exists_qlib_data
-from qlib.utils import init_instance_by_config
-
-
-if __name__ == "__main__":
-
- # use default data
- provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
- if not exists_qlib_data(provider_uri):
- print(f"Qlib data is not found in {provider_uri}")
- sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts")))
- from get_data import GetData
-
- GetData().qlib_data(target_dir=provider_uri, region=REG_CN)
-
- qlib.init(provider_uri=provider_uri, region=REG_CN)
-
- MARKET = "csi300"
- BENCHMARK = "SH000300"
-
- ###################################
- # train model
- ###################################
- DATA_HANDLER_CONFIG = {
- "start_time": "2008-01-01",
- "end_time": "2020-08-01",
- "fit_start_time": "2008-01-01",
- "fit_end_time": "2014-12-31",
- "instruments": MARKET,
- }
-
- TRAINER_CONFIG = {
- "train_start_time": "2008-01-01",
- "train_end_time": "2014-12-31",
- "validate_start_time": "2015-01-01",
- "validate_end_time": "2016-12-31",
- "test_start_time": "2017-01-01",
- "test_end_time": "2020-08-01",
- }
-
- task = {
- "model": {
- "class": "GAT",
- "module_path": "qlib.contrib.model.pytorch_gats",
- "kwargs": {
- "d_feat": 6,
- "hidden_size": 64,
- "num_layers": 2,
- "dropout": 0.7,
- "n_epochs": 200,
- "lr": 1e-4,
- "early_stop": 20,
- "metric": "loss",
- "loss": "mse",
- "base_model": "LSTM",
- "with_pretrain": True,
- "seed": 0,
- "GPU": "0",
- },
- },
- "dataset": {
- "class": "DatasetH",
- "module_path": "qlib.data.dataset",
- "kwargs": {
- "handler": {
- "class": "ALPHA360_Denoise",
- "module_path": "qlib.contrib.data.handler",
- "kwargs": DATA_HANDLER_CONFIG,
- },
- "segments": {
- "train": ("2008-01-01", "2014-12-31"),
- "valid": ("2015-01-01", "2016-12-31"),
- "test": ("2017-01-01", "2020-08-01"),
- },
- },
- }
- # You shoud record the data in specific sequence
- # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
- }
-
- model = init_instance_by_config(task["model"])
- dataset = init_instance_by_config(task["dataset"])
- model.fit(dataset)
-
- pred_score = model.predict(dataset)
-
- # save pred_score to file
- pred_score_path = Path("~/tmp/qlib/pred_score.pkl").expanduser()
- pred_score_path.parent.mkdir(exist_ok=True, parents=True)
- pred_score.to_pickle(pred_score_path)
-
- ###################################
- # backtest
- ###################################
- STRATEGY_CONFIG = {
- "topk": 50,
- "n_drop": 5,
- }
- BACKTEST_CONFIG = {
- "verbose": False,
- "limit_threshold": 0.095,
- "account": 100000000,
- "benchmark": BENCHMARK,
- "deal_price": "close",
- "open_cost": 0.0005,
- "close_cost": 0.0015,
- "min_cost": 5,
- }
-
- # use default strategy
- # custom Strategy, refer to: TODO: Strategy API url
- strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)
- report_normal, positions_normal = normal_backtest(pred_score, strategy=strategy, **BACKTEST_CONFIG)
-
- ###################################
- # analyze
- # If need a more detailed analysis, refer to: examples/train_and_bakctest.ipynb
- ###################################
- analysis = dict()
- analysis["excess_return_without_cost"] = risk_analysis(report_normal["return"] - report_normal["bench"])
- analysis["excess_return_with_cost"] = risk_analysis(
- report_normal["return"] - report_normal["bench"] - report_normal["cost"]
- )
- analysis_df = pd.concat(analysis) # type: pd.DataFrame
- print(analysis_df)
diff --git a/examples/workflow_by_code_gru.py b/examples/workflow_by_code_gru.py
deleted file mode 100644
index dece520d1..000000000
--- a/examples/workflow_by_code_gru.py
+++ /dev/null
@@ -1,144 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-import sys
-from pathlib import Path
-
-import qlib
-import pandas as pd
-from qlib.config import REG_CN
-from qlib.contrib.model.pytorch_gru import GRU
-from qlib.contrib.data.handler import ALPHA360_Denoise
-from qlib.contrib.strategy.strategy import TopkDropoutStrategy
-from qlib.contrib.evaluate import (
- backtest as normal_backtest,
- risk_analysis,
-)
-from qlib.utils import exists_qlib_data
-
-# from qlib.model.learner import train_model
-from qlib.utils import init_instance_by_config
-
-import pickle
-
-if __name__ == "__main__":
-
- # use default data
- provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
- if not exists_qlib_data(provider_uri):
- print(f"Qlib data is not found in {provider_uri}")
- sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts")))
- from get_data import GetData
-
- GetData().qlib_data(target_dir=provider_uri, region=REG_CN)
-
- qlib.init(provider_uri=provider_uri, region=REG_CN)
-
- MARKET = "csi300"
- BENCHMARK = "SH000300"
-
- ###################################
- # train model
- ###################################
- DATA_HANDLER_CONFIG = {
- "start_time": "2008-01-01",
- "end_time": "2020-08-01",
- "fit_start_time": "2008-01-01",
- "fit_end_time": "2014-12-31",
- "instruments": MARKET,
- }
-
- TRAINER_CONFIG = {
- "train_start_time": "2008-01-01",
- "train_end_time": "2014-12-31",
- "validate_start_time": "2015-01-01",
- "validate_end_time": "2016-12-31",
- "test_start_time": "2017-01-01",
- "test_end_time": "2020-08-01",
- }
-
- task = {
- "model": {
- "class": "GRU",
- "module_path": "qlib.contrib.model.pytorch_gru",
- "kwargs": {
- "d_feat": 6,
- "hidden_size": 64,
- "num_layers": 2,
- "dropout": 0.0,
- "n_epochs": 200,
- "lr": 1e-3,
- "early_stop": 20,
- "batch_size": 800,
- "metric": "loss",
- "loss": "mse",
- "seed": 0,
- "GPU": 0,
- },
- },
- "dataset": {
- "class": "DatasetH",
- "module_path": "qlib.data.dataset",
- "kwargs": {
- "handler": {
- "class": "ALPHA360_Denoise",
- "module_path": "qlib.contrib.data.handler",
- "kwargs": DATA_HANDLER_CONFIG,
- },
- "segments": {
- "train": ("2008-01-01", "2014-12-31"),
- "valid": ("2015-01-01", "2016-12-31"),
- "test": ("2017-01-01", "2020-08-01"),
- },
- },
- }
- # You shoud record the data in specific sequence
- # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
- }
-
- # model = train_model(task)
- model = init_instance_by_config(task["model"])
- dataset = init_instance_by_config(task["dataset"])
- model.fit(dataset)
-
- pred_score = model.predict(dataset)
-
- # save pred_score to file
- pred_score_path = Path("~/tmp/qlib/pred_score.pkl").expanduser()
- pred_score_path.parent.mkdir(exist_ok=True, parents=True)
- pred_score.to_pickle(pred_score_path)
-
- ###################################
- # backtest
- ###################################
- STRATEGY_CONFIG = {
- "topk": 50,
- "n_drop": 5,
- }
- BACKTEST_CONFIG = {
- "verbose": False,
- "limit_threshold": 0.095,
- "account": 100000000,
- "benchmark": BENCHMARK,
- "deal_price": "close",
- "open_cost": 0.0005,
- "close_cost": 0.0015,
- "min_cost": 5,
- }
-
- # use default strategy
- # custom Strategy, refer to: TODO: Strategy API url
- strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)
- report_normal, positions_normal = normal_backtest(pred_score, strategy=strategy, **BACKTEST_CONFIG)
-
- ###################################
- # analyze
- # If need a more detailed analysis, refer to: examples/train_and_bakctest.ipynb
- ###################################
- analysis = dict()
- analysis["excess_return_without_cost"] = risk_analysis(report_normal["return"] - report_normal["bench"])
- analysis["excess_return_with_cost"] = risk_analysis(
- report_normal["return"] - report_normal["bench"] - report_normal["cost"]
- )
- analysis_df = pd.concat(analysis) # type: pd.DataFrame
- print(analysis_df)
diff --git a/examples/workflow_by_code_hats.py b/examples/workflow_by_code_hats.py
deleted file mode 100644
index 64bc860b4..000000000
--- a/examples/workflow_by_code_hats.py
+++ /dev/null
@@ -1,136 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-import sys
-from pathlib import Path
-import qlib
-import pandas as pd
-from qlib.config import REG_CN
-from qlib.contrib.strategy.strategy import TopkDropoutStrategy
-from qlib.contrib.evaluate import (
- backtest as normal_backtest,
- risk_analysis,
-)
-from qlib.utils import exists_qlib_data
-from qlib.utils import init_instance_by_config
-
-if __name__ == "__main__":
-
- # use default data
- provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
- if not exists_qlib_data(provider_uri):
- print(f"Qlib data is not found in {provider_uri}")
- sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts")))
- from get_data import GetData
-
- GetData().qlib_data(target_dir=provider_uri, region=REG_CN)
-
- qlib.init(provider_uri=provider_uri, region=REG_CN)
-
- MARKET = "csi300"
- BENCHMARK = "SH000300"
-
- ###################################
- # train model
- ###################################
- DATA_HANDLER_CONFIG = {
- "start_time": "2008-01-01",
- "end_time": "2020-08-01",
- "fit_start_time": "2008-01-01",
- "fit_end_time": "2014-12-31",
- "instruments": MARKET,
- }
-
- TRAINER_CONFIG = {
- "train_start_time": "2008-01-01",
- "train_end_time": "2014-12-31",
- "validate_start_time": "2015-01-01",
- "validate_end_time": "2016-12-31",
- "test_start_time": "2017-01-01",
- "test_end_time": "2020-08-01",
- }
-
- task = {
- "model": {
- "class": "HATS",
- "module_path": "qlib.contrib.model.pytorch_hats",
- "kwargs": {
- "d_feat": 6,
- "hidden_size": 64,
- "num_layers": 2,
- "dropout": 0.7,
- "n_epochs": 200,
- "lr": 1e-4,
- "early_stop": 20,
- "metric": "loss",
- "loss": "mse",
- "base_model": "LSTM",
- "seed": 0,
- "GPU": "2",
- },
- },
- "dataset": {
- "class": "DatasetH",
- "module_path": "qlib.data.dataset",
- "kwargs": {
- "handler": {
- "class": "ALPHA360_Denoise",
- "module_path": "qlib.contrib.data.handler",
- "kwargs": DATA_HANDLER_CONFIG,
- },
- "segments": {
- "train": ("2008-01-01", "2014-12-31"),
- "valid": ("2015-01-01", "2016-12-31"),
- "test": ("2017-01-01", "2020-08-01"),
- },
- },
- }
- # You shoud record the data in specific sequence
- # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
- }
-
- model = init_instance_by_config(task["model"])
- dataset = init_instance_by_config(task["dataset"])
- model.fit(dataset, save_path="benchmarks/HATS/model_hat.pkl")
-
- pred_score = model.predict(dataset)
-
- # save pred_score to file
- pred_score_path = Path("~/tmp/qlib/pred_score.pkl").expanduser()
- pred_score_path.parent.mkdir(exist_ok=True, parents=True)
- pred_score.to_pickle(pred_score_path)
-
- ###################################
- # backtest
- ###################################
- STRATEGY_CONFIG = {
- "topk": 50,
- "n_drop": 5,
- }
- BACKTEST_CONFIG = {
- "verbose": False,
- "limit_threshold": 0.095,
- "account": 100000000,
- "benchmark": BENCHMARK,
- "deal_price": "close",
- "open_cost": 0.0005,
- "close_cost": 0.0015,
- "min_cost": 5,
- }
-
- # use default strategy
- # custom Strategy, refer to: TODO: Strategy API url
- strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)
- report_normal, positions_normal = normal_backtest(pred_score, strategy=strategy, **BACKTEST_CONFIG)
-
- ###################################
- # analyze
- # If need a more detailed analysis, refer to: examples/train_and_bakctest.ipynb
- ###################################
- analysis = dict()
- analysis["excess_return_without_cost"] = risk_analysis(report_normal["return"] - report_normal["bench"])
- analysis["excess_return_with_cost"] = risk_analysis(
- report_normal["return"] - report_normal["bench"] - report_normal["cost"]
- )
- analysis_df = pd.concat(analysis) # type: pd.DataFrame
- print(analysis_df)
diff --git a/examples/workflow_by_code_lstm.py b/examples/workflow_by_code_lstm.py
deleted file mode 100644
index ee50c9aff..000000000
--- a/examples/workflow_by_code_lstm.py
+++ /dev/null
@@ -1,144 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-import sys
-from pathlib import Path
-
-import qlib
-import pandas as pd
-from qlib.config import REG_CN
-from qlib.contrib.model.pytorch_lstm import LSTM
-from qlib.contrib.data.handler import ALPHA360_Denoise
-from qlib.contrib.strategy.strategy import TopkDropoutStrategy
-from qlib.contrib.evaluate import (
- backtest as normal_backtest,
- risk_analysis,
-)
-from qlib.utils import exists_qlib_data
-
-# from qlib.model.learner import train_model
-from qlib.utils import init_instance_by_config
-
-import pickle
-
-if __name__ == "__main__":
-
- # use default data
- provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
- if not exists_qlib_data(provider_uri):
- print(f"Qlib data is not found in {provider_uri}")
- sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts")))
- from get_data import GetData
-
- GetData().qlib_data(target_dir=provider_uri, region=REG_CN)
-
- qlib.init(provider_uri=provider_uri, region=REG_CN)
-
- MARKET = "csi300"
- BENCHMARK = "SH000300"
-
- ###################################
- # train model
- ###################################
- DATA_HANDLER_CONFIG = {
- "start_time": "2008-01-01",
- "end_time": "2020-08-01",
- "fit_start_time": "2008-01-01",
- "fit_end_time": "2014-12-31",
- "instruments": MARKET,
- }
-
- TRAINER_CONFIG = {
- "train_start_time": "2008-01-01",
- "train_end_time": "2014-12-31",
- "validate_start_time": "2015-01-01",
- "validate_end_time": "2016-12-31",
- "test_start_time": "2017-01-01",
- "test_end_time": "2020-08-01",
- }
-
- task = {
- "model": {
- "class": "LSTM",
- "module_path": "qlib.contrib.model.pytorch_lstm",
- "kwargs": {
- "d_feat": 6,
- "hidden_size": 64,
- "num_layers": 2,
- "dropout": 0.0,
- "n_epochs": 200,
- "lr": 1e-3,
- "early_stop": 20,
- "batch_size": 800,
- "metric": "IC",
- "loss": "mse",
- "seed": 0,
- "GPU": 0,
- },
- },
- "dataset": {
- "class": "DatasetH",
- "module_path": "qlib.data.dataset",
- "kwargs": {
- "handler": {
- "class": "ALPHA360_Denoise",
- "module_path": "qlib.contrib.data.handler",
- "kwargs": DATA_HANDLER_CONFIG,
- },
- "segments": {
- "train": ("2008-01-01", "2014-12-31"),
- "valid": ("2015-01-01", "2016-12-31"),
- "test": ("2017-01-01", "2020-08-01"),
- },
- },
- }
- # You shoud record the data in specific sequence
- # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
- }
-
- # model = train_model(task)
- model = init_instance_by_config(task["model"])
- dataset = init_instance_by_config(task["dataset"])
- model.fit(dataset)
-
- pred_score = model.predict(dataset)
-
- # save pred_score to file
- pred_score_path = Path("~/tmp/qlib/pred_score.pkl").expanduser()
- pred_score_path.parent.mkdir(exist_ok=True, parents=True)
- pred_score.to_pickle(pred_score_path)
-
- ###################################
- # backtest
- ###################################
- STRATEGY_CONFIG = {
- "topk": 50,
- "n_drop": 5,
- }
- BACKTEST_CONFIG = {
- "verbose": False,
- "limit_threshold": 0.095,
- "account": 100000000,
- "benchmark": BENCHMARK,
- "deal_price": "close",
- "open_cost": 0.0005,
- "close_cost": 0.0015,
- "min_cost": 5,
- }
-
- # use default strategy
- # custom Strategy, refer to: TODO: Strategy API url
- strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)
- report_normal, positions_normal = normal_backtest(pred_score, strategy=strategy, **BACKTEST_CONFIG)
-
- ###################################
- # analyze
- # If need a more detailed analysis, refer to: examples/train_and_bakctest.ipynb
- ###################################
- analysis = dict()
- analysis["excess_return_without_cost"] = risk_analysis(report_normal["return"] - report_normal["bench"])
- analysis["excess_return_with_cost"] = risk_analysis(
- report_normal["return"] - report_normal["bench"] - report_normal["cost"]
- )
- analysis_df = pd.concat(analysis) # type: pd.DataFrame
- print(analysis_df)
diff --git a/examples/workflow_by_code_sfm.py b/examples/workflow_by_code_sfm.py
deleted file mode 100644
index 5bd91ded8..000000000
--- a/examples/workflow_by_code_sfm.py
+++ /dev/null
@@ -1,158 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-from pathlib import Path
-
-import qlib
-import pandas as pd
-from qlib.config import REG_CN
-from qlib.contrib.model.pytorch_gru import GRU
-from qlib.contrib.data.handler import ALPHA360_Denoise
-from qlib.contrib.strategy.strategy import TopkDropoutStrategy
-from qlib.contrib.evaluate import (
- backtest as normal_backtest,
- risk_analysis,
-)
-from qlib.utils import exists_qlib_data
-from qlib.utils import init_instance_by_config
-
-import pickle
-
-if __name__ == "__main__":
-
- # use default data
- provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
- if not exists_qlib_data(provider_uri):
- print(f"Qlib data is not found in {provider_uri}")
- sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts")))
- from get_data import GetData
-
- GetData().qlib_data(target_dir=provider_uri, region=REG_CN)
-
- qlib.init(provider_uri=provider_uri, region=REG_CN)
-
- MARKET = "csi300"
- BENCHMARK = "SH000300"
-
- ###################################
- # train model
- ###################################
- DATA_HANDLER_CONFIG = {
- "start_time": "2008-01-01",
- "end_time": "2020-08-01",
- "fit_start_time": "2008-01-01",
- "fit_end_time": "2014-12-31",
- "instruments": MARKET,
- }
-
- TRAINER_CONFIG = {
- "train_start_time": "2008-01-01",
- "train_end_time": "2014-12-31",
- "validate_start_time": "2015-01-01",
- "validate_end_time": "2016-12-31",
- "test_start_time": "2017-01-01",
- "test_end_time": "2020-08-01",
- }
-
- task = {
- "model": {
- "class": "SFM",
- "module_path": "qlib.contrib.model.pytorch_sfm",
- "kwargs": {
- "d_feat": 6,
- "hidden_size": 64,
- "output_dim": 32,
- "freq_dim": 25,
- "dropout_W": 0.5,
- "dropout_U": 0.5,
- "n_epochs": 15,
- "lr": 1e-3,
- "metric": "",
- "batch_size": 1600,
- "early_stop": 20,
- "eval_steps": 5,
- "loss": "mse",
- "lr_decay": 0.96,
- "lr_decay_steps": 100,
- "optimizer": "adam",
- "GPU": 3,
- "seed": 710,
- },
- },
- "dataset": {
- "class": "DatasetH",
- "module_path": "qlib.data.dataset",
- "kwargs": {
- "handler": {
- "class": "ALPHA360_Denoise",
- "module_path": "qlib.contrib.data.handler",
- "kwargs": DATA_HANDLER_CONFIG,
- },
- "segments": {
- "train": ("2008-01-01", "2014-12-31"),
- "valid": ("2015-01-01", "2016-12-31"),
- "test": ("2017-01-01", "2020-08-01"),
- },
- },
- }
- # You shoud record the data in specific sequence
- # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
- }
-
- # model = train_model(task)
- model = init_instance_by_config(task["model"])
- dataset = init_instance_by_config(task["dataset"])
- model.fit(dataset)
-
- pred_score = model.predict(dataset)
-
- # save pred_score to file
- pred_score_path = Path("~/tmp/qlib/pred_score.pkl").expanduser()
- pred_score_path.parent.mkdir(exist_ok=True, parents=True)
- pred_score.to_pickle(pred_score_path)
-
- ###################################
- # backtest
- ###################################
- STRATEGY_CONFIG = {
- "topk": 50,
- "n_drop": 5,
- }
- BACKTEST_CONFIG = {
- "verbose": False,
- "limit_threshold": 0.095,
- "account": 100000000,
- "benchmark": BENCHMARK,
- "deal_price": "close",
- "open_cost": 0.0005,
- "close_cost": 0.0015,
- "min_cost": 5,
- }
-
- # use default strategy
- # custom Strategy, refer to: TODO: Strategy API url
- strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)
- report_normal, positions_normal = normal_backtest(pred_score, strategy=strategy, **BACKTEST_CONFIG)
-
- ###################################
- # analyze
- # If need a more detailed analysis, refer to: examples/train_and_bakctest.ipynb
- ###################################
- analysis = dict()
- analysis["excess_return_without_cost"] = risk_analysis(report_normal["return"] - report_normal["bench"])
- analysis["excess_return_with_cost"] = risk_analysis(
- report_normal["return"] - report_normal["bench"] - report_normal["cost"]
- )
- analysis_df = pd.concat(analysis) # type: pd.DataFrame
- print(analysis_df)
diff --git a/examples/workflow_by_code_tabnet.py b/examples/workflow_by_code_tabnet.py
deleted file mode 100644
index 3778b9d59..000000000
--- a/examples/workflow_by_code_tabnet.py
+++ /dev/null
@@ -1,142 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-import sys
-from pathlib import Path
-
-import qlib
-import pandas as pd
-from qlib.config import REG_CN
-from qlib.contrib.model.tabnet import TabNetModel
-from qlib.contrib.data.handler import ALPHA360_Denoise
-from qlib.contrib.strategy.strategy import TopkDropoutStrategy
-from qlib.contrib.evaluate import (
- backtest as normal_backtest,
- risk_analysis,
-)
-from qlib.utils import exists_qlib_data
-
-# from qlib.model.learner import train_model
-from qlib.utils import init_instance_by_config
-
-import pickle
-
-if __name__ == "__main__":
-
- # use default data
- provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
- if not exists_qlib_data(provider_uri):
- print(f"Qlib data is not found in {provider_uri}")
- sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts")))
- from get_data import GetData
-
- GetData().qlib_data(target_dir=provider_uri, region=REG_CN)
-
- qlib.init(provider_uri=provider_uri, region=REG_CN)
-
- MARKET = "csi300"
- BENCHMARK = "SH000300"
-
- ###################################
- # train model
- ###################################
- DATA_HANDLER_CONFIG = {
- "start_time": "2008-01-01",
- "end_time": "2020-08-01",
- "fit_start_time": "2008-01-01",
- "fit_end_time": "2014-12-31",
- "instruments": MARKET,
- }
-
- TRAINER_CONFIG = {
- "train_start_time": "2008-01-01",
- "train_end_time": "2014-12-31",
- "validate_start_time": "2015-01-01",
- "validate_end_time": "2016-12-31",
- "test_start_time": "2017-01-01",
- "test_end_time": "2020-08-01",
- }
-
- task = {
- "model": {
- "class": "TabNetModel",
- "module_path": "qlib.contrib.model.tabnet",
- "kwargs": {
- "n_d": 8,
- "n_a": 8,
- "n_steps": 3,
- "gamma": 1.3,
- "n_independent": 2,
- "n_shared": 2,
- "seed": 0,
- "momentum": 0.02,
- "lambda_sparse": 1e-3,
- "optimizer_params": {"lr": 2e-3},
- },
- },
- "dataset": {
- "class": "DatasetH",
- "module_path": "qlib.data.dataset",
- "kwargs": {
- "handler": {
- "class": "ALPHA360_Denoise",
- "module_path": "qlib.contrib.data.handler",
- "kwargs": DATA_HANDLER_CONFIG,
- },
- "segments": {
- "train": ("2008-01-01", "2014-12-31"),
- "valid": ("2015-01-01", "2016-12-31"),
- "test": ("2017-01-01", "2020-08-01"),
- },
- },
- }
- # You shoud record the data in specific sequence
- # "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
- }
-
- # model = train_model(task)
- model = init_instance_by_config(task["model"])
- dataset = init_instance_by_config(task["dataset"])
- model.fit(dataset)
-
- pred_score = model.predict(dataset)
-
- # save pred_score to file
- pred_score_path = Path("~/tmp/qlib/pred_score.pkl").expanduser()
- pred_score_path.parent.mkdir(exist_ok=True, parents=True)
- pred_score.to_pickle(pred_score_path)
-
- ###################################
- # backtest
- ###################################
- STRATEGY_CONFIG = {
- "topk": 50,
- "n_drop": 5,
- }
- BACKTEST_CONFIG = {
- "verbose": False,
- "limit_threshold": 0.095,
- "account": 100000000,
- "benchmark": BENCHMARK,
- "deal_price": "close",
- "open_cost": 0.0005,
- "close_cost": 0.0015,
- "min_cost": 5,
- }
-
- # use default strategy
- # custom Strategy, refer to: TODO: Strategy API url
- strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)
- report_normal, positions_normal = normal_backtest(pred_score, strategy=strategy, **BACKTEST_CONFIG)
-
- ###################################
- # analyze
- # If need a more detailed analysis, refer to: examples/train_and_bakctest.ipynb
- ###################################
- analysis = dict()
- analysis["excess_return_without_cost"] = risk_analysis(report_normal["return"] - report_normal["bench"])
- analysis["excess_return_with_cost"] = risk_analysis(
- report_normal["return"] - report_normal["bench"] - report_normal["cost"]
- )
- analysis_df = pd.concat(analysis) # type: pd.DataFrame
- print(analysis_df)
diff --git a/qlib/contrib/model/catboost_model.py b/qlib/contrib/model/catboost_model.py
index bba006c35..01830d1b5 100644
--- a/qlib/contrib/model/catboost_model.py
+++ b/qlib/contrib/model/catboost_model.py
@@ -41,7 +41,9 @@ class CatBoostModel(Model):
**kwargs
):
df_train, df_valid = dataset.prepare(
- ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
+ ["train", "valid"],
+ col_set=["feature", "label"],
+ data_key=DataHandlerLP.DK_L,
)
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]
diff --git a/qlib/contrib/model/pytorch_alstm.py b/qlib/contrib/model/pytorch_alstm.py
index 1b23d2401..40c2f8226 100644
--- a/qlib/contrib/model/pytorch_alstm.py
+++ b/qlib/contrib/model/pytorch_alstm.py
@@ -11,7 +11,12 @@ import pandas as pd
import copy
from sklearn.metrics import roc_auc_score, mean_squared_error
import logging
-from ...utils import unpack_archive_with_buffer, save_multiple_parts_file, create_save_path, drop_nan_by_y_index
+from ...utils import (
+ unpack_archive_with_buffer,
+ save_multiple_parts_file,
+ create_save_path,
+ drop_nan_by_y_index,
+)
from ...log import get_module_logger, TimeInspector
import torch
@@ -109,7 +114,10 @@ class ALSTM(Model):
)
self.ALSTM_model = ALSTMModel(
- d_feat=self.d_feat, hidden_size=self.hidden_size, num_layers=self.num_layers, dropout=self.dropout
+ d_feat=self.d_feat,
+ hidden_size=self.hidden_size,
+ num_layers=self.num_layers,
+ dropout=self.dropout,
)
if optimizer.lower() == "adam":
self.train_optimizer = optim.Adam(self.ALSTM_model.parameters(), lr=self.lr)
@@ -141,7 +149,7 @@ class ALSTM(Model):
mask = torch.isfinite(label)
- if self.metric == "" or self.metric == "loss": # use loss
+ if self.metric == "" or self.metric == "loss":
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)
@@ -219,7 +227,9 @@ class ALSTM(Model):
):
df_train, df_valid, df_test = dataset.prepare(
- ["train", "valid", "test"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
+ ["train", "valid", "test"],
+ col_set=["feature", "label"],
+ data_key=DataHandlerLP.DK_L,
)
x_train, y_train = df_train["feature"], df_train["label"]
@@ -328,10 +338,16 @@ class ALSTMModel(nn.Module):
)
self.fc_out = nn.Linear(in_features=self.hid_size * 2, out_features=1)
self.att_net = nn.Sequential()
- self.att_net.add_module("att_fc_in", nn.Linear(in_features=self.hid_size, out_features=int(self.hid_size / 2)))
+ self.att_net.add_module(
+ "att_fc_in",
+ nn.Linear(in_features=self.hid_size, out_features=int(self.hid_size / 2)),
+ )
self.att_net.add_module("att_dropout", torch.nn.Dropout(self.dropout))
self.att_net.add_module("att_act", nn.Tanh())
- self.att_net.add_module("att_fc_out", nn.Linear(in_features=int(self.hid_size / 2), out_features=1, bias=False))
+ self.att_net.add_module(
+ "att_fc_out",
+ nn.Linear(in_features=int(self.hid_size / 2), out_features=1, bias=False),
+ )
self.att_net.add_module("att_softmax", nn.Softmax(dim=1))
def forward(self, inputs):
diff --git a/qlib/contrib/model/pytorch_gats.py b/qlib/contrib/model/pytorch_gats.py
old mode 100755
new mode 100644
index 77a02a9b2..e9cbcf9cb
--- a/qlib/contrib/model/pytorch_gats.py
+++ b/qlib/contrib/model/pytorch_gats.py
@@ -12,6 +12,7 @@ import copy
from ...utils import create_save_path
from ...log import get_module_logger
+
import torch
import torch.nn as nn
import torch.optim as optim
@@ -19,10 +20,12 @@ import torch.optim as optim
from ...model.base import Model
from ...data.dataset import DatasetH
from ...data.dataset.handler import DataHandlerLP
+from ...contrib.model.pytorch_lstm import LSTMModel
+from ...contrib.model.pytorch_gru import GRUModel
-class GAT(Model):
- """GAT Model
+class GATs(Model):
+ """GATs Model
Parameters
----------
@@ -57,8 +60,8 @@ class GAT(Model):
**kwargs
):
# Set logger.
- self.logger = get_module_logger("GAT")
- self.logger.info("GAT pytorch version...")
+ self.logger = get_module_logger("GATs")
+ self.logger.info("GATs pytorch version...")
# set hyper-parameters.
self.d_feat = d_feat
@@ -78,7 +81,7 @@ class GAT(Model):
self.seed = seed
self.logger.info(
- "GAT parameters setting:"
+ "GATs parameters setting:"
"\nd_feat : {}"
"\nhidden_size : {}"
"\nnum_layers : {}"
@@ -149,18 +152,18 @@ class GAT(Model):
mask = torch.isfinite(label)
- if self.metric == "" or self.metric == "loss": # use loss
+ if self.metric == "" or self.metric == "loss":
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)
def get_daily_inter(self, df, shuffle=False):
- # organize the train data into daily inter as daily batches
+ # organize the train data into daily batches
daily_count = df.groupby(level=0).size().values
daily_index = np.roll(np.cumsum(daily_count), 1)
daily_index[0] = 0
if shuffle:
- # shuffle the daily inter data
+ # shuffle data
daily_shuffle = list(zip(daily_index, daily_count))
np.random.shuffle(daily_shuffle)
daily_index, daily_count = zip(*daily_shuffle)
@@ -172,7 +175,7 @@ class GAT(Model):
y_train_values = np.squeeze(y_train.values)
self.GAT_model.train()
- # organize the train data into daily inter as daily batches
+ # organize the train data into daily batches
daily_index, daily_count = self.get_daily_inter(x_train, shuffle=True)
for idx, count in zip(daily_index, daily_count):
@@ -203,7 +206,7 @@ class GAT(Model):
scores = []
losses = []
- # organize the test data into daily inter as daily batches
+ # organize the test data into daily batches
daily_index, daily_count = self.get_daily_inter(data_x, shuffle=False)
for idx, count in zip(daily_index, daily_count):
@@ -233,7 +236,9 @@ class GAT(Model):
):
df_train, df_valid, df_test = dataset.prepare(
- ["train", "valid", "test"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
+ ["train", "valid", "test"],
+ col_set=["feature", "label"],
+ data_key=DataHandlerLP.DK_L,
)
x_train, y_train = df_train["feature"], df_train["label"]
@@ -251,15 +256,13 @@ class GAT(Model):
if self.with_pretrain:
self.logger.info("Loading pretrained model...")
if self.base_model == "LSTM":
- from ...contrib.model.pytorch_lstm import LSTMModel
-
pretrained_model = LSTMModel()
pretrained_model.load_state_dict(torch.load("benchmarks/LSTM/model_lstm_csi300.pkl"))
- elif self.base_model == "GRU":
- from ...contrib.model.pytorch_gru import GRUModel
+ elif self.base_model == "GRU":
pretrained_model = GRUModel()
pretrained_model.load_state_dict(torch.load("benchmarks/GRU/model_gru_csi300.pkl"))
+
model_dict = self.GAT_model.state_dict()
pretrained_dict = {k: v for k, v in pretrained_model.state_dict().items() if k in model_dict}
model_dict.update(pretrained_dict)
@@ -269,7 +272,6 @@ class GAT(Model):
# train
self.logger.info("training...")
self._fitted = True
- # return
for step in range(self.n_epochs):
self.logger.info("Epoch%d:", step)
@@ -310,7 +312,7 @@ class GAT(Model):
x_values = x_test.values
preds = []
- # organize the data into daily inter as daily batches
+ # organize the data into daily batches
daily_index, daily_count = self.get_daily_inter(x_test, shuffle=False)
for idx, count in zip(daily_index, daily_count):
@@ -355,22 +357,29 @@ class GATModel(nn.Module):
raise ValueError("unknown base model name `%s`" % base_model)
self.hidden_size = hidden_size
- self.bn1 = nn.BatchNorm1d(num_features=hidden_size, track_running_stats=False)
- self.fc = nn.Linear(hidden_size, hidden_size)
- self.bn2 = nn.BatchNorm1d(num_features=hidden_size, track_running_stats=False)
+ self.d_feat = d_feat
+ self.transformation = nn.Linear(self.hidden_size, self.hidden_size)
+ self.a = nn.Parameter(torch.randn(self.hidden_size * 2, 1))
+ self.a.requires_grad = True
+ self.fc = nn.Linear(self.hidden_size, self.hidden_size)
self.fc_out = nn.Linear(hidden_size, 1)
self.leaky_relu = nn.LeakyReLU()
self.softmax = nn.Softmax(dim=1)
- self.d_feat = d_feat
- def cal_convariance(self, x, y): # the 2nd dimension of x and y are the same
- e_x = torch.mean(x, dim=1).reshape(-1, 1)
- e_y = torch.mean(y, dim=1).reshape(-1, 1)
- e_x_e_y = e_x.mm(torch.t(e_y))
- x_extend = x.reshape(x.shape[0], 1, x.shape[1]).repeat(1, y.shape[0], 1)
- y_extend = y.reshape(1, y.shape[0], y.shape[1]).repeat(x.shape[0], 1, 1)
- e_xy = torch.mean(x_extend * y_extend, dim=2)
- return e_xy - e_x_e_y
+ def cal_attention(self, x, y):
+ x = self.transformation(x)
+ y = self.transformation(y)
+
+ sample_num = x.shape[0]
+ dim = x.shape[1]
+ e_x = x.expand(sample_num, sample_num, dim)
+ e_y = torch.transpose(e_x, 0, 1)
+ attention_in = torch.cat((e_x, e_y), 2).view(-1, dim * 2)
+ self.a_t = torch.t(self.a)
+ attention_out = self.a_t.mm(torch.t(attention_in)).view(sample_num, sample_num)
+ attention_out = self.leaky_relu(attention_out)
+ att_weight = self.softmax(attention_out)
+ return att_weight
def forward(self, x):
# x: [N, F*T]
@@ -378,10 +387,8 @@ class GATModel(nn.Module):
x = x.permute(0, 2, 1) # [N, T, F]
out, _ = self.rnn(x)
hidden = out[:, -1, :]
- hidden = self.bn1(hidden)
- gamma = self.cal_convariance(hidden, hidden)
- output = gamma.mm(hidden)
- output = self.fc(output)
- output = self.bn2(output)
- output = self.leaky_relu(output)
- return self.fc_out(output).squeeze()
+ att_weight = self.cal_attention(hidden, hidden)
+ hidden = att_weight.mm(hidden) + hidden
+ hidden = self.fc(hidden)
+ hidden = self.leaky_relu(hidden)
+ return self.fc_out(hidden).squeeze()
diff --git a/qlib/contrib/model/pytorch_gru.py b/qlib/contrib/model/pytorch_gru.py
index 02664b6ac..5daf4707e 100755
--- a/qlib/contrib/model/pytorch_gru.py
+++ b/qlib/contrib/model/pytorch_gru.py
@@ -11,7 +11,12 @@ import pandas as pd
import copy
from sklearn.metrics import roc_auc_score, mean_squared_error
import logging
-from ...utils import unpack_archive_with_buffer, save_multiple_parts_file, create_save_path, drop_nan_by_y_index
+from ...utils import (
+ unpack_archive_with_buffer,
+ save_multiple_parts_file,
+ create_save_path,
+ drop_nan_by_y_index,
+)
from ...log import get_module_logger, TimeInspector
import torch
@@ -109,7 +114,10 @@ class GRU(Model):
)
self.gru_model = GRUModel(
- d_feat=self.d_feat, hidden_size=self.hidden_size, num_layers=self.num_layers, dropout=self.dropout
+ d_feat=self.d_feat,
+ hidden_size=self.hidden_size,
+ num_layers=self.num_layers,
+ dropout=self.dropout,
)
if optimizer.lower() == "adam":
self.train_optimizer = optim.Adam(self.gru_model.parameters(), lr=self.lr)
@@ -141,7 +149,7 @@ class GRU(Model):
mask = torch.isfinite(label)
- if self.metric == "" or self.metric == "loss": # use loss
+ if self.metric == "" or self.metric == "loss":
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)
@@ -219,7 +227,9 @@ class GRU(Model):
):
df_train, df_valid, df_test = dataset.prepare(
- ["train", "valid", "test"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
+ ["train", "valid", "test"],
+ col_set=["feature", "label"],
+ data_key=DataHandlerLP.DK_L,
)
x_train, y_train = df_train["feature"], df_train["label"]
diff --git a/qlib/contrib/model/pytorch_hats.py b/qlib/contrib/model/pytorch_hats.py
deleted file mode 100644
index 7affea73c..000000000
--- a/qlib/contrib/model/pytorch_hats.py
+++ /dev/null
@@ -1,491 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-from __future__ import division
-from __future__ import print_function
-
-import os
-import numpy as np
-import pandas as pd
-import copy
-from ...utils import create_save_path
-from ...log import get_module_logger
-
-import torch
-import torch.nn as nn
-import torch.optim as optim
-
-from ...model.base import Model
-from ...data.dataset import DatasetH
-from ...data.dataset.handler import DataHandlerLP
-
-
-class HATS(Model):
- """HATS Model
-
- Parameters
- ----------
- d_feat : int
- input dimension for each time step
- metric: str
- the evaluate metric used in early stop
- optimizer : str
- optimizer name
- GPU : str
- the GPU ID(s) used for training
- """
-
- def __init__(
- self,
- d_feat=6,
- hidden_size=64,
- num_layers=2,
- dropout=0.5,
- n_epochs=200,
- lr=0.01,
- metric="",
- early_stop=20,
- loss="mse",
- base_model="GRU",
- with_pretrain=True,
- optimizer="adam",
- GPU="0",
- seed=0,
- **kwargs
- ):
- # Set logger.
- self.logger = get_module_logger("HATS")
- self.logger.info("HATS pytorch version...")
-
- # set hyper-parameters.
- self.d_feat = d_feat
- self.hidden_size = hidden_size
- self.num_layers = num_layers
- self.dropout = dropout
- self.n_epochs = n_epochs
- self.lr = lr
- self.metric = metric
- self.early_stop = early_stop
- self.optimizer = optimizer.lower()
- self.loss = loss
- self.base_model = base_model
- self.with_pretrain = with_pretrain
- self.visible_GPU = GPU
- self.use_gpu = torch.cuda.is_available()
- self.seed = seed
-
- self.logger.info(
- "HATS parameters setting:"
- "\nd_feat : {}"
- "\nhidden_size : {}"
- "\nnum_layers : {}"
- "\ndropout : {}"
- "\nn_epochs : {}"
- "\nlr : {}"
- "\nmetric : {}"
- "\nearly_stop : {}"
- "\noptimizer : {}"
- "\nloss_type : {}"
- "\nbase_model : {}"
- "\nwith_pretrain : {}"
- "\nvisible_GPU : {}"
- "\nuse_GPU : {}"
- "\nseed : {}".format(
- d_feat,
- hidden_size,
- num_layers,
- dropout,
- n_epochs,
- lr,
- metric,
- early_stop,
- optimizer.lower(),
- loss,
- base_model,
- with_pretrain,
- GPU,
- self.use_gpu,
- seed,
- )
- )
-
- self.HATS_model = HATSModel(
- d_feat=self.d_feat,
- hidden_size=self.hidden_size,
- num_layers=self.num_layers,
- dropout=self.dropout,
- base_model=self.base_model,
- )
- if optimizer.lower() == "adam":
- self.train_optimizer = optim.Adam(self.HATS_model.parameters(), lr=self.lr)
- elif optimizer.lower() == "gd":
- self.train_optimizer = optim.SGD(self.HATS_model.parameters(), lr=self.lr)
- else:
- raise NotImplementedError("optimizer {} is not supported!".format(optimizer))
-
- self._fitted = False
- if self.use_gpu:
- self.HATS_model.cuda()
- # set the visible GPU
- if self.visible_GPU:
- os.environ["CUDA_VISIBLE_DEVICES"] = self.visible_GPU
-
- def mse(self, pred, label):
- loss = (pred - label) ** 2
- return torch.mean(loss)
-
- def loss_fn(self, pred, label):
- mask = ~torch.isnan(label)
-
- if self.loss == "mse":
- return self.mse(pred[mask], label[mask])
-
- raise ValueError("unknown loss `%s`" % self.loss)
-
- def metric_fn(self, pred, label):
- mask = torch.isfinite(label)
-
- if self.metric == "" or self.metric == "loss": # use loss
- return -self.loss_fn(pred[mask], label[mask])
-
- raise ValueError("unknown metric `%s`" % self.metric)
-
- def get_daily_inter(self, df, shuffle=False):
- # organize the train data into daily inter as daily batches
- daily_count = df.groupby(level=0).size().values
- daily_index = np.roll(np.cumsum(daily_count), 1)
- daily_index[0] = 0
- if shuffle:
- # shuffle the daily inter data
- daily_shuffle = list(zip(daily_index, daily_count))
- np.random.shuffle(daily_shuffle)
- daily_index, daily_count = zip(*daily_shuffle)
- return daily_index, daily_count
-
- def train_epoch(self, x_train, y_train):
-
- x_train_values = x_train.values
- y_train_values = np.squeeze(y_train.values)
-
- self.HATS_model.train()
-
- # organize the train data into daily inter as daily batches
- daily_index, daily_count = self.get_daily_inter(x_train, shuffle=True)
-
- for idx, count in zip(daily_index, daily_count):
- batch = slice(idx, idx + count)
- feature = torch.from_numpy(x_train_values[batch]).float()
- label = torch.from_numpy(y_train_values[batch]).float()
-
- if self.use_gpu:
- feature = feature.cuda()
- label = label.cuda()
-
- pred = self.HATS_model(feature)
- loss = self.loss_fn(pred, label)
-
- self.train_optimizer.zero_grad()
- loss.backward()
- torch.nn.utils.clip_grad_value_(self.HATS_model.parameters(), 3.0)
- self.train_optimizer.step()
-
- def test_epoch(self, data_x, data_y):
-
- # prepare testing data
- x_values = data_x.values
- y_values = np.squeeze(data_y.values)
-
- self.HATS_model.eval()
-
- scores = []
- losses = []
-
- # organize the test data into daily inter as daily batches
- daily_index, daily_count = self.get_daily_inter(data_x, shuffle=False)
-
- for idx, count in zip(daily_index, daily_count):
- batch = slice(idx, idx + count)
- feature = torch.from_numpy(x_values[batch]).float()
- label = torch.from_numpy(y_values[batch]).float()
-
- if self.use_gpu:
- feature = feature.cuda()
- label = label.cuda()
-
- pred = self.HATS_model(feature)
- loss = self.loss_fn(pred, label)
- losses.append(loss.item())
-
- score = self.metric_fn(pred, label)
- scores.append(score.item())
-
- return np.mean(losses), np.mean(scores)
-
- def fit(
- self,
- dataset: DatasetH,
- evals_result=dict(),
- verbose=True,
- save_path=None,
- ):
-
- df_train, df_valid, df_test = dataset.prepare(
- ["train", "valid", "test"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
- )
-
- x_train, y_train = df_train["feature"], df_train["label"]
- x_valid, y_valid = df_valid["feature"], df_valid["label"]
-
- if save_path == None:
- save_path = create_save_path(save_path)
- stop_steps = 0
- best_score = -np.inf
- best_epoch = 0
- evals_result["train"] = []
- evals_result["valid"] = []
-
- # load pretrained base_model
- if self.with_pretrain:
- self.logger.info("Loading pretrained model...")
- if self.base_model == "LSTM":
- from ...contrib.model.pytorch_lstm import LSTMModel
-
- pretrained_model = LSTMModel()
- pretrained_model.load_state_dict(torch.load("benchmarks/LSTM/model_lstm_csi300.pkl"))
- elif self.base_model == "GRU":
- from ...contrib.model.pytorch_gru import GRUModel
-
- pretrained_model = GRUModel()
- pretrained_model.load_state_dict(torch.load("benchmarks/GRU/model_gru_csi300.pkl"))
- model_dict = self.HATS_model.state_dict()
- pretrained_dict = {k: v for k, v in pretrained_model.state_dict().items() if k in model_dict}
- model_dict.update(pretrained_dict)
- self.HATS_model.load_state_dict(model_dict)
- self.logger.info("Loading pretrained model Done...")
-
- # train
- self.logger.info("training...")
- self._fitted = True
-
- for step in range(self.n_epochs):
- self.logger.info("Epoch%d:", step)
- self.logger.info("training...")
- self.train_epoch(x_train, y_train)
- self.logger.info("evaluating...")
- train_loss, train_score = self.test_epoch(x_train, y_train)
- val_loss, val_score = self.test_epoch(x_valid, y_valid)
- self.logger.info("train %.6f, valid %.6f" % (train_score, val_score))
- evals_result["train"].append(train_score)
- evals_result["valid"].append(val_score)
-
- if val_score > best_score:
- best_score = val_score
- stop_steps = 0
- best_epoch = step
- best_param = copy.deepcopy(self.HATS_model.state_dict())
- else:
- stop_steps += 1
- if stop_steps >= self.early_stop:
- self.logger.info("early stop")
- break
-
- self.logger.info("best score: %.6lf @ %d" % (best_score, best_epoch))
- self.HATS_model.load_state_dict(best_param)
- torch.save(best_param, save_path)
-
- if self.use_gpu:
- torch.cuda.empty_cache()
-
- def predict(self, dataset):
- if not self._fitted:
- raise ValueError("model is not fitted yet!")
-
- x_test = dataset.prepare("test", col_set="feature")
- index = x_test.index
- self.HATS_model.eval()
- x_values = x_test.values
- sample_num = x_values.shape[0]
- preds = []
-
- # organize the data into daily inter as daily batches
- daily_index, daily_count = self.get_daily_inter(x_test, shuffle=False)
-
- for idx, count in zip(daily_index, daily_count):
- batch = slice(idx, idx + count)
- x_batch = torch.from_numpy(x_values[batch]).float()
-
- if self.use_gpu:
- x_batch = x_batch.cuda()
-
- with torch.no_grad():
- if self.use_gpu:
- pred = self.HATS_model(x_batch).detach().cpu().numpy()
- else:
- pred = self.HATS_model(x_batch).detach().numpy()
-
- preds.append(pred)
-
- return pd.Series(np.concatenate(preds), index=index)
-
-
-class HATSModel(nn.Module):
- def __init__(self, d_feat=6, hidden_size=64, num_layers=2, dropout=0.0, base_model="GRU"):
- super().__init__()
-
- if base_model == "GRU":
- self.model = nn.GRU(
- input_size=d_feat,
- hidden_size=hidden_size,
- num_layers=num_layers,
- batch_first=True,
- dropout=dropout,
- )
- elif base_model == "LSTM":
- self.model = nn.LSTM(
- input_size=d_feat,
- hidden_size=hidden_size,
- num_layers=num_layers,
- batch_first=True,
- dropout=dropout,
- )
- else:
- raise ValueError("unknown base model name `%s`" % base_model)
-
- self.hidden_size = hidden_size
- self.bn1 = nn.BatchNorm1d(num_features=hidden_size, track_running_stats=False)
- self.fc = nn.Linear(hidden_size, hidden_size)
- self.bn2 = nn.BatchNorm1d(num_features=hidden_size, track_running_stats=False)
- self.fc_out = nn.Linear(hidden_size, 1)
- self.leaky_relu = nn.LeakyReLU()
- self.softmax = nn.Softmax(dim=1)
- self.d_feat = d_feat
-
- num_head_att = [1] * num_layers
- hidden_dim = [hidden_size] * num_layers
- dims = [d_feat] + [d * nh for (d, nh) in zip(hidden_dim, num_head_att[:-1])] + [num_head_att[-1]]
- in_dims = dims[:-1]
- out_dims = [d // nh for (d, nh) in zip(dims[1:], num_head_att)]
- self.attn = nn.ModuleList(
- [GraphAttention(i, o, nh, dropout) for (i, o, nh) in zip(in_dims, out_dims, num_head_att)]
- )
- self.bns = nn.ModuleList([nn.BatchNorm1d(dim) for dim in dims[1:-1]])
- self.dropout = nn.Dropout(dropout)
- self.elu = nn.ELU()
-
- def forward(self, x):
- x = x.reshape(len(x), self.d_feat, -1) # [N, F, T]
- x = x.permute(0, 2, 1) # [N, T, F]
- out, _ = self.model(x)
- hidden = out[:, -1, :]
- hidden = self.bn1(hidden)
- attention = GraphAttention.cal_attention(hidden, hidden)
- output = attention.mm(hidden)
- output = self.fc(output)
- output = self.bn2(output)
- output = self.leaky_relu(output)
- return self.fc_out(output).squeeze()
-
-
-class GraphAttention(nn.Module):
- def __init__(self, input_dim, output_dim, num_heads, dropout=0.5):
-
- super().__init__()
-
- """
- Parameters
- ----------
- input_dim : int
- Dimension of input node features.
- output_dim : int
- Dimension of output node features.
- num_heads : list of ints
- Number of attention heads in each hidden layer and output layer. Must be non empty. Note that len(num_heads) = len(hidden_dims)+1.
- dropout : float
- Dropout rate. Default: 0.5.
- """
-
- self.input_dim = input_dim
- self.output_dim = output_dim
- self.num_heads = num_heads
-
- self.fcs = nn.ModuleList([nn.Linear(input_dim, output_dim) for _ in range(num_heads)])
- self.a = nn.ModuleList([nn.Linear(2 * output_dim, 1) for _ in range(num_heads)])
-
- self.dropout = nn.Dropout(dropout)
- self.softmax = nn.Softmax(dim=0)
- self.leakyrelu = nn.LeakyReLU()
-
- def forward(self, features, nodes, mappings, rows):
-
- """
- Parameters
- ----------
- features : torch.Tensor
- An (n' x input_dim) tensor of input node features.
- nodes : list of numpy array
- nodes[i] is an array of the nodes in the ith layer of the
- computation graph.
- mappings : list of dictionary
- mappings[i] is a dictionary mappings node v (labelled 0 to |V|-1)
- in nodes[i] to its position in nodes[i]. For example,
- if nodes[i] = [2,5], then mappings[i][2] = 0 and
- mappings[i][5] = 1.
- rows : numpy array
- rows[i] is an array of neighbors of node i.
- Returns
- -------
- out : torch.Tensor
- An (len(node_layers[-1]) x output_dim) tensor of output node features.
- """
-
- nprime = features.shape[0]
- rows = [np.array([mappings[v] for v in row], dtype=np.int64) for row in rows]
- sum_degs = np.hstack(([0], np.cumsum([len(row) for row in rows])))
- mapped_nodes = [mappings[v] for v in nodes]
- indices = torch.LongTensor([[v, c] for (v, row) in zip(mapped_nodes, rows) for c in row]).t()
-
- out = []
- for k in range(self.num_heads):
- h = self.fcs[k](features)
-
- nbr_h = torch.cat(tuple([h[row] for row in rows]), dim=0)
- self_h = torch.cat(
- tuple([h[mappings[nodes[i]]].repeat(len(row), 1) for (i, row) in enumerate(rows)]), dim=0
- )
- cat_h = torch.cat((self_h, nbr_h), dim=1)
-
- e = self.leakyrelu(self.a[k](cat_h))
-
- alpha = [self.softmax(e[lo:hi]) for (lo, hi) in zip(sum_degs, sum_degs[1:])]
- alpha = torch.cat(tuple(alpha), dim=0)
- alpha = alpha.squeeze(1)
- alpha = self.dropout(alpha)
-
- adj = torch.sparse.FloatTensor(indices, alpha, torch.Size([nprime, nprime]))
- out.append(torch.sparse.mm(adj, h)[mapped_nodes])
-
- return out
-
- @staticmethod
- def cal_attention(x, y):
- att_x = torch.mean(x, dim=1).reshape(-1, 1)
- att_y = torch.mean(y, dim=1).reshape(-1, 1)
- att = att_x.mm(torch.t(att_y))
- return (
- torch.mean(
- x.reshape(x.shape[0], 1, x.shape[1]).repeat(1, y.shape[0], 1)
- * y.reshape(1, y.shape[0], y.shape[1]).repeat(x.shape[0], 1, 1),
- dim=2,
- )
- - att
- )
diff --git a/qlib/contrib/model/pytorch_lstm.py b/qlib/contrib/model/pytorch_lstm.py
index f8951509a..eef1680ec 100755
--- a/qlib/contrib/model/pytorch_lstm.py
+++ b/qlib/contrib/model/pytorch_lstm.py
@@ -11,7 +11,12 @@ import pandas as pd
import copy
from sklearn.metrics import roc_auc_score, mean_squared_error
import logging
-from ...utils import unpack_archive_with_buffer, save_multiple_parts_file, create_save_path, drop_nan_by_y_index
+from ...utils import (
+ unpack_archive_with_buffer,
+ save_multiple_parts_file,
+ create_save_path,
+ drop_nan_by_y_index,
+)
from ...log import get_module_logger, TimeInspector
import torch
@@ -109,7 +114,10 @@ class LSTM(Model):
)
self.lstm_model = LSTMModel(
- d_feat=self.d_feat, hidden_size=self.hidden_size, num_layers=self.num_layers, dropout=self.dropout
+ d_feat=self.d_feat,
+ hidden_size=self.hidden_size,
+ num_layers=self.num_layers,
+ dropout=self.dropout,
)
if optimizer.lower() == "adam":
self.train_optimizer = optim.Adam(self.lstm_model.parameters(), lr=self.lr)
@@ -141,7 +149,7 @@ class LSTM(Model):
mask = torch.isfinite(label)
- if self.metric == "" or self.metric == "loss": # use loss
+ if self.metric == "" or self.metric == "loss":
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)
@@ -219,7 +227,9 @@ class LSTM(Model):
):
df_train, df_valid, df_test = dataset.prepare(
- ["train", "valid", "test"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
+ ["train", "valid", "test"],
+ col_set=["feature", "label"],
+ data_key=DataHandlerLP.DK_L,
)
x_train, y_train = df_train["feature"], df_train["label"]
diff --git a/qlib/contrib/model/pytorch_sfm.py b/qlib/contrib/model/pytorch_sfm.py
index 1d27f3927..8fddd1612 100644
--- a/qlib/contrib/model/pytorch_sfm.py
+++ b/qlib/contrib/model/pytorch_sfm.py
@@ -19,7 +19,12 @@ import pandas as pd
import copy
from sklearn.metrics import roc_auc_score, mean_squared_error
import logging
-from ...utils import unpack_archive_with_buffer, save_multiple_parts_file, create_save_path, drop_nan_by_y_index
+from ...utils import (
+ unpack_archive_with_buffer,
+ save_multiple_parts_file,
+ create_save_path,
+ drop_nan_by_y_index,
+)
from ...log import get_module_logger, TimeInspector
import torch
@@ -33,7 +38,16 @@ from ...data.dataset.handler import DataHandlerLP
class SFM_Model(nn.Module):
- def __init__(self, d_feat=6, output_dim=1, freq_dim=10, hidden_size=64, dropout_W=0.0, dropout_U=0.0, device="cpu"):
+ def __init__(
+ self,
+ d_feat=6,
+ output_dim=1,
+ freq_dim=10,
+ hidden_size=64,
+ dropout_W=0.0,
+ dropout_U=0.0,
+ device="cpu",
+ ):
super().__init__()
self.input_dim = d_feat
@@ -157,7 +171,16 @@ class SFM_Model(nn.Module):
init_state_time = torch.tensor(0).to(self.device)
- self.states = [init_state_p, init_state_h, init_state_S_re, init_state_S_im, init_state_time, None, None, None]
+ self.states = [
+ init_state_p,
+ init_state_h,
+ init_state_S_re,
+ init_state_S_im,
+ init_state_time,
+ None,
+ None,
+ None,
+ ]
def get_constants(self, x):
constants = []
@@ -352,7 +375,9 @@ class SFM(Model):
):
df_train, df_valid = dataset.prepare(
- ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
+ ["train", "valid"],
+ col_set=["feature", "label"],
+ data_key=DataHandlerLP.DK_L,
)
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]
@@ -409,7 +434,7 @@ class SFM(Model):
mask = torch.isfinite(label)
- if self.metric == "" or self.metric == "loss": # use loss
+ if self.metric == "" or self.metric == "loss":
return -self.loss_fn(pred[mask], label[mask])
raise ValueError("unknown metric `%s`" % self.metric)
diff --git a/qlib/contrib/model/tabnet.py b/qlib/contrib/model/tabnet.py
deleted file mode 100644
index bc13d1f62..000000000
--- a/qlib/contrib/model/tabnet.py
+++ /dev/null
@@ -1,85 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-import numpy as np
-import pandas as pd
-from pytorch_tabnet.tab_model import TabNetRegressor
-
-from ...model.base import Model
-from ...data.dataset import DatasetH
-from ...data.dataset.handler import DataHandlerLP
-
-
-class TabNetModel(Model):
- """TabNetModel Model"""
-
- def __init__(
- self,
- n_d,
- n_a,
- n_steps,
- gamma,
- n_independent,
- n_shared,
- seed,
- momentum,
- lambda_sparse,
- optimizer_params,
- **kwargs
- ):
- self.model = None
-
- self.n_d = n_d
- self.n_a = n_a
- self.n_steps = n_steps
- self.gamma = gamma
- self.n_independent = n_independent
- self.n_shared = n_shared
- self.seed = seed
- self.momentum = momentum
- self.lambda_sparse = lambda_sparse
- self.optimizer_params = optimizer_params
-
- def fit(
- self,
- dataset: DatasetH,
- n_d=8,
- n_a=8,
- n_steps=3,
- gamma=1.3,
- n_independent=2,
- n_shared=2,
- seed=0,
- momentum=0.02,
- lambda_sparse=1e-3,
- optimizer_params={"lr": 2e-3},
- **kwargs
- ):
-
- df_train, df_valid = dataset.prepare(
- ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
- )
- x_train, y_train = df_train["feature"].values, df_train["label"].values * 100
- x_valid, y_valid = df_valid["feature"].values, df_valid["label"].values * 100
-
- self.model = TabNetRegressor(
- n_d=self.n_d,
- n_a=self.n_a,
- n_steps=self.n_steps,
- gamma=self.gamma,
- n_independent=self.n_independent,
- n_shared=self.n_shared,
- seed=self.seed,
- momentum=self.momentum,
- lambda_sparse=self.lambda_sparse,
- optimizer_params=self.optimizer_params,
- **kwargs
- )
- self.model.fit(x_train, y_train, eval_set=[(x_valid, y_valid)])
-
- def predict(self, dataset):
- if self.model is None:
- raise ValueError("model is not fitted yet!")
- x_test = dataset.prepare("test", col_set="feature")
- test_pred = self.model.predict(x_test.values)
- return pd.Series(test_pred.reshape([-1]), index=x_test.index)
diff --git a/qlib/contrib/model/xgboost.py b/qlib/contrib/model/xgboost.py
index 039fd2c80..c9e45d4ac 100755
--- a/qlib/contrib/model/xgboost.py
+++ b/qlib/contrib/model/xgboost.py
@@ -38,7 +38,9 @@ class XGBModel(Model):
):
df_train, df_valid = dataset.prepare(
- ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
+ ["train", "valid"],
+ col_set=["feature", "label"],
+ data_key=DataHandlerLP.DK_L,
)
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]
diff --git a/qlib/contrib/report/graph.py b/qlib/contrib/report/graph.py
index 15cc5fd0e..3fa688d36 100644
--- a/qlib/contrib/report/graph.py
+++ b/qlib/contrib/report/graph.py
@@ -96,7 +96,19 @@ class BaseGraph(object):
"""
py.init_notebook_mode()
for _fig in figure_list:
- py.iplot(_fig)
+ # NOTE: displays figures: https://plotly.com/python/renderers/
+ # default: plotly_mimetype+notebook
+ # support renderers: import plotly.io as pio; print(pio.renderers)
+ renderer = None
+ try:
+ # in notebook
+ _ipykernel = str(type(get_ipython()))
+ if "google.colab" in _ipykernel:
+ renderer = "colab"
+ except NameError:
+ pass
+
+ _fig.show(renderer=renderer)
def _get_layout(self) -> go.Layout:
"""
diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py
index e4fc8eef9..0ef062021 100644
--- a/qlib/model/trainer.py
+++ b/qlib/model/trainer.py
@@ -1,5 +1,6 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
+
from qlib.utils import init_instance_by_config, flatten_dict
from qlib.workflow import R
from qlib.workflow.record_temp import SignalRecord
diff --git a/qlib/workflow/cli.py b/qlib/workflow/cli.py
index 08c13de2a..65d9a14b4 100644
--- a/qlib/workflow/cli.py
+++ b/qlib/workflow/cli.py
@@ -1,13 +1,14 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
-import sys
+import sys, os
from pathlib import Path
import qlib
import fire
import pandas as pd
import ruamel.yaml as yaml
+from qlib.config import C
from qlib.model.trainer import task_train
@@ -41,7 +42,7 @@ def sys_config(config, config_path):
# worflow handler function
-def workflow(config_path, experiment_name="workflow"):
+def workflow(config_path, experiment_name="workflow", uri_folder="mlruns"):
with open(config_path) as fp:
config = yaml.load(fp, Loader=yaml.Loader)
@@ -50,7 +51,9 @@ def workflow(config_path, experiment_name="workflow"):
provider_uri = config.get("provider_uri")
region = config.get("region")
- qlib.init(provider_uri=provider_uri, region=region)
+ exp_manager = C["exp_manager"]
+ exp_manager["kwargs"]["uri"] = "file:" + str(Path(os.getcwd()).resolve() / uri_folder)
+ qlib.init(provider_uri=provider_uri, region=region, exp_manager=exp_manager)
task_train(config, experiment_name=experiment_name)
diff --git a/qlib/workflow/expm.py b/qlib/workflow/expm.py
index 156beb690..80d471845 100644
--- a/qlib/workflow/expm.py
+++ b/qlib/workflow/expm.py
@@ -239,20 +239,17 @@ class MLflowExpManager(ExpManager):
return self._client
def start_exp(self, experiment_name=None, recorder_name=None, uri=None):
+ # set the tracking uri
+ if uri is None:
+ logger.info("No tracking URI is provided. Use the default tracking URI.")
+ else:
+ self.uri = uri
# create experiment
experiment, _ = self._get_or_create_exp(experiment_name=experiment_name)
# set up active experiment
self.active_experiment = experiment
# start the experiment
self.active_experiment.start(recorder_name)
- # set the tracking uri
- if uri is None:
- logger.info(
- "No tracking URI is provided. The default tracking URI is set as `mlruns` under the working directory."
- )
- else:
- self.uri = uri
- mlflow.set_tracking_uri(self.uri)
return self.active_experiment
diff --git a/qlib/workflow/recorder.py b/qlib/workflow/recorder.py
index b3069b9ac..4c1ddfdfe 100644
--- a/qlib/workflow/recorder.py
+++ b/qlib/workflow/recorder.py
@@ -224,6 +224,8 @@ class MLflowRecorder(Recorder):
)
def start_run(self):
+ # set the tracking uri
+ mlflow.set_tracking_uri(self._uri)
# start the run
run = mlflow.start_run(self.id, self.experiment_id, self.name)
# save the run id and artifact_uri
diff --git a/requirements.txt b/requirements.txt
index d3511d780..638ce22f4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -22,5 +22,4 @@ scikit_learn==0.23.2
torch==1.6.0
tqdm==4.49.0
yahooquery==2.2.7
-mlflow==1.12.1
-pytorch-tabnet==2.0.1
\ No newline at end of file
+mlflow==1.12.1
\ No newline at end of file
diff --git a/scripts/data_collector/yahoo/collector.py b/scripts/data_collector/yahoo/collector.py
index 69c7f8f15..0d41251f1 100644
--- a/scripts/data_collector/yahoo/collector.py
+++ b/scripts/data_collector/yahoo/collector.py
@@ -44,6 +44,7 @@ class YahooCollector:
delay=0,
check_data_length: bool = False,
limit_nums: int = None,
+ show_1m_logging: bool = False,
):
"""
@@ -67,10 +68,13 @@ class YahooCollector:
check data length, by default False
limit_nums: int
using for debug, by default None
+ show_1m_logging: bool
+ show 1m logging, by default False; if True, there may be many warning logs
"""
self.save_dir = Path(save_dir).expanduser().resolve()
self.save_dir.mkdir(parents=True, exist_ok=True)
self._delay = delay
+ self._show_1m_logging = show_1m_logging
self.stock_list = sorted(set(self.get_stock_list()))
if limit_nums is not None:
try:
@@ -83,7 +87,7 @@ class YahooCollector:
self._interval = interval
self._check_small_data = check_data_length
self._start_datetime = pd.Timestamp(str(start)) if start else self.START_DATETIME
- self._end_datetime = pd.Timestamp(str(end)) if end else self.END_DATETIME
+ self._end_datetime = min(pd.Timestamp(str(end)) if end else self.END_DATETIME, self.END_DATETIME)
if self._interval == "1m":
self._start_datetime = max(self._start_datetime, self.HIGH_FREQ_START_DATETIME)
elif self._interval == "1d":
@@ -91,8 +95,12 @@ class YahooCollector:
else:
raise ValueError(f"interval error: {self._interval}")
+ # using for 1m
+ self._next_datetime = self.convert_datetime(self._start_datetime.date() + pd.Timedelta(days=1))
+ self._latest_datetime = self.convert_datetime(self._end_datetime.date())
+
self._start_datetime = self.convert_datetime(self._start_datetime)
- self._end_datetime = self.convert_datetime(min(self._end_datetime, self.END_DATETIME))
+ self._end_datetime = self.convert_datetime(self._end_datetime)
@property
@abc.abstractmethod
@@ -100,20 +108,24 @@ class YahooCollector:
# daily, one year: 252 / 4
# us 1min, a week: 6.5 * 60 * 5
# cn 1min, a week: 4 * 60 * 5
- raise NotImplementedError("rewirte min_numbers_trading")
+ raise NotImplementedError("rewrite min_numbers_trading")
@abc.abstractmethod
def get_stock_list(self):
- raise NotImplementedError("rewirte get_stock_list")
+ raise NotImplementedError("rewrite get_stock_list")
@property
- @abc.abstractclassmethod
+ @abc.abstractmethod
def _timezone(self):
raise NotImplementedError("rewrite get_timezone")
- def convert_datetime(self, dt: pd.Timestamp):
- dt = pd.Timestamp(dt, tz=self._timezone).timestamp()
- return pd.Timestamp(dt, tz=tzlocal(), unit="s")
+ def convert_datetime(self, dt: [pd.Timestamp, datetime.date, str]):
+ try:
+ dt = pd.Timestamp(dt, tz=self._timezone).timestamp()
+ dt = pd.Timestamp(dt, tz=tzlocal(), unit="s")
+ except ValueError as e:
+ pass
+ return dt
def _sleep(self):
time.sleep(self._delay)
@@ -136,7 +148,7 @@ class YahooCollector:
df["symbol"] = symbol
if stock_path.exists():
with stock_path.open("a") as fp:
- df.to_csv(fp, index=False, header=None)
+ df.to_csv(fp, index=False, header=False)
else:
with stock_path.open("w") as fp:
df.to_csv(fp, index=False)
@@ -155,34 +167,47 @@ class YahooCollector:
def _get_from_remote(self, symbol):
def _get_simple(start_, end_):
self._sleep()
+ error_msg = f"{symbol}-{self._interval}-{start_}-{end_}"
+
+ def _show_logging_func():
+ if self._interval == "1m" and self._show_1m_logging:
+ logger.warning(f"{error_msg}:{_resp}")
+
try:
_resp = Ticker(symbol, asynchronous=False).history(interval=self._interval, start=start_, end=end_)
if isinstance(_resp, pd.DataFrame):
return _resp.reset_index()
+ elif isinstance(_resp, dict):
+ _temp_data = _resp.get(symbol, {})
+ if isinstance(_temp_data, str) or (
+ isinstance(_resp, dict) and _temp_data.get("indicators", {}).get("quote", None) is None
+ ):
+ _show_logging_func()
else:
- logger.warning(f"{symbol}-{self._interval}-{start_}-{end_}:{_resp}")
+ _show_logging_func()
except Exception as e:
- logger.warning(f"{symbol}-{self._interval}-{start_}-{end_}:{e}")
+ logger.warning(f"{error_msg}:{e}")
_result = None
if self._interval == "1d":
_result = _get_simple(self._start_datetime, self._end_datetime)
elif self._interval == "1m":
- _start_date = self._start_datetime.date() + pd.Timedelta(days=1)
- _end_date = self._end_datetime.date()
- if _start_date >= _end_date:
+ if self._next_datetime >= self._latest_datetime:
_result = _get_simple(self._start_datetime, self._end_datetime)
else:
_res = []
def _get_multi(start_, end_):
_resp = _get_simple(start_, end_)
- if _resp is not None:
+ if _resp is not None and not _resp.empty:
_res.append(_resp)
- for _s, _e in ((self._start_datetime, _start_date), (_end_date, self._end_datetime)):
+ for _s, _e in (
+ (self._start_datetime, self._next_datetime),
+ (self._latest_datetime, self._end_datetime),
+ ):
_get_multi(_s, _e)
- for _start in pd.date_range(_start_date, _end_date, closed="left"):
+ for _start in pd.date_range(self._next_datetime, self._latest_datetime, closed="left"):
_end = _start + pd.Timedelta(days=1)
self._sleep()
_get_multi(_start, _end)
@@ -472,6 +497,7 @@ class Run:
interval="1d",
check_data_length=False,
limit_nums=None,
+ show_1m_logging=False,
):
"""download data from Internet
@@ -491,6 +517,9 @@ class Run:
check data length, by default False
limit_nums: int
using for debug, by default None
+ show_1m_logging: bool
+ show 1m logging, by default False; if True, there may be many warning logs
+
Examples
---------
# get daily data
@@ -510,6 +539,7 @@ class Run:
interval=interval,
check_data_length=check_data_length,
limit_nums=limit_nums,
+ show_1m_logging=show_1m_logging,
).collector_data()
def normalize_data(self):
@@ -531,6 +561,7 @@ class Run:
interval="1d",
check_data_length=False,
limit_nums=None,
+ show_1m_logging=False,
):
"""download -> normalize
@@ -550,6 +581,9 @@ class Run:
check data length, by default False
limit_nums: int
using for debug, by default None
+ show_1m_logging: bool
+ show 1m logging, by default False; if True, there may be many warning logs
+
Examples
-------
python collector.py collector_data --source_dir ~/.qlib/stock_data/source --normalize_dir ~/.qlib/stock_data/normalize --region CN --start 2020-11-01 --end 2020-11-10 --delay 0.1 --interval 1d
@@ -562,6 +596,7 @@ class Run:
interval=interval,
check_data_length=check_data_length,
limit_nums=limit_nums,
+ show_1m_logging=show_1m_logging,
)
self.normalize_data()