diff --git a/README.md b/README.md index 5a035662b..f03c565d7 100644 --- a/README.md +++ b/README.md @@ -46,11 +46,11 @@ For more details, please refer to our paper ["Qlib: An AI-oriented Quantitative -At the module level, Qlib is a platform that consists of the above components. The components are designed as loose-coupled modules and each component could be used stand-alone. +At the module level, Qlib is a platform that consists of the above components. The components are designed as loose-coupled modules, and each component could be used stand-alone. | Name | Description | | ------ | ----- | -| `Infrastructure` layer | `Infrastructure` layer provides underlying support for Quant research. `DataServer` provides high-performance infrastructure for users to manage and retrieve raw data. `Trainer` provides flexible interface to control the training process of models which enable algorithms controlling the training process. | +| `Infrastructure` layer | `Infrastructure` layer provides underlying support for Quant research. `DataServer` provides a high-performance infrastructure for users to manage and retrieve raw data. `Trainer` provides a flexible interface to control the training process of models, which enable algorithms to control the training process. | | `Workflow` layer | `Workflow` layer covers the whole workflow of quantitative investment. `Information Extractor` extracts data for models. `Forecast Model` focuses on producing all kinds of forecast signals (e.g. _alpha_, risk) for other modules. With these signals `Portfolio Generator` will generate the target portfolio and produce orders to be executed by `Order Executor`. | | `Interface` layer | `Interface` layer tries to present a user-friendly interface for the underlying system. `Analyser` module will provide users detailed analysis reports of forecasting signals, portfolios and execution results | @@ -130,7 +130,8 @@ This dataset is created by public data collected by [crawler scripts](scripts/da the same repository. Users could create the same dataset with it. -*Please pay **ATTENTION** that the data is collected from [Yahoo Finance](https://finance.yahoo.com/lookup) and the data might not be perfect. We recommend users to prepare their own data if they have high-quality dataset. For more information, users can refer to the [related document](https://qlib.readthedocs.io/en/latest/component/data.html#converting-csv-format-into-qlib-format)*. +*Please pay **ATTENTION** that the data is collected from [Yahoo Finance](https://finance.yahoo.com/lookup), and the data might not be perfect. +We recommend users to prepare their own data if they have a high-quality dataset. For more information, users can refer to the [related document](https://qlib.readthedocs.io/en/latest/component/data.html#converting-csv-format-into-qlib-format)*. ## Building Customized Quant Research Workflow by Code -The automatic workflow may not suite the research workflow of all Quant researchers. To support a flexible Quant research workflow, Qlib also provides a modularized interface to allow researchers to build their own workflow by code. [Here](examples/workflow_by_code.ipynb) is a demo for customized Quant research workflow by code. +The automatic workflow may not suit the research workflow of all Quant researchers. To support a flexible Quant research workflow, Qlib also provides a modularized interface to allow researchers to build their own workflow by code. [Here](examples/workflow_by_code.ipynb) is a demo for customized Quant research workflow by code. # [Quant Model Zoo](examples/benchmarks) @@ -313,7 +314,7 @@ which creates a dataset (14 features/factors) from the basic OHLCV daily data of * `+(-)E` indicates with (out) `ExpressionCache` * `+(-)D` indicates with (out) `DatasetCache` -Most general-purpose databases take too much time on loading data. After looking into the underlying implementation, we find that data go through too many layers of interfaces and unnecessary format transformations in general-purpose database solutions. +Most general-purpose databases take too much time to load data. After looking into the underlying implementation, we find that data go through too many layers of interfaces and unnecessary format transformations in general-purpose database solutions. Such overheads greatly slow down the data loading process. Qlib data are stored in a compact format, which is efficient to be combined into arrays for scientific computation. diff --git a/examples/benchmarks/LSTM/workflow_config_lstm_Alpha158.yaml b/examples/benchmarks/LSTM/workflow_config_lstm_Alpha158.yaml index 15fa20ec3..14dd69d0a 100755 --- a/examples/benchmarks/LSTM/workflow_config_lstm_Alpha158.yaml +++ b/examples/benchmarks/LSTM/workflow_config_lstm_Alpha158.yaml @@ -29,7 +29,7 @@ data_handler_config: &data_handler_config - class: CSRankNorm kwargs: fields_group: label - label: ["Ref($close, -2) / Ref($close, -1) - 1"] + label: ["Ref($close, -2) / Ref($close, -1) - 1"] port_analysis_config: &port_analysis_config strategy: diff --git a/examples/benchmarks/TabNet/workflow_config_TabNet_Alpha158.yaml b/examples/benchmarks/TabNet/workflow_config_TabNet_Alpha158.yaml index 243505a89..cf8ef7411 100644 --- a/examples/benchmarks/TabNet/workflow_config_TabNet_Alpha158.yaml +++ b/examples/benchmarks/TabNet/workflow_config_TabNet_Alpha158.yaml @@ -44,6 +44,7 @@ task: class: TabnetModel module_path: qlib.contrib.model.pytorch_tabnet kwargs: + d_feat: 158 pretrain: True dataset: class: DatasetH diff --git a/examples/benchmarks/TabNet/workflow_config_TabNet_Alpha360.yaml b/examples/benchmarks/TabNet/workflow_config_TabNet_Alpha360.yaml new file mode 100644 index 000000000..5023e9b3d --- /dev/null +++ b/examples/benchmarks/TabNet/workflow_config_TabNet_Alpha360.yaml @@ -0,0 +1,75 @@ +qlib_init: + provider_uri: "~/.qlib/qlib_data/cn_data" + region: cn +market: &market csi300 +benchmark: &benchmark SH000300 +data_handler_config: &data_handler_config + start_time: 2008-01-01 + end_time: 2020-08-01 + fit_start_time: 2008-01-01 + fit_end_time: 2014-12-31 + instruments: *market + infer_processors: + - class: RobustZScoreNorm + kwargs: + fields_group: feature + clip_outlier: true + - class: Fillna + kwargs: + fields_group: feature + learn_processors: + - class: DropnaLabel + - class: CSRankNorm + kwargs: + fields_group: label + label: ["Ref($close, -2) / Ref($close, -1) - 1"] +port_analysis_config: &port_analysis_config + strategy: + class: TopkDropoutStrategy + module_path: qlib.contrib.strategy.strategy + kwargs: + topk: 50 + n_drop: 5 + backtest: + verbose: False + limit_threshold: 0.095 + account: 100000000 + benchmark: *benchmark + deal_price: close + open_cost: 0.0005 + close_cost: 0.0015 + min_cost: 5 +task: + model: + class: TabnetModel + module_path: qlib.contrib.model.pytorch_tabnet + kwargs: + d_feat: 360 + pretrain: True + dataset: + class: DatasetH + module_path: qlib.data.dataset + kwargs: + handler: + class: Alpha360 + module_path: qlib.contrib.data.handler + kwargs: *data_handler_config + segments: + pretrain: [2008-01-01, 2014-12-31] + pretrain_validation: [2015-01-01, 2016-12-31] + train: [2008-01-01, 2014-12-31] + valid: [2015-01-01, 2016-12-31] + test: [2017-01-01, 2020-08-01] + record: + - class: SignalRecord + module_path: qlib.workflow.record_temp + kwargs: {} + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 + - class: PortAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + config: *port_analysis_config diff --git a/qlib/contrib/model/pytorch_alstm.py b/qlib/contrib/model/pytorch_alstm.py index 3c008ae9a..a149272da 100644 --- a/qlib/contrib/model/pytorch_alstm.py +++ b/qlib/contrib/model/pytorch_alstm.py @@ -9,15 +9,13 @@ import os import numpy as np import pandas as pd import copy -from sklearn.metrics import roc_auc_score, mean_squared_error -import logging from ...utils import ( unpack_archive_with_buffer, save_multiple_parts_file, get_or_create_path, drop_nan_by_y_index, ) -from ...log import get_module_logger, TimeInspector +from ...log import get_module_logger import torch import torch.nn as nn diff --git a/qlib/contrib/model/pytorch_alstm_ts.py b/qlib/contrib/model/pytorch_alstm_ts.py index eb6e856ef..c38727b9e 100644 --- a/qlib/contrib/model/pytorch_alstm_ts.py +++ b/qlib/contrib/model/pytorch_alstm_ts.py @@ -9,15 +9,13 @@ import os import numpy as np import pandas as pd import copy -from sklearn.metrics import roc_auc_score, mean_squared_error -import logging from ...utils import ( unpack_archive_with_buffer, save_multiple_parts_file, get_or_create_path, drop_nan_by_y_index, ) -from ...log import get_module_logger, TimeInspector +from ...log import get_module_logger import torch import torch.nn as nn diff --git a/qlib/contrib/model/pytorch_gats.py b/qlib/contrib/model/pytorch_gats.py index 4edbc8bcf..53afb5404 100644 --- a/qlib/contrib/model/pytorch_gats.py +++ b/qlib/contrib/model/pytorch_gats.py @@ -9,15 +9,13 @@ import os import numpy as np import pandas as pd import copy -from sklearn.metrics import roc_auc_score, mean_squared_error -import logging from ...utils import ( unpack_archive_with_buffer, save_multiple_parts_file, get_or_create_path, drop_nan_by_y_index, ) -from ...log import get_module_logger, TimeInspector +from ...log import get_module_logger import torch import torch.nn as nn import torch.optim as optim diff --git a/qlib/contrib/model/pytorch_gats_ts.py b/qlib/contrib/model/pytorch_gats_ts.py index dd83c00f9..f02bf1e47 100644 --- a/qlib/contrib/model/pytorch_gats_ts.py +++ b/qlib/contrib/model/pytorch_gats_ts.py @@ -9,15 +9,13 @@ import os import numpy as np import pandas as pd import copy -from sklearn.metrics import roc_auc_score, mean_squared_error -import logging from ...utils import ( unpack_archive_with_buffer, save_multiple_parts_file, get_or_create_path, drop_nan_by_y_index, ) -from ...log import get_module_logger, TimeInspector +from ...log import get_module_logger import torch import torch.nn as nn import torch.optim as optim diff --git a/qlib/contrib/model/pytorch_gru.py b/qlib/contrib/model/pytorch_gru.py index 0070d1811..5eba33595 100755 --- a/qlib/contrib/model/pytorch_gru.py +++ b/qlib/contrib/model/pytorch_gru.py @@ -9,15 +9,13 @@ import os import numpy as np import pandas as pd import copy -from sklearn.metrics import roc_auc_score, mean_squared_error -import logging from ...utils import ( unpack_archive_with_buffer, save_multiple_parts_file, get_or_create_path, drop_nan_by_y_index, ) -from ...log import get_module_logger, TimeInspector +from ...log import get_module_logger import torch import torch.nn as nn diff --git a/qlib/contrib/model/pytorch_gru_ts.py b/qlib/contrib/model/pytorch_gru_ts.py index 4553c7537..2839b35e4 100755 --- a/qlib/contrib/model/pytorch_gru_ts.py +++ b/qlib/contrib/model/pytorch_gru_ts.py @@ -9,15 +9,13 @@ import os import numpy as np import pandas as pd import copy -from sklearn.metrics import roc_auc_score, mean_squared_error -import logging from ...utils import ( unpack_archive_with_buffer, save_multiple_parts_file, get_or_create_path, drop_nan_by_y_index, ) -from ...log import get_module_logger, TimeInspector +from ...log import get_module_logger import torch import torch.nn as nn diff --git a/qlib/contrib/model/pytorch_lstm.py b/qlib/contrib/model/pytorch_lstm.py index c7385c6a7..636ef6e3a 100755 --- a/qlib/contrib/model/pytorch_lstm.py +++ b/qlib/contrib/model/pytorch_lstm.py @@ -9,15 +9,13 @@ import os import numpy as np import pandas as pd import copy -from sklearn.metrics import roc_auc_score, mean_squared_error -import logging from ...utils import ( unpack_archive_with_buffer, save_multiple_parts_file, get_or_create_path, drop_nan_by_y_index, ) -from ...log import get_module_logger, TimeInspector +from ...log import get_module_logger import torch import torch.nn as nn diff --git a/qlib/contrib/model/pytorch_lstm_ts.py b/qlib/contrib/model/pytorch_lstm_ts.py index 288bdc202..c978e84c7 100755 --- a/qlib/contrib/model/pytorch_lstm_ts.py +++ b/qlib/contrib/model/pytorch_lstm_ts.py @@ -9,15 +9,13 @@ import os import numpy as np import pandas as pd import copy -from sklearn.metrics import roc_auc_score, mean_squared_error -import logging from ...utils import ( unpack_archive_with_buffer, save_multiple_parts_file, get_or_create_path, drop_nan_by_y_index, ) -from ...log import get_module_logger, TimeInspector +from ...log import get_module_logger import torch import torch.nn as nn diff --git a/qlib/contrib/model/pytorch_nn.py b/qlib/contrib/model/pytorch_nn.py index fad466165..caf34b330 100644 --- a/qlib/contrib/model/pytorch_nn.py +++ b/qlib/contrib/model/pytorch_nn.py @@ -6,7 +6,6 @@ from __future__ import division from __future__ import print_function import os -import logging import numpy as np import pandas as pd from sklearn.metrics import roc_auc_score, mean_squared_error @@ -20,7 +19,7 @@ from ...model.base import Model from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP from ...utils import unpack_archive_with_buffer, save_multiple_parts_file, get_or_create_path, drop_nan_by_y_index -from ...log import get_module_logger, TimeInspector +from ...log import get_module_logger from ...workflow import R diff --git a/qlib/contrib/model/pytorch_sfm.py b/qlib/contrib/model/pytorch_sfm.py index f013d81a3..db3e8bb12 100644 --- a/qlib/contrib/model/pytorch_sfm.py +++ b/qlib/contrib/model/pytorch_sfm.py @@ -8,15 +8,13 @@ import os import numpy as np import pandas as pd import copy -from sklearn.metrics import roc_auc_score, mean_squared_error -import logging from ...utils import ( unpack_archive_with_buffer, save_multiple_parts_file, get_or_create_path, drop_nan_by_y_index, ) -from ...log import get_module_logger, TimeInspector +from ...log import get_module_logger import torch import torch.nn as nn diff --git a/qlib/contrib/model/pytorch_tabnet.py b/qlib/contrib/model/pytorch_tabnet.py index c1dce9308..450e6f5d1 100644 --- a/qlib/contrib/model/pytorch_tabnet.py +++ b/qlib/contrib/model/pytorch_tabnet.py @@ -7,15 +7,13 @@ import os import numpy as np import pandas as pd import copy -from sklearn.metrics import roc_auc_score, mean_squared_error -import logging from ...utils import ( unpack_archive_with_buffer, save_multiple_parts_file, get_or_create_path, drop_nan_by_y_index, ) -from ...log import get_module_logger, TimeInspector +from ...log import get_module_logger import torch import torch.nn as nn @@ -93,12 +91,8 @@ class TabnetModel(Model): np.random.seed(self.seed) torch.manual_seed(self.seed) - self.tabnet_model = TabNet( - inp_dim=self.d_feat, out_dim=self.out_dim, vbs=vbs, relax=relax, device=self.device - ).to(self.device) - self.tabnet_decoder = TabNet_Decoder(self.out_dim, self.d_feat, n_shared, n_ind, vbs, n_steps, self.device).to( - self.device - ) + self.tabnet_model = TabNet(inp_dim=self.d_feat, out_dim=self.out_dim, vbs=vbs, relax=relax).to(self.device) + self.tabnet_decoder = TabNet_Decoder(self.out_dim, self.d_feat, n_shared, n_ind, vbs, n_steps).to(self.device) self.logger.info("model:\n{:}\n{:}".format(self.tabnet_model, self.tabnet_decoder)) self.logger.info("model size: {:.4f} MB".format(count_parameters([self.tabnet_model, self.tabnet_decoder]))) @@ -410,9 +404,9 @@ class FinetuneModel(nn.Module): class DecoderStep(nn.Module): - def __init__(self, inp_dim, out_dim, shared, n_ind, vbs, device): + def __init__(self, inp_dim, out_dim, shared, n_ind, vbs): super().__init__() - self.fea_tran = FeatureTransformer(inp_dim, out_dim, shared, n_ind, vbs, device) + self.fea_tran = FeatureTransformer(inp_dim, out_dim, shared, n_ind, vbs) self.fc = nn.Linear(out_dim, out_dim) def forward(self, x): @@ -421,13 +415,12 @@ class DecoderStep(nn.Module): class TabNet_Decoder(nn.Module): - def __init__(self, inp_dim, out_dim, n_shared, n_ind, vbs, n_steps, device): + def __init__(self, inp_dim, out_dim, n_shared, n_ind, vbs, n_steps): """ TabNet decoder that is used in pre-training """ - self.out_dim = out_dim - super().__init__() + self.out_dim = out_dim if n_shared > 0: self.shared = nn.ModuleList() self.shared.append(nn.Linear(inp_dim, 2 * out_dim)) @@ -438,7 +431,7 @@ class TabNet_Decoder(nn.Module): self.n_steps = n_steps self.steps = nn.ModuleList() for x in range(n_steps): - self.steps.append(DecoderStep(inp_dim, out_dim, self.shared, n_ind, vbs, device)) + self.steps.append(DecoderStep(inp_dim, out_dim, self.shared, n_ind, vbs)) def forward(self, x): out = torch.zeros(x.size(0), self.out_dim).to(x.device) @@ -448,9 +441,7 @@ class TabNet_Decoder(nn.Module): class TabNet(nn.Module): - def __init__( - self, inp_dim=6, out_dim=6, n_d=64, n_a=64, n_shared=2, n_ind=2, n_steps=5, relax=1.2, vbs=1024, device="cpu" - ): + def __init__(self, inp_dim=6, out_dim=6, n_d=64, n_a=64, n_shared=2, n_ind=2, n_steps=5, relax=1.2, vbs=1024): """ TabNet AKA the original encoder @@ -474,10 +465,10 @@ class TabNet(nn.Module): else: self.shared = None - self.first_step = FeatureTransformer(inp_dim, n_d + n_a, self.shared, n_ind, vbs, device) + self.first_step = FeatureTransformer(inp_dim, n_d + n_a, self.shared, n_ind, vbs) self.steps = nn.ModuleList() for x in range(n_steps - 1): - self.steps.append(DecisionStep(inp_dim, n_d, n_a, self.shared, n_ind, relax, vbs, device)) + self.steps.append(DecisionStep(inp_dim, n_d, n_a, self.shared, n_ind, relax, vbs)) self.fc = nn.Linear(n_d, out_dim) self.bn = nn.BatchNorm1d(inp_dim, momentum=0.01) self.n_d = n_d @@ -486,14 +477,14 @@ class TabNet(nn.Module): assert not torch.isnan(x).any() x = self.bn(x) x_a = self.first_step(x)[:, self.n_d :] - sparse_loss = torch.zeros(1).to(x.device) + sparse_loss = [] out = torch.zeros(x.size(0), self.n_d).to(x.device) for step in self.steps: x_te, l = step(x, x_a, priors) out += F.relu(x_te[:, : self.n_d]) # split the feautre from feat_transformer x_a = x_te[:, self.n_d :] - sparse_loss += l - return self.fc(out), sparse_loss + sparse_loss.append(l) + return self.fc(out), sum(sparse_loss) class GBN(nn.Module): @@ -511,9 +502,12 @@ class GBN(nn.Module): self.vbs = vbs def forward(self, x): - chunk = torch.chunk(x, x.size(0) // self.vbs, 0) - res = [self.bn(y) for y in chunk] - return torch.cat(res, 0) + if x.size(0) <= self.vbs: # can not be chunked + return self.bn(x) + else: + chunk = torch.chunk(x, x.size(0) // self.vbs, 0) + res = [self.bn(y) for y in chunk] + return torch.cat(res, 0) class GLU(nn.Module): @@ -561,7 +555,7 @@ class AttentionTransformer(nn.Module): class FeatureTransformer(nn.Module): - def __init__(self, inp_dim, out_dim, shared, n_ind, vbs, device): + def __init__(self, inp_dim, out_dim, shared, n_ind, vbs): super().__init__() first = True self.shared = nn.ModuleList() @@ -577,7 +571,7 @@ class FeatureTransformer(nn.Module): self.independ.append(GLU(inp, out_dim, vbs=vbs)) for x in range(first, n_ind): self.independ.append(GLU(out_dim, out_dim, vbs=vbs)) - self.scale = torch.sqrt(torch.tensor([0.5], device=device)) + self.scale = float(np.sqrt(0.5)) def forward(self, x): if self.shared: @@ -596,10 +590,10 @@ class DecisionStep(nn.Module): One step for the TabNet """ - def __init__(self, inp_dim, n_d, n_a, shared, n_ind, relax, vbs, device): + def __init__(self, inp_dim, n_d, n_a, shared, n_ind, relax, vbs): super().__init__() self.atten_tran = AttentionTransformer(n_a, inp_dim, relax, vbs) - self.fea_tran = FeatureTransformer(inp_dim, n_d + n_a, shared, n_ind, vbs, device) + self.fea_tran = FeatureTransformer(inp_dim, n_d + n_a, shared, n_ind, vbs) def forward(self, x, a, priors): mask = self.atten_tran(a, priors)