liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-07 04:50:56 +08:00
Code Activity

Adjust rolling api (#1594)

Browse Source 
* Intermediate version

* Fix yaml template & Successfully run rolling

* Be compatible with benchmark

* Get same results with previous linear model

* Black formatting

* Update black

* Update the placeholder mechanism

* Update CI

* Update CI

* Upgrade Black

* Fix CI and simplify code

* Fix CI

* Move the data processing caching mechanism into utils.

* Adjusting DDG-DA

* Organize import
This commit is contained in:
you-n-g
2023-07-14 12:16:12 +08:00
committed by GitHub
parent 8d3adf34ac
commit be4646b4b7
148 changed files with 1035 additions and 1028 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
qlib/__init__.py
View File

@@ -77,7 +77,6 @@ def init(default_conf="client", **kwargs):
def _mount_nfs_uri(provider_uri, mount_path, auto_mount: bool = False):
LOG = get_module_logger("mount nfs", level=logging.INFO)
if mount_path is None:
raise ValueError(f"Invalid mount path: {mount_path}!")

1
qlib/backtest/__init__.py
View File

@@ -182,7 +182,6 @@ def get_strategy_executor(
exchange_kwargs: dict = {},
pos_type: str = "Position",
) -> Tuple[BaseStrategy, BaseExecutor]:
# NOTE:
# - for avoiding recursive import
# - typing annotations is not reliable

1
qlib/backtest/exchange.py
View File

@@ -638,7 +638,6 @@ class Exchange:
random.seed(0)
random.shuffle(sorted_ids)
for stock_id in sorted_ids:
# Do not generate order for the non-tradable stocks
if not self.is_stock_tradable(stock_id=stock_id, start_time=start_time, end_time=end_time):
continue

1
qlib/config.py
View File

@@ -293,7 +293,6 @@ class QlibConfig(Config):
"""
def __init__(self, provider_uri: Union[str, Path, dict], mount_path: Union[str, Path, dict]):
"""
The relation of `provider_uri` and `mount_path`
- `mount_path` is used only if provider_uri is an NFS path

5
qlib/contrib/data/dataset.py
View File

@@ -130,7 +130,6 @@ class MTSDatasetH(DatasetH):
input_size=None,
**kwargs,
):
assert num_states == 0 or horizon > 0, "please specify `horizon` to avoid data leakage"
assert memory_mode in ["sample", "daily"], "unsupported memory mode"
assert memory_mode == "sample" or batch_size < 0, "daily memory requires daily sampling (`batch_size < 0`)"
@@ -153,7 +152,6 @@ class MTSDatasetH(DatasetH):
super().__init__(handler, segments, **kwargs)
def setup_data(self, handler_kwargs: dict = None, **kwargs):
super().setup_data(**kwargs)
if handler_kwargs is not None:
@@ -288,7 +286,6 @@ class MTSDatasetH(DatasetH):
daily_count = [] # store number of samples for each day
for j in indices[i : i + batch_size]:
# normal sampling: self.batch_size > 0 => slices is a list => slices_subset is a slice
# daily sampling: self.batch_size < 0 => slices is a nested list => slices_subset is a list
slices_subset = slices[j]
@@ -297,7 +294,6 @@ class MTSDatasetH(DatasetH):
# each slices_subset contains a list of slices for multiple stocks
# NOTE: daily sampling is used in 1) eval mode, 2) train mode with self.batch_size < 0
if self.batch_size < 0:
# store daily index
idx = self._daily_index.index[j] # daily_index.index is the index of the original data
daily_index.append(idx)
@@ -320,7 +316,6 @@ class MTSDatasetH(DatasetH):
slices_subset = [slices_subset]
for slc in slices_subset:
# legacy support for Alpha360 data by `input_size`
if self.input_size:
data.append(self._data[slc.stop - 1].reshape(self.input_size, -1).T)

2
qlib/contrib/data/highfreq_handler.py
View File

@@ -17,7 +17,6 @@ class HighFreqHandler(DataHandlerLP):
fit_end_time=None,
drop_raw=True,
):
infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time)
learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time)
@@ -318,7 +317,6 @@ class HighFreqOrderHandler(DataHandlerLP):
inst_processors=None,
drop_raw=True,
):
infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time)
learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time)

1
qlib/contrib/data/highfreq_processor.py
View File

@@ -29,7 +29,6 @@ class HighFreqNorm(Processor):
feature_save_dir: str,
norm_groups: Dict[str, int],
):
self.fit_start_time = fit_start_time
self.fit_end_time = fit_end_time
self.feature_save_dir = feature_save_dir

2
qlib/contrib/meta/data_selection/dataset.py
View File

@@ -49,6 +49,8 @@ class InternalData:
# 1) prepare the prediction of proxy models
perf_task_tpl = deepcopy(self.task_tpl) # this task is supposed to contains no complicated objects
# The only thing we want to save is the prediction
perf_task_tpl["record"] = ["qlib.workflow.record_temp.SignalRecord"]
trainer = auto_filter_kwargs(trainer)(experiment_name=self.exp_name, **trainer_kwargs)
# NOTE:

2
qlib/contrib/model/pytorch_adarnn.py
View File

@@ -246,7 +246,6 @@ class ADARNN(Model):
evals_result=dict(),
save_path=None,
):
df_train, df_valid = dataset.prepare(
["train", "valid"],
col_set=["feature", "label"],
@@ -318,7 +317,6 @@ class ADARNN(Model):
preds = []
for begin in range(sample_num)[:: self.batch_size]:
if sample_num - begin < self.batch_size:
end = sample_num
else:

7
qlib/contrib/model/pytorch_alstm.py
View File

@@ -146,7 +146,6 @@ class ALSTM(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -155,7 +154,6 @@ class ALSTM(Model):
raise ValueError("unknown metric `%s`" % self.metric)
def train_epoch(self, x_train, y_train):
x_train_values = x_train.values
y_train_values = np.squeeze(y_train.values)
@@ -165,7 +163,6 @@ class ALSTM(Model):
np.random.shuffle(indices)
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -181,7 +178,6 @@ class ALSTM(Model):
self.train_optimizer.step()
def test_epoch(self, data_x, data_y):
# prepare training data
x_values = data_x.values
y_values = np.squeeze(data_y.values)
@@ -194,7 +190,6 @@ class ALSTM(Model):
indices = np.arange(len(x_values))
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -217,7 +212,6 @@ class ALSTM(Model):
evals_result=dict(),
save_path=None,
):
df_train, df_valid, df_test = dataset.prepare(
["train", "valid", "test"],
col_set=["feature", "label"],
@@ -282,7 +276,6 @@ class ALSTM(Model):
preds = []
for begin in range(sample_num)[:: self.batch_size]:
if sample_num - begin < self.batch_size:
end = sample_num
else:

9
qlib/contrib/model/pytorch_alstm_ts.py
View File

@@ -156,7 +156,6 @@ class ALSTM(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -165,10 +164,9 @@ class ALSTM(Model):
raise ValueError("unknown metric `%s`" % self.metric)
def train_epoch(self, data_loader):
self.ALSTM_model.train()
for (data, weight) in data_loader:
for data, weight in data_loader:
feature = data[:, :, 0:-1].to(self.device)
label = data[:, -1, -1].to(self.device)
@@ -181,14 +179,12 @@ class ALSTM(Model):
self.train_optimizer.step()
def test_epoch(self, data_loader):
self.ALSTM_model.eval()
scores = []
losses = []
for (data, weight) in data_loader:
for data, weight in data_loader:
feature = data[:, :, 0:-1].to(self.device)
# feature[torch.isnan(feature)] = 0
label = data[:, -1, -1].to(self.device)
@@ -295,7 +291,6 @@ class ALSTM(Model):
preds = []
for data in test_loader:
feature = data[:, :, 0:-1].to(self.device)
with torch.no_grad():

4
qlib/contrib/model/pytorch_gats.py
View File

@@ -154,7 +154,6 @@ class GATs(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -175,7 +174,6 @@ class GATs(Model):
return daily_index, daily_count
def train_epoch(self, x_train, y_train):
x_train_values = x_train.values
y_train_values = np.squeeze(y_train.values)
self.GAT_model.train()
@@ -197,7 +195,6 @@ class GATs(Model):
self.train_optimizer.step()
def test_epoch(self, data_x, data_y):
# prepare training data
x_values = data_x.values
y_values = np.squeeze(data_y.values)
@@ -230,7 +227,6 @@ class GATs(Model):
evals_result=dict(),
save_path=None,
):
df_train, df_valid, df_test = dataset.prepare(
["train", "valid", "test"],
col_set=["feature", "label"],

8
qlib/contrib/model/pytorch_gats_ts.py
View File

@@ -32,7 +32,6 @@ class DailyBatchSampler(Sampler):
self.daily_index[0] = 0
def __iter__(self):
for idx, count in zip(self.daily_index, self.daily_count):
yield np.arange(idx, idx + count)
@@ -173,7 +172,6 @@ class GATs(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -194,11 +192,9 @@ class GATs(Model):
return daily_index, daily_count
def train_epoch(self, data_loader):
self.GAT_model.train()
for data in data_loader:
data = data.squeeze()
feature = data[:, :, 0:-1].to(self.device)
label = data[:, -1, -1].to(self.device)
@@ -212,14 +208,12 @@ class GATs(Model):
self.train_optimizer.step()
def test_epoch(self, data_loader):
self.GAT_model.eval()
scores = []
losses = []
for data in data_loader:
data = data.squeeze()
feature = data[:, :, 0:-1].to(self.device)
# feature[torch.isnan(feature)] = 0
@@ -240,7 +234,6 @@ class GATs(Model):
evals_result=dict(),
save_path=None,
):
dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
if dl_train.empty or dl_valid.empty:
@@ -329,7 +322,6 @@ class GATs(Model):
preds = []
for data in test_loader:
data = data.squeeze()
feature = data[:, :, 0:-1].to(self.device)

7
qlib/contrib/model/pytorch_gru.py
View File

@@ -146,7 +146,6 @@ class GRU(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -155,7 +154,6 @@ class GRU(Model):
raise ValueError("unknown metric `%s`" % self.metric)
def train_epoch(self, x_train, y_train):
x_train_values = x_train.values
y_train_values = np.squeeze(y_train.values)
@@ -165,7 +163,6 @@ class GRU(Model):
np.random.shuffle(indices)
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -181,7 +178,6 @@ class GRU(Model):
self.train_optimizer.step()
def test_epoch(self, data_x, data_y):
# prepare training data
x_values = data_x.values
y_values = np.squeeze(data_y.values)
@@ -194,7 +190,6 @@ class GRU(Model):
indices = np.arange(len(x_values))
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -217,7 +212,6 @@ class GRU(Model):
evals_result=dict(),
save_path=None,
):
df_train, df_valid, df_test = dataset.prepare(
["train", "valid", "test"],
col_set=["feature", "label"],
@@ -282,7 +276,6 @@ class GRU(Model):
preds = []
for begin in range(sample_num)[:: self.batch_size]:
if sample_num - begin < self.batch_size:
end = sample_num
else:

9
qlib/contrib/model/pytorch_gru_ts.py
View File

@@ -154,7 +154,6 @@ class GRU(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -163,10 +162,9 @@ class GRU(Model):
raise ValueError("unknown metric `%s`" % self.metric)
def train_epoch(self, data_loader):
self.GRU_model.train()
for (data, weight) in data_loader:
for data, weight in data_loader:
feature = data[:, :, 0:-1].to(self.device)
label = data[:, -1, -1].to(self.device)
@@ -179,14 +177,12 @@ class GRU(Model):
self.train_optimizer.step()
def test_epoch(self, data_loader):
self.GRU_model.eval()
scores = []
losses = []
for (data, weight) in data_loader:
for data, weight in data_loader:
feature = data[:, :, 0:-1].to(self.device)
# feature[torch.isnan(feature)] = 0
label = data[:, -1, -1].to(self.device)
@@ -293,7 +289,6 @@ class GRU(Model):
preds = []
for data in test_loader:
feature = data[:, :, 0:-1].to(self.device)
with torch.no_grad():

3
qlib/contrib/model/pytorch_hist.py
View File

@@ -160,7 +160,6 @@ class HIST(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric == "ic":
@@ -189,7 +188,6 @@ class HIST(Model):
return daily_index, daily_count
def train_epoch(self, x_train, y_train, stock_index):
stock2concept_matrix = np.load(self.stock2concept)
x_train_values = x_train.values
y_train_values = np.squeeze(y_train.values)
@@ -214,7 +212,6 @@ class HIST(Model):
self.train_optimizer.step()
def test_epoch(self, data_x, data_y, stock_index):
# prepare training data
stock2concept_matrix = np.load(self.stock2concept)
x_values = data_x.values

4
qlib/contrib/model/pytorch_igmtf.py
View File

@@ -153,7 +153,6 @@ class IGMTF(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric == "ic":
@@ -201,7 +200,6 @@ class IGMTF(Model):
return train_hidden, train_hidden_day
def train_epoch(self, x_train, y_train, train_hidden, train_hidden_day):
x_train_values = x_train.values
y_train_values = np.squeeze(y_train.values)
@@ -222,7 +220,6 @@ class IGMTF(Model):
self.train_optimizer.step()
def test_epoch(self, data_x, data_y, train_hidden, train_hidden_day):
# prepare training data
x_values = data_x.values
y_values = np.squeeze(data_y.values)
@@ -254,7 +251,6 @@ class IGMTF(Model):
evals_result=dict(),
save_path=None,
):
df_train, df_valid = dataset.prepare(
["train", "valid"],
col_set=["feature", "label"],

8
qlib/contrib/model/pytorch_localformer.py
View File

@@ -46,7 +46,6 @@ class LocalformerModel(Model):
seed=None,
**kwargs
):
# set hyper-parameters.
self.d_model = d_model
self.dropout = dropout
@@ -96,7 +95,6 @@ class LocalformerModel(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -105,7 +103,6 @@ class LocalformerModel(Model):
raise ValueError("unknown metric `%s`" % self.metric)
def train_epoch(self, x_train, y_train):
x_train_values = x_train.values
y_train_values = np.squeeze(y_train.values)
@@ -115,7 +112,6 @@ class LocalformerModel(Model):
np.random.shuffle(indices)
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -131,7 +127,6 @@ class LocalformerModel(Model):
self.train_optimizer.step()
def test_epoch(self, data_x, data_y):
# prepare training data
x_values = data_x.values
y_values = np.squeeze(data_y.values)
@@ -144,7 +139,6 @@ class LocalformerModel(Model):
indices = np.arange(len(x_values))
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -167,7 +161,6 @@ class LocalformerModel(Model):
evals_result=dict(),
save_path=None,
):
df_train, df_valid, df_test = dataset.prepare(
["train", "valid", "test"],
col_set=["feature", "label"],
@@ -232,7 +225,6 @@ class LocalformerModel(Model):
preds = []
for begin in range(sample_num)[:: self.batch_size]:
if sample_num - begin < self.batch_size:
end = sample_num
else:

6
qlib/contrib/model/pytorch_localformer_ts.py
View File

@@ -44,7 +44,6 @@ class LocalformerModel(Model):
seed=None,
**kwargs
):
# set hyper-parameters.
self.d_model = d_model
self.dropout = dropout
@@ -96,7 +95,6 @@ class LocalformerModel(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -105,7 +103,6 @@ class LocalformerModel(Model):
raise ValueError("unknown metric `%s`" % self.metric)
def train_epoch(self, data_loader):
self.model.train()
for data in data_loader:
@@ -121,14 +118,12 @@ class LocalformerModel(Model):
self.train_optimizer.step()
def test_epoch(self, data_loader):
self.model.eval()
scores = []
losses = []
for data in data_loader:
feature = data[:, :, 0:-1].to(self.device)
label = data[:, -1, -1].to(self.device)
@@ -148,7 +143,6 @@ class LocalformerModel(Model):
evals_result=dict(),
save_path=None,
):
dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
if dl_train.empty or dl_valid.empty:

6
qlib/contrib/model/pytorch_lstm.py
View File

@@ -142,7 +142,6 @@ class LSTM(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -151,7 +150,6 @@ class LSTM(Model):
raise ValueError("unknown metric `%s`" % self.metric)
def train_epoch(self, x_train, y_train):
x_train_values = x_train.values
y_train_values = np.squeeze(y_train.values)
@@ -161,7 +159,6 @@ class LSTM(Model):
np.random.shuffle(indices)
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -177,7 +174,6 @@ class LSTM(Model):
self.train_optimizer.step()
def test_epoch(self, data_x, data_y):
# prepare training data
x_values = data_x.values
y_values = np.squeeze(data_y.values)
@@ -190,7 +186,6 @@ class LSTM(Model):
indices = np.arange(len(x_values))
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -212,7 +207,6 @@ class LSTM(Model):
evals_result=dict(),
save_path=None,
):
df_train, df_valid, df_test = dataset.prepare(
["train", "valid", "test"],
col_set=["feature", "label"],

9
qlib/contrib/model/pytorch_lstm_ts.py
View File

@@ -150,7 +150,6 @@ class LSTM(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -159,10 +158,9 @@ class LSTM(Model):
raise ValueError("unknown metric `%s`" % self.metric)
def train_epoch(self, data_loader):
self.LSTM_model.train()
for (data, weight) in data_loader:
for data, weight in data_loader:
feature = data[:, :, 0:-1].to(self.device)
label = data[:, -1, -1].to(self.device)
@@ -175,14 +173,12 @@ class LSTM(Model):
self.train_optimizer.step()
def test_epoch(self, data_loader):
self.LSTM_model.eval()
scores = []
losses = []
for (data, weight) in data_loader:
for data, weight in data_loader:
feature = data[:, :, 0:-1].to(self.device)
# feature[torch.isnan(feature)] = 0
label = data[:, -1, -1].to(self.device)
@@ -288,7 +284,6 @@ class LSTM(Model):
preds = []
for data in test_loader:
feature = data[:, :, 0:-1].to(self.device)
with torch.no_grad():

6
qlib/contrib/model/pytorch_sfm.py
View File

@@ -306,7 +306,6 @@ class SFM(Model):
return self.device != torch.device("cpu")
def test_epoch(self, data_x, data_y):
# prepare training data
x_values = data_x.values
y_values = np.squeeze(data_y.values)
@@ -319,7 +318,6 @@ class SFM(Model):
indices = np.arange(len(x_values))
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -336,7 +334,6 @@ class SFM(Model):
return np.mean(losses), np.mean(scores)
def train_epoch(self, x_train, y_train):
x_train_values = x_train.values
y_train_values = np.squeeze(y_train.values)
@@ -346,7 +343,6 @@ class SFM(Model):
np.random.shuffle(indices)
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -367,7 +363,6 @@ class SFM(Model):
evals_result=dict(),
save_path=None,
):
df_train, df_valid = dataset.prepare(
["train", "valid"],
col_set=["feature", "label"],
@@ -431,7 +426,6 @@ class SFM(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):

4
qlib/contrib/model/pytorch_tabnet.py
View File

@@ -256,7 +256,6 @@ class TabnetModel(Model):
indices = np.arange(len(x_values))
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
feature = x_values[indices[i : i + self.batch_size]].float().to(self.device)
@@ -283,7 +282,6 @@ class TabnetModel(Model):
np.random.shuffle(indices)
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -308,7 +306,6 @@ class TabnetModel(Model):
self.tabnet_decoder.train()
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -339,7 +336,6 @@ class TabnetModel(Model):
losses = []
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break

6
qlib/contrib/model/pytorch_tcn.py
View File

@@ -154,7 +154,6 @@ class TCN(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -163,7 +162,6 @@ class TCN(Model):
raise ValueError("unknown metric `%s`" % self.metric)
def train_epoch(self, x_train, y_train):
x_train_values = x_train.values
y_train_values = np.squeeze(y_train.values)
@@ -173,7 +171,6 @@ class TCN(Model):
np.random.shuffle(indices)
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -200,7 +197,6 @@ class TCN(Model):
indices = np.arange(len(x_values))
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -223,7 +219,6 @@ class TCN(Model):
evals_result=dict(),
save_path=None,
):
df_train, df_valid, df_test = dataset.prepare(
["train", "valid", "test"],
col_set=["feature", "label"],
@@ -286,7 +281,6 @@ class TCN(Model):
preds = []
for begin in range(sample_num)[:: self.batch_size]:
if sample_num - begin < self.batch_size:
end = sample_num
else:

4
qlib/contrib/model/pytorch_tcn_ts.py
View File

@@ -155,7 +155,6 @@ class TCN(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -164,7 +163,6 @@ class TCN(Model):
raise ValueError("unknown metric `%s`" % self.metric)
def train_epoch(self, data_loader):
self.TCN_model.train()
for data in data_loader:
@@ -181,7 +179,6 @@ class TCN(Model):
self.train_optimizer.step()
def test_epoch(self, data_loader):
self.TCN_model.eval()
scores = []
@@ -277,7 +274,6 @@ class TCN(Model):
preds = []
for data in test_loader:
feature = data[:, :, 0:-1].to(self.device)
with torch.no_grad():

7
qlib/contrib/model/pytorch_tcts.py
View File

@@ -119,7 +119,6 @@ class TCTS(Model):
)
def loss_fn(self, pred, label, weight):
if self.mode == "hard":
loc = torch.argmax(weight, 1)
loss = (pred - label[np.arange(weight.shape[0]), loc]) ** 2
@@ -157,7 +156,6 @@ class TCTS(Model):
for i in range(self.steps):
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -191,7 +189,6 @@ class TCTS(Model):
# fix forecasting model and valid weight model
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -212,7 +209,6 @@ class TCTS(Model):
self.weight_optimizer.step()
def test_epoch(self, data_x, data_y):
# prepare training data
x_values = data_x.values
y_values = np.squeeze(data_y.values)
@@ -224,7 +220,6 @@ class TCTS(Model):
indices = np.arange(len(x_values))
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -282,7 +277,6 @@ class TCTS(Model):
verbose=True,
save_path=None,
):
self.fore_model = GRUModel(
d_feat=self.d_feat,
hidden_size=self.hidden_size,
@@ -366,7 +360,6 @@ class TCTS(Model):
preds = []
for begin in range(sample_num)[:: self.batch_size]:
if sample_num - begin < self.batch_size:
end = sample_num
else:

10
qlib/contrib/model/pytorch_tra.py
View File

@@ -84,7 +84,6 @@ class TRAModel(Model):
transport_method="none",
memory_mode="sample",
):
self.logger = get_module_logger("TRA")
assert memory_mode in ["sample", "daily"], "invalid memory mode"
@@ -136,7 +135,6 @@ class TRAModel(Model):
self._init_model()
def _init_model(self):
self.logger.info("init TRAModel...")
self.model = eval(self.model_type)(**self.model_config).to(device)
@@ -176,7 +174,6 @@ class TRAModel(Model):
self.global_step = -1
def train_epoch(self, epoch, data_set, is_pretrain=False):
self.model.train()
self.tra.train()
data_set.train()
@@ -274,7 +271,6 @@ class TRAModel(Model):
return total_loss
def test_epoch(self, epoch, data_set, return_pred=False, prefix="test", is_pretrain=False):
self.model.eval()
self.tra.eval()
data_set.eval()
@@ -360,7 +356,6 @@ class TRAModel(Model):
return metrics, preds, probs, P_all
def _fit(self, train_set, valid_set, test_set, evals_result, is_pretrain=True):
best_score = -1
best_epoch = 0
stop_rounds = 0
@@ -419,7 +414,6 @@ class TRAModel(Model):
return best_score
def fit(self, dataset, evals_result=dict()):
assert isinstance(dataset, MTSDatasetH), "TRAModel only supports `qlib.contrib.data.dataset.MTSDatasetH`"
train_set, valid_set, test_set = dataset.prepare(["train", "valid", "test"])
@@ -503,7 +497,6 @@ class TRAModel(Model):
json.dump(info, f)
def predict(self, dataset, segment="test"):
assert isinstance(dataset, MTSDatasetH), "TRAModel only supports `qlib.contrib.data.dataset.MTSDatasetH`"
if not self.fitted:
@@ -571,7 +564,6 @@ class RNN(nn.Module):
self.output_size = hidden_size
def forward(self, x):
if self.input_proj is not None:
x = self.input_proj(x)
@@ -647,7 +639,6 @@ class Transformer(nn.Module):
self.output_size = hidden_size
def forward(self, x):
x = x.permute(1, 0, 2).contiguous() # the first dim need to be time
x = self.pe(x)
@@ -713,7 +704,6 @@ class TRA(nn.Module):
child.reset_parameters()
def forward(self, hidden, hist_loss):
preds = self.predictors(hidden)
if self.num_states == 1: # no need for router when having only one prediction

8
qlib/contrib/model/pytorch_transformer.py
View File

@@ -45,7 +45,6 @@ class TransformerModel(Model):
seed=None,
**kwargs
):
# set hyper-parameters.
self.d_model = d_model
self.dropout = dropout
@@ -95,7 +94,6 @@ class TransformerModel(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -104,7 +102,6 @@ class TransformerModel(Model):
raise ValueError("unknown metric `%s`" % self.metric)
def train_epoch(self, x_train, y_train):
x_train_values = x_train.values
y_train_values = np.squeeze(y_train.values)
@@ -114,7 +111,6 @@ class TransformerModel(Model):
np.random.shuffle(indices)
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -130,7 +126,6 @@ class TransformerModel(Model):
self.train_optimizer.step()
def test_epoch(self, data_x, data_y):
# prepare training data
x_values = data_x.values
y_values = np.squeeze(data_y.values)
@@ -143,7 +138,6 @@ class TransformerModel(Model):
indices = np.arange(len(x_values))
for i in range(len(indices))[:: self.batch_size]:
if len(indices) - i < self.batch_size:
break
@@ -166,7 +160,6 @@ class TransformerModel(Model):
evals_result=dict(),
save_path=None,
):
df_train, df_valid, df_test = dataset.prepare(
["train", "valid", "test"],
col_set=["feature", "label"],
@@ -231,7 +224,6 @@ class TransformerModel(Model):
preds = []
for begin in range(sample_num)[:: self.batch_size]:
if sample_num - begin < self.batch_size:
end = sample_num
else:

6
qlib/contrib/model/pytorch_transformer_ts.py
View File

@@ -43,7 +43,6 @@ class TransformerModel(Model):
seed=None,
**kwargs
):
# set hyper-parameters.
self.d_model = d_model
self.dropout = dropout
@@ -93,7 +92,6 @@ class TransformerModel(Model):
raise ValueError("unknown loss `%s`" % self.loss)
def metric_fn(self, pred, label):
mask = torch.isfinite(label)
if self.metric in ("", "loss"):
@@ -102,7 +100,6 @@ class TransformerModel(Model):
raise ValueError("unknown metric `%s`" % self.metric)
def train_epoch(self, data_loader):
self.model.train()
for data in data_loader:
@@ -118,14 +115,12 @@ class TransformerModel(Model):
self.train_optimizer.step()
def test_epoch(self, data_loader):
self.model.eval()
scores = []
losses = []
for data in data_loader:
feature = data[:, :, 0:-1].to(self.device)
label = data[:, -1, -1].to(self.device)
@@ -145,7 +140,6 @@ class TransformerModel(Model):
evals_result=dict(),
save_path=None,
):
dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)

1
qlib/contrib/model/xgboost.py
View File

@@ -30,7 +30,6 @@ class XGBModel(Model, FeatureInt):
reweighter=None,
**kwargs
):
df_train, df_valid = dataset.prepare(
["train", "valid"],
col_set=["feature", "label"],

1
qlib/contrib/report/data/ana.py
View File

@@ -30,7 +30,6 @@ class CombFeaAna(FeaAnalyser):
"""The statistics of features are finished in the underlying analysers"""
def plot_all(self, *args, **kwargs):
ax_gen = iter(sub_fig_generator(row_n=len(self._fea_ana_l), *args, **kwargs))
for col in self._dataset:

1
qlib/contrib/report/data/base.py
View File

@@ -28,7 +28,6 @@ class FeaAnalyser:
return False
def plot_all(self, *args, **kwargs):
ax_gen = iter(sub_fig_generator(*args, **kwargs))
for col in self._dataset:
if not self.skip(col):

1
qlib/contrib/report/graph.py
View File

@@ -15,7 +15,6 @@ from plotly.figure_factory import create_distplot
class BaseGraph:
_name = None
def __init__(

7
qlib/contrib/rolling/__init__.py Normal file
View File

@@ -0,0 +1,7 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""
The difference between me and the scripts in examples/benchmarks/benchmarks_dynamic
- This module only focus provide a general rolling implementation.
Anything specific that benchmark is placed in examples/benchmarks/benchmarks_dynamic
"""

16
qlib/contrib/rolling/__main__.py Normal file
View File

@@ -0,0 +1,16 @@
import fire
from qlib import auto_init
from qlib.contrib.rolling.base import Rolling
from qlib.utils.mod import find_all_classes
if __name__ == "__main__":
sub_commands = {}
for cls in find_all_classes("qlib.contrib.rolling", Rolling):
sub_commands[cls.__module__.split(".")[-1]] = cls
# The sub_commands will be like
# {'base': <class 'qlib.contrib.rolling.base.Rolling'>, ...}
# So the you can run it with commands like command below
# - `python -m qlib.contrib.rolling base --conf_path <path to the yaml> run`
# - base can be replace with other module names
auto_init()
fire.Fire(sub_commands)

246
qlib/contrib/rolling/base.py Normal file
View File

@@ -0,0 +1,246 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
from copy import deepcopy
from pathlib import Path
from typing import List, Optional, Union
import fire
import pandas as pd
import yaml
from qlib import auto_init
from qlib.log import get_module_logger
from qlib.model.ens.ensemble import RollingEnsemble
from qlib.model.trainer import TrainerR
from qlib.utils import get_cls_kwargs, init_instance_by_config
from qlib.utils.data import update_config
from qlib.workflow import R
from qlib.workflow.record_temp import SignalRecord
from qlib.workflow.task.collect import RecorderCollector
from qlib.workflow.task.gen import RollingGen, task_generator
from qlib.workflow.task.utils import replace_task_handler_with_cache
class Rolling:
"""
The motivation of Rolling Module
- It only focus **offlinely** turn a specific task to rollinng
- To make the implementation easier, following factors are ignored.
- The tasks is dependent (e.g. time series).
Related modules and difference from me:
- MetaController: It is learning how to handle a task (e.g. learning to learn).
- But rolling is about how to split a single task into tasks in time series and run them.
- OnlineStrategy: It is focusing on serving a model, the model can be updated time dependently in time.
- Rolling is much simpler and is only for testing rolling models offline. It does not want to share the interface with OnlineStrategy.
The code about rolling is shared in `task_generator` & `RollingGen` level between me and the above modules
But it is for different purpose, so other parts are not shared.
.. code-block:: shell
# here is an typical use case of the module.
python -m qlib.contrib.rolling.base --conf_path <path to the yaml> run
**NOTE**
before running the example, please clean your previous results with following command
- `rm -r mlruns`
- Because it is very hard to permanently delete a experiment (it will be moved into .trash and raise error when creating experiment with same name).
"""
def __init__(
self,
conf_path: Union[str, Path],
exp_name: Optional[str] = None,
horizon: Optional[int] = 20,
step: int = 20,
h_path: Optional[str] = None,
train_start: Optional[str] = None,
test_end: Optional[str] = None,
task_ext_conf: Optional[dict] = None,
rolling_exp: Optional[str] = None,
) -> None:
"""
Parameters
----------
conf_path : str
Path to the config for rolling.
exp_name : Optional[str]
The exp name of the outputs (Output is a record which contains the concatenated predictions of rolling records).
horizon: Optional[int] = 20,
The horizon of the prediction target.
This is used to override the prediction horizon of the file.
h_path : Optional[str]
the dumped data handler;
It may come from other data source. It will override the data handler in the config.
test_end : Optional[str]
the test end for the data. It is typically used together with the handler
You can do the same thing with task_ext_conf in a more complicated way
train_start : Optional[str]
the train start for the data. It is typically used together with the handler.
You can do the same thing with task_ext_conf in a more complicated way
task_ext_conf : Optional[dict]
some option to update the task config.
rolling_exp : Optional[str]
The name for the experiments for rolling.
It will contains a lot of record in an experiment. Each record corresponds to a specific rolling.
Please note that it is different from the final experiments
"""
self.logger = get_module_logger("Rolling")
self.conf_path = Path(conf_path)
self.exp_name = exp_name
self._rid = None # the final combined recorder id in `exp_name`
self.step = step
assert horizon is not None, "Current version does not support extracting horizon from the underlying dataset"
self.horizon = horizon
if rolling_exp is None:
datetime_suffix = pd.Timestamp.now().strftime("%Y%m%d%H%M%S")
self.rolling_exp = f"rolling_models_{datetime_suffix}"
else:
self.rolling_exp = rolling_exp
self.logger.warning(
"Using user specifiied name for rolling models. So the experiment names duplicateds. "
"Please manually remove your experiment for rolling model with command like `rm -r mlruns`."
" Otherwise it will prevents the creating of experimen with same name"
)
self.train_start = train_start
self.test_end = test_end
self.task_ext_conf = task_ext_conf
self.h_path = h_path
# FIXME:
# - the qlib_init section will be ignored by me.
# - So we have to design a priority mechanism to solve this issue.
def _raw_conf(self) -> dict:
with self.conf_path.open("r") as f:
return yaml.safe_load(f)
def _replace_hanler_with_cache(self, task: dict):
"""
Due to the data processing part in original rolling is slow. So we have to
This class tries to add more feature
"""
if self.h_path is not None:
h_path = Path(self.h_path)
task["dataset"]["kwargs"]["handler"] = f"file://{h_path}"
else:
task = replace_task_handler_with_cache(task, self.conf_path.parent)
return task
def _update_start_end_time(self, task: dict):
if self.train_start is not None:
seg = task["dataset"]["kwargs"]["segments"]["train"]
task["dataset"]["kwargs"]["segments"]["train"] = pd.Timestamp(self.train_start), seg[1]
if self.test_end is not None:
seg = task["dataset"]["kwargs"]["segments"]["test"]
task["dataset"]["kwargs"]["segments"]["test"] = seg[0], pd.Timestamp(self.test_end)
return task
def basic_task(self, enable_handler_cache: Optional[bool] = True):
"""
The basic task may not be the exactly same as the config from `conf_path` from __init__ due to
- some parameters could be overriding by some parameters from __init__
- user could implementing sublcass to change it for higher performance
"""
task: dict = self._raw_conf()["task"]
task = deepcopy(task)
# modify dataset horizon
# NOTE:
# It assumpts that the label can be modifiled in the handler's kwargs
# But is not always a valid. It is only valid in the predefined dataset `Alpha158` & `Alpha360`
if self.horizon is None:
# TODO:
# - get horizon automatically from the expression!!!!
raise NotImplementedError(f"This type of input is not supported")
else:
self.logger.info("The prediction horizon is overrided")
task["dataset"]["kwargs"]["handler"]["kwargs"]["label"] = [
"Ref($close, -{}) / Ref($close, -1) - 1".format(self.horizon + 1)
]
if enable_handler_cache:
task = self._replace_hanler_with_cache(task)
task = self._update_start_end_time(task)
if self.task_ext_conf is not None:
task = update_config(task, self.task_ext_conf)
self.logger.info(task)
return task
def get_task_list(self) -> List[dict]:
"""return a batch of tasks for rolling."""
task = self.basic_task()
task_l = task_generator(
task, RollingGen(step=self.step, trunc_days=self.horizon + 1)
) # the last two days should be truncated to avoid information leakage
for t in task_l:
# when we rolling tasks. No further analyis is needed.
# analyis are postponed to the final ensemble.
t["record"] = ["qlib.workflow.record_temp.SignalRecord"]
return task_l
def _train_rolling_tasks(self):
task_l = self.get_task_list()
self.logger.info("Deleting previous Rolling results")
try:
# TODO: mlflow does not support permanently delete experiment
# it will be moved to .trash and prevents creating the experiments with the same name
R.delete_exp(experiment_name=self.rolling_exp) # We should remove the rolling experiments.
except ValueError:
self.logger.info("No previous rolling results")
trainer = TrainerR(experiment_name=self.rolling_exp)
trainer(task_l)
def _ens_rolling(self):
rc = RecorderCollector(
experiment=self.rolling_exp,
artifacts_key=["pred", "label"],
process_list=[RollingEnsemble()],
# rec_key_func=lambda rec: (self.COMB_EXP, rec.info["id"]),
artifacts_path={"pred": "pred.pkl", "label": "label.pkl"},
)
res = rc()
with R.start(experiment_name=self.exp_name):
R.log_params(exp_name=self.rolling_exp)
R.save_objects(**{"pred.pkl": res["pred"], "label.pkl": res["label"]})
self._rid = R.get_recorder().id
def _update_rolling_rec(self):
"""
Evaluate the combined rolling results
"""
rec = R.get_recorder(experiment_name=self.exp_name, recorder_id=self._rid)
# Follow the original analyser
records = self._raw_conf()["task"].get("record", [])
if isinstance(records, dict): # prevent only one dict
records = [records]
for record in records:
if issubclass(get_cls_kwargs(record)[0], SignalRecord):
# skip the signal record.
continue
r = init_instance_by_config(
record,
recorder=rec,
default_module="qlib.workflow.record_temp",
)
r.generate()
print(f"Your evaluation results can be found in the experiment named `{self.exp_name}`.")
def run(self):
# the results will be save in mlruns.
# 1) each rolling task is saved in rolling_models
self._train_rolling_tasks()
# 2) combined rolling tasks and evaluation results are saved in rolling
self._ens_rolling()
self._update_rolling_rec()
if __name__ == "__main__":
auto_init()
fire.Fire(Rolling)

343
qlib/contrib/rolling/ddgda.py Normal file
View File

@@ -0,0 +1,343 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
from pathlib import Path
import pickle
from typing import Optional, Union
import pandas as pd
import yaml
from qlib.contrib.meta.data_selection.dataset import InternalData, MetaDatasetDS
from qlib.contrib.meta.data_selection.model import MetaModelDS
from qlib.data.dataset.handler import DataHandlerLP
from qlib.model.meta.task import MetaTask
from qlib.model.trainer import TrainerR
from qlib.typehint import Literal
from qlib.utils import init_instance_by_config
from qlib.workflow import R
from qlib.workflow.task.utils import replace_task_handler_with_cache
from .base import Rolling
# LGBM is designed for feature importance & similarity
LGBM_MODEL = """
class: LGBModel
module_path: qlib.contrib.model.gbdt
kwargs:
loss: mse
colsample_bytree: 0.8879
learning_rate: 0.2
subsample: 0.8789
lambda_l1: 205.6999
lambda_l2: 580.9768
max_depth: 8
num_leaves: 210
num_threads: 20
"""
# covnert the yaml to dict
LGBM_MODEL = yaml.load(LGBM_MODEL, Loader=yaml.FullLoader)
LINEAR_MODEL = """
class: LinearModel
module_path: qlib.contrib.model.linear
kwargs:
estimator: ridge
alpha: 0.05
"""
LINEAR_MODEL = yaml.load(LINEAR_MODEL, Loader=yaml.FullLoader)
PROC_ARGS = """
infer_processors:
- class: RobustZScoreNorm
kwargs:
fields_group: feature
clip_outlier: true
- class: Fillna
kwargs:
fields_group: feature
learn_processors:
- class: DropnaLabel
- class: CSRankNorm
kwargs:
fields_group: label
"""
PROC_ARGS = yaml.load(PROC_ARGS, Loader=yaml.FullLoader)
UTIL_MODEL_TYPE = Literal["linear", "gbdt"]
class DDGDA(Rolling):
"""
It is a rolling based on DDG-DA
**NOTE**
before running the example, please clean your previous results with following command
- `rm -r mlruns`
"""
def __init__(
self,
sim_task_model: UTIL_MODEL_TYPE = "gbdt",
meta_1st_train_end: Optional[str] = None,
alpha: float = 0.01,
working_dir: Optional[Union[str, Path]] = None,
**kwargs,
):
"""
Parameters
----------
sim_task_model: Literal["linear", "gbdt"] = "gbdt",
The model for calculating similarity between data.
meta_1st_train_end: Optional[str]
the datetime of training end of the first meta_task
alpha: float
Setting the L2 regularization for ridge
The `alpha` is only passed to MetaModelDS (it is not passed to sim_task_model currently..)
"""
# NOTE:
# the horizon must match the meaning in the base task template
self.meta_exp_name = "DDG-DA"
self.sim_task_model: UTIL_MODEL_TYPE = sim_task_model # The model to capture the distribution of data.
self.alpha = alpha
self.meta_1st_train_end = meta_1st_train_end
super().__init__(**kwargs)
self.working_dir = self.conf_path.parent if working_dir is None else Path(working_dir)
self.proxy_hd = self.working_dir / "handler_proxy.pkl"
def _adjust_task(self, task: dict, astype: UTIL_MODEL_TYPE):
"""
some task are use for special purpose.
For example:
- GBDT for calculating feature importance
- Linear or GBDT for calculating similarity
- Datset (well processed) that aligned to Linear that for meta learning
"""
# NOTE: here is just for aligning with previous implementation
# It is not necessary for the current implementation
handler = task["dataset"].setdefault("kwargs", {}).setdefault("handler", {})
if astype == "gbdt":
task["model"] = LGBM_MODEL
if isinstance(handler, dict):
for k in ["infer_processors", "learn_processors"]:
if k in handler.setdefault("kwargs", {}):
handler["kwargs"].pop(k)
elif astype == "linear":
task["model"] = LINEAR_MODEL
handler["kwargs"].update(PROC_ARGS)
else:
raise ValueError(f"astype not supported: {astype}")
return task
def _get_feature_importance(self):
# this must be lightGBM, because it needs to get the feature importance
task = self.basic_task(enable_handler_cache=False)
task = self._adjust_task(task, astype="gbdt")
task = replace_task_handler_with_cache(task, self.working_dir)
with R.start(experiment_name="feature_importance"):
model = init_instance_by_config(task["model"])
dataset = init_instance_by_config(task["dataset"])
model.fit(dataset)
fi = model.get_feature_importance()
# Because the model use numpy instead of dataframe for training lightgbm
# So the we must use following extra steps to get the right feature importance
df = dataset.prepare(segments=slice(None), col_set="feature", data_key=DataHandlerLP.DK_R)
cols = df.columns
fi_named = {cols[int(k.split("_")[1])]: imp for k, imp in fi.to_dict().items()}
return pd.Series(fi_named)
def _dump_data_for_proxy_model(self):
"""
Dump data for training meta model.
The meta model will be trained upon the proxy forecasting model.
This dataset is for the proxy forecasting model.
"""
topk = 30
fi = self._get_feature_importance()
col_selected = fi.nlargest(topk)
# NOTE: adjusting to `self.sim_task_model` just for aligning with previous implementation.
task = self._adjust_task(self.basic_task(enable_handler_cache=False), self.sim_task_model)
task = replace_task_handler_with_cache(task, self.working_dir)
dataset = init_instance_by_config(task["dataset"])
prep_ds = dataset.prepare(slice(None), col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
feature_df = prep_ds["feature"]
label_df = prep_ds["label"]
feature_selected = feature_df.loc[:, col_selected.index]
feature_selected = feature_selected.groupby("datetime", group_keys=False).apply(
lambda df: (df - df.mean()).div(df.std())
)
feature_selected = feature_selected.fillna(0.0)
df_all = {
"label": label_df.reindex(feature_selected.index),
"feature": feature_selected,
}
df_all = pd.concat(df_all, axis=1)
df_all.to_pickle(self.working_dir / "fea_label_df.pkl")
# dump data in handler format for aligning the interface
handler = DataHandlerLP(
data_loader={
"class": "qlib.data.dataset.loader.StaticDataLoader",
"kwargs": {"config": self.working_dir / "fea_label_df.pkl"},
}
)
handler.to_pickle(self.working_dir / self.proxy_hd, dump_all=True)
@property
def _internal_data_path(self):
return self.working_dir / f"internal_data_s{self.step}.pkl"
def _dump_meta_ipt(self):
"""
Dump data for training meta model.
This function will dump the input data for meta model
"""
# According to the experiments, the choice of the model type is very important for achieving good results
sim_task = self._adjust_task(self.basic_task(enable_handler_cache=False), astype=self.sim_task_model)
sim_task = replace_task_handler_with_cache(sim_task, self.working_dir)
if self.sim_task_model == "gbdt":
sim_task["model"].setdefault("kwargs", {}).update({"early_stopping_rounds": None, "num_boost_round": 150})
exp_name_sim = f"data_sim_s{self.step}"
internal_data = InternalData(sim_task, self.step, exp_name=exp_name_sim)
internal_data.setup(trainer=TrainerR)
with self._internal_data_path.open("wb") as f:
pickle.dump(internal_data, f)
def _train_meta_model(self, fill_method="max"):
"""
training a meta model based on a simplified linear proxy model;
"""
# 1) leverage the simplified proxy forecasting model to train meta model.
# - Only the dataset part is important, in current version of meta model will integrate the
# the train_start for training meta model does not necessarily align with final rolling
train_start = "2008-01-01" if self.train_start is None else self.train_start
train_end = "2010-12-31" if self.meta_1st_train_end is None else self.meta_1st_train_end
test_start = (pd.Timestamp(train_end) + pd.Timedelta(days=1)).strftime("%Y-%m-%d")
proxy_forecast_model_task = {
# "model": "qlib.contrib.model.linear.LinearModel",
"dataset": {
"class": "qlib.data.dataset.DatasetH",
"kwargs": {
"handler": f"file://{(self.working_dir / self.proxy_hd).absolute()}",
"segments": {
"train": (train_start, train_end),
"test": (test_start, self.basic_task()["dataset"]["kwargs"]["segments"]["test"][1]),
},
},
},
# "record": ["qlib.workflow.record_temp.SignalRecord"]
}
# the proxy_forecast_model_task will be used to create meta tasks.
# The test date of first task will be 2011-01-01. Each test segment will be about 20days
# The tasks include all training tasks and test tasks.
# 2) preparing meta dataset
kwargs = dict(
task_tpl=proxy_forecast_model_task,
step=self.step,
segments=0.62, # keep test period consistent with the dataset yaml
trunc_days=1 + self.horizon,
hist_step_n=30,
fill_method=fill_method,
rolling_ext_days=0,
)
# NOTE:
# the input of meta model (internal data) are shared between proxy model and final forecasting model
# but their task test segment are not aligned! It worked in my previous experiment.
# So the misalignment will not affect the effectiveness of the method.
with self._internal_data_path.open("rb") as f:
internal_data = pickle.load(f)
md = MetaDatasetDS(exp_name=internal_data, **kwargs)
# 3) train and logging meta model
with R.start(experiment_name=self.meta_exp_name):
R.log_params(**kwargs)
mm = MetaModelDS(
step=self.step, hist_step_n=kwargs["hist_step_n"], lr=0.001, max_epoch=30, seed=43, alpha=self.alpha
)
mm.fit(md)
R.save_objects(model=mm)
@property
def _task_path(self):
return self.working_dir / f"tasks_s{self.step}.pkl"
def get_task_list(self):
"""
Leverage meta-model for inference:
- Given
- baseline tasks
- input for meta model(internal data)
- meta model (its learnt knowledge on proxy forecasting model is expected to transfer to normal forecasting model)
"""
# 1) get meta model
exp = R.get_exp(experiment_name=self.meta_exp_name)
rec = exp.list_recorders(rtype=exp.RT_L)[0]
meta_model: MetaModelDS = rec.load_object("model")
# 2)
# we are transfer to knowledge of meta model to final forecasting tasks.
# Create MetaTaskDataset for the final forecasting tasks
# Aligning the setting of it to the MetaTaskDataset when training Meta model is necessary
# 2.1) get previous config
param = rec.list_params()
trunc_days = int(param["trunc_days"])
step = int(param["step"])
hist_step_n = int(param["hist_step_n"])
fill_method = param.get("fill_method", "max")
task_l = super().get_task_list()
# 2.2) create meta dataset for final dataset
kwargs = dict(
task_tpl=task_l,
step=step,
segments=0.0, # all the tasks are for testing
trunc_days=trunc_days,
hist_step_n=hist_step_n,
fill_method=fill_method,
task_mode=MetaTask.PROC_MODE_TRANSFER,
)
with self._internal_data_path.open("rb") as f:
internal_data = pickle.load(f)
mds = MetaDatasetDS(exp_name=internal_data, **kwargs)
# 3) meta model make inference and get new qlib task
new_tasks = meta_model.inference(mds)
with self._task_path.open("wb") as f:
pickle.dump(new_tasks, f)
return new_tasks
def run(self):
# prepare the meta model for rolling ---------
# 1) file: handler_proxy.pkl (self.proxy_hd)
self._dump_data_for_proxy_model()
# 2)
# file: internal_data_s20.pkl
# mlflow: data_sim_s20, models for calculating meta_ipt
self._dump_meta_ipt()
# 3) meta model will be stored in `DDG-DA`
self._train_meta_model()
# Run rolling --------------------------------
# 4) new_tasks are saved in "tasks_s20.pkl" (reweighter is added)
# - the meta inference are done when calling `get_task_list`
# 5) load the saved tasks and train model
super().run()

1
qlib/contrib/strategy/optimizer/optimizer.py
View File

@@ -112,7 +112,6 @@ class PortfolioOptimizer(BaseOptimizer):
return w
def _optimize(self, S: np.ndarray, r: Optional[np.ndarray] = None, w0: Optional[np.ndarray] = None) -> np.ndarray:
# inverse volatility
if self.method == self.OPT_INV:
if r is not None:

1
qlib/contrib/strategy/rule_strategy.py
View File

@@ -522,7 +522,6 @@ class ACStrategy(BaseStrategy):
_order_amount = min(_order_amount, self.trade_amount[order.stock_id])
if _order_amount > 1e-5:
_order = Order(
stock_id=order.stock_id,
amount=_order_amount,

2
qlib/contrib/strategy/signal_strategy.py
View File

@@ -435,7 +435,6 @@ class EnhancedIndexingStrategy(WeightStrategyBase):
self._riskdata_cache = {}
def get_risk_data(self, date):
if date in self._riskdata_cache:
return self._riskdata_cache[date]
@@ -462,7 +461,6 @@ class EnhancedIndexingStrategy(WeightStrategyBase):
return self._riskdata_cache[date]
def generate_target_weight_position(self, score, current, trade_start_time, trade_end_time):
trade_date = trade_start_time
pre_date = get_pre_trading_date(trade_date, future=True) # previous trade date

2
qlib/contrib/tuner/config.py
View File

@@ -11,7 +11,6 @@ import os
class TunerConfigManager:
def __init__(self, config_path):
if not config_path:
raise ValueError("Config path is invalid.")
self.config_path = config_path
@@ -58,7 +57,6 @@ class PipelineExperimentConfig:
class OptimizationConfig:
def __init__(self, config, TUNER_CONFIG_MANAGER):
self.report_type = config.get("report_type", "pred_long")
if self.report_type not in [
"pred_long",

4
qlib/contrib/tuner/pipeline.py
View File

@@ -15,11 +15,9 @@ from ...utils import get_module_by_module_path
class Pipeline:
GLOBAL_BEST_PARAMS_NAME = "global_best_params.json"
def __init__(self, tuner_config_manager):
self.logger = get_module_logger("Pipeline", sh_level=logging.INFO)
self.tuner_config_manager = tuner_config_manager
@@ -37,7 +35,6 @@ class Pipeline:
self.best_tuner_index = None
def run(self):
TimeInspector.set_time_mark()
for tuner_index, tuner_config in enumerate(self.pipeline_config):
tuner = self.init_tuner(tuner_index, tuner_config)
@@ -77,7 +74,6 @@ class Pipeline:
return tuner_class(tuner_config, self.optim_config)
def save_tuner_exp_info(self):
TimeInspector.set_time_mark()
save_path = os.path.join(self.pipeline_ex_config.tuner_ex_dir, Pipeline.GLOBAL_BEST_PARAMS_NAME)
with open(save_path, "w") as fp:

7
qlib/contrib/tuner/tuner.py
View File

@@ -24,7 +24,6 @@ from hyperopt import STATUS_OK, STATUS_FAIL
class Tuner:
def __init__(self, tuner_config, optim_config):
self.logger = get_module_logger("Tuner", sh_level=logging.INFO)
self.tuner_config = tuner_config
@@ -42,7 +41,6 @@ class Tuner:
self.space = self.setup_space()
def tune(self):
TimeInspector.set_time_mark()
fmin(
fn=self.objective,
@@ -84,7 +82,6 @@ class Tuner:
class QLibTuner(Tuner):
ESTIMATOR_CONFIG_NAME = "estimator_config.yaml"
EXP_INFO_NAME = "exp_info.json"
EXP_RESULT_DIR = "sacred/{}"
@@ -92,7 +89,6 @@ class QLibTuner(Tuner):
LOCAL_BEST_PARAMS_NAME = "local_best_params.json"
def objective(self, params):
# 1. Setup an config for a specific estimator process
estimator_path = self.setup_estimator_config(params)
self.logger.info("Searching params: {} ".format(params))
@@ -120,7 +116,6 @@ class QLibTuner(Tuner):
return {"loss": res, "status": status}
def fetch_result(self):
# 1. Get experiment information
exp_info_path = os.path.join(self.ex_dir, QLibTuner.EXP_INFO_NAME)
with open(exp_info_path) as fp:
@@ -155,7 +150,6 @@ class QLibTuner(Tuner):
return np.abs(res.values[0] - 1)
def setup_estimator_config(self, params):
estimator_config = copy.deepcopy(self.tuner_config)
estimator_config["model"].update({"args": params["model_space"]})
estimator_config["strategy"].update({"args": params["strategy_space"]})
@@ -212,7 +206,6 @@ class QLibTuner(Tuner):
return space
def save_local_best_params(self):
TimeInspector.set_time_mark()
local_best_params_path = os.path.join(self.ex_dir, QLibTuner.LOCAL_BEST_PARAMS_NAME)
with open(local_best_params_path, "w") as fp:

5
qlib/data/cache.py
View File

@@ -583,7 +583,6 @@ class DiskExpressionCache(ExpressionCache):
r.tofile(str(cache_path))
def update(self, sid, cache_uri, freq: str = "day"):
cp_cache_uri = self.get_cache_dir(freq).joinpath(sid).joinpath(cache_uri)
meta_path = cp_cache_uri.with_suffix(".meta")
if not self.check_cache_exists(cp_cache_uri, suffix_list=[".meta"]):
@@ -696,7 +695,6 @@ class DiskDatasetCache(DatasetCache):
def _dataset(
self, instruments, fields, start_time=None, end_time=None, freq="day", disk_cache=0, inst_processors=[]
):
if disk_cache == 0:
# In this case, data_set cache is configured but will not be used.
return self.provider.dataset(
@@ -801,7 +799,6 @@ class DiskDatasetCache(DatasetCache):
KEY = "df"
def __init__(self, cache_path: Union[str, Path]):
self.index_path = cache_path.with_suffix(".index")
self._data = None
self.logger = get_module_logger(self.__class__.__name__)
@@ -1126,7 +1123,6 @@ class DatasetURICache(DatasetCache):
def dataset(
self, instruments, fields, start_time=None, end_time=None, freq="day", disk_cache=0, inst_processors=[]
):
if "local" in C.dataset_provider.lower():
# use LocalDatasetProvider
return self.provider.dataset(
@@ -1189,7 +1185,6 @@ class MemoryCalendarCache(CalendarCache):
uri = self._uri(start_time, end_time, freq, future)
result, expire = MemCacheExpire.get_cache(H["c"], uri)
if result is None or expire:
result = self.provider.calendar(start_time, end_time, freq, future)
MemCacheExpire.set_cache(H["c"], uri, result)

1
qlib/data/data.py
View File

@@ -1096,7 +1096,6 @@ class ClientDatasetProvider(DatasetProvider):
else:
return data
else:
"""
Call the server to generate the data-set cache, get the uri of the cache file.
Then load the data from the file on NFS directly.

1
qlib/data/dataset/processor.py
View File

@@ -132,7 +132,6 @@ class FilterCol(Processor):
self.col_list = col_list
def __call__(self, df):
cols = get_group_columns(df, self.fields_group)
all_cols = df.columns
diff_cols = np.setdiff1d(all_cols.get_level_values(-1), cols.get_level_values(-1))

8
qlib/data/dataset/utils.py
View File

@@ -71,15 +71,11 @@ def fetch_df_by_index(
if fetch_orig:
for slc in idx_slc:
if slc != slice(None, None):
return df.loc[
pd.IndexSlice[idx_slc],
]
return df.loc[pd.IndexSlice[idx_slc],] # noqa: E231
else: # pylint: disable=W0120
return df
else:
return df.loc[
pd.IndexSlice[idx_slc],
]
return df.loc[pd.IndexSlice[idx_slc],] # noqa: E231
def fetch_df_by_col(df: pd.DataFrame, col_set: Union[str, List[str]]) -> pd.DataFrame:

1
qlib/data/pit.py
View File

@@ -22,7 +22,6 @@ from .data import Cal
class P(ElemOperator):
def _load_internal(self, instrument, start_index, end_index, freq):
_calendar = Cal.calendar(freq=freq)
resample_data = np.empty(end_index - start_index + 1, dtype="float32")

3
qlib/data/storage/file_storage.py
View File

@@ -190,7 +190,6 @@ class FileCalendarStorage(FileStorageMixin, CalendarStorage):
class FileInstrumentStorage(FileStorageMixin, InstrumentStorage):
INSTRUMENT_SEP = "\t"
INSTRUMENT_START_FIELD = "start_datetime"
INSTRUMENT_END_FIELD = "end_datetime"
@@ -260,7 +259,6 @@ class FileInstrumentStorage(FileStorageMixin, InstrumentStorage):
return self._read_instrument()[k]
def update(self, *args, **kwargs) -> None:
if len(args) > 1:
raise TypeError(f"update expected at most 1 arguments, got {len(args)}")
inst = self._read_instrument()
@@ -358,7 +356,6 @@ class FileFeatureStorage(FileStorageMixin, FeatureStorage):
storage_end_index = self.end_index
with self.uri.open("rb") as fp:
if isinstance(i, int):
if storage_start_index > i:
raise IndexError(f"{i}: start index is {storage_start_index}")
fp.seek(4 * (i - storage_start_index) + 4)

1
qlib/log.py
View File

@@ -84,7 +84,6 @@ get_module_logger = _QLibLoggerManager()
class TimeInspector:
timer_logger = get_module_logger("timer")
time_marks = []

1
qlib/model/riskmodel/poet.py
View File

@@ -43,7 +43,6 @@ class POETCovEstimator(RiskModel):
self.thresh_method = thresh_method
def _predict(self, X: np.ndarray) -> np.ndarray:
Y = X.T # NOTE: to match POET's implementation
p, n = Y.shape

2
qlib/tests/__init__.py
View File

@@ -14,7 +14,6 @@ from qlib.data.storage import CalendarStorage, InstrumentStorage, FeatureStorage
class TestAutoData(unittest.TestCase):
_setup_kwargs = {}
provider_uri = "~/.qlib/qlib_data/cn_data_simple" # target_dir
provider_uri_1day = "~/.qlib/qlib_data/cn_data" # target_dir
@@ -286,6 +285,5 @@ class TestMockData(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
provider_uri = "Not necessary."
init(region=REG_TW, provider_uri=provider_uri, expression_cache=None, dataset_cache=None, **cls._setup_kwargs)

230
qlib/utils/__init__.py
View File

@@ -7,12 +7,9 @@ from __future__ import division
from __future__ import print_function
import os
import pickle
import re
import sys
import copy
import json
from qlib.typehint import InstConf
import yaml
import redis
import bisect
@@ -22,15 +19,11 @@ import inspect
import hashlib
import datetime
import requests
import importlib
import contextlib
import collections
import numpy as np
import pandas as pd
from pathlib import Path
from typing import List, Dict, Union, Tuple, Any, Optional, Callable
from types import ModuleType
from urllib.parse import urlparse
from typing import List, Union, Optional, Callable
from packaging import version
from .file import get_or_create_path, save_multiple_parts_file, unpack_archive_with_buffer, get_tmp_file_with_buffer
from ..config import C
@@ -288,182 +281,6 @@ def parse_field(field):
return field
def get_module_by_module_path(module_path: Union[str, ModuleType]):
"""Load module path
:param module_path:
:return:
:raises: ModuleNotFoundError
"""
if module_path is None:
raise ModuleNotFoundError("None is passed in as parameters as module_path")
if isinstance(module_path, ModuleType):
module = module_path
else:
if module_path.endswith(".py"):
module_name = re.sub("^[^a-zA-Z_]+", "", re.sub("[^0-9a-zA-Z_]", "", module_path[:-3].replace("/", "_")))
module_spec = importlib.util.spec_from_file_location(module_name, module_path)
module = importlib.util.module_from_spec(module_spec)
sys.modules[module_name] = module
module_spec.loader.exec_module(module)
else:
module = importlib.import_module(module_path)
return module
def split_module_path(module_path: str) -> Tuple[str, str]:
"""
Parameters
----------
module_path : str
e.g. "a.b.c.ClassName"
Returns
-------
Tuple[str, str]
e.g. ("a.b.c", "ClassName")
"""
*m_path, cls = module_path.split(".")
m_path = ".".join(m_path)
return m_path, cls
def get_callable_kwargs(config: InstConf, default_module: Union[str, ModuleType] = None) -> (type, dict):
"""
extract class/func and kwargs from config info
Parameters
----------
config : [dict, str]
similar to config
please refer to the doc of init_instance_by_config
default_module : Python module or str
It should be a python module to load the class type
This function will load class from the config['module_path'] first.
If config['module_path'] doesn't exists, it will load the class from default_module.
Returns
-------
(type, dict):
the class/func object and it's arguments.
Raises
------
ModuleNotFoundError
"""
if isinstance(config, dict):
key = "class" if "class" in config else "func"
if isinstance(config[key], str):
# 1) get module and class
# - case 1): "a.b.c.ClassName"
# - case 2): {"class": "ClassName", "module_path": "a.b.c"}
m_path, cls = split_module_path(config[key])
if m_path == "":
m_path = config.get("module_path", default_module)
module = get_module_by_module_path(m_path)
# 2) get callable
_callable = getattr(module, cls) # may raise AttributeError
else:
_callable = config[key] # the class type itself is passed in
kwargs = config.get("kwargs", {})
elif isinstance(config, str):
# a.b.c.ClassName
m_path, cls = split_module_path(config)
module = get_module_by_module_path(default_module if m_path == "" else m_path)
_callable = getattr(module, cls)
kwargs = {}
else:
raise NotImplementedError(f"This type of input is not supported")
return _callable, kwargs
get_cls_kwargs = get_callable_kwargs # NOTE: this is for compatibility for the previous version
def init_instance_by_config(
config: InstConf,
default_module=None,
accept_types: Union[type, Tuple[type]] = (),
try_kwargs: Dict = {},
**kwargs,
) -> Any:
"""
get initialized instance with config
Parameters
----------
config : InstConf
default_module : Python module
Optional. It should be a python module.
NOTE: the "module_path" will be override by `module` arguments
This function will load class from the config['module_path'] first.
If config['module_path'] doesn't exists, it will load the class from default_module.
accept_types: Union[type, Tuple[type]]
Optional. If the config is a instance of specific type, return the config directly.
This will be passed into the second parameter of isinstance.
try_kwargs: Dict
Try to pass in kwargs in `try_kwargs` when initialized the instance
If error occurred, it will fail back to initialization without try_kwargs.
Returns
-------
object:
An initialized object based on the config info
"""
if isinstance(config, accept_types):
return config
if isinstance(config, (str, Path)):
if isinstance(config, str):
# path like 'file:///<path to pickle file>/obj.pkl'
pr = urlparse(config)
if pr.scheme == "file":
pr_path = os.path.join(pr.netloc, pr.path) if bool(pr.path) else pr.netloc
with open(os.path.normpath(pr_path), "rb") as f:
return pickle.load(f)
else:
with config.open("rb") as f:
return pickle.load(f)
klass, cls_kwargs = get_callable_kwargs(config, default_module=default_module)
try:
return klass(**cls_kwargs, **try_kwargs, **kwargs)
except (TypeError,):
# TypeError for handling errors like
# 1: `XXX() got multiple values for keyword argument 'YYY'`
# 2: `XXX() got an unexpected keyword argument 'YYY'
return klass(**cls_kwargs, **kwargs)
@contextlib.contextmanager
def class_casting(obj: object, cls: type):
"""
Python doesn't provide the downcasting mechanism.
We use the trick here to downcast the class
Parameters
----------
obj : object
the object to be cast
cls : type
the target class type
"""
orig_cls = obj.__class__
obj.__class__ = cls
yield
obj.__class__ = orig_cls
def compare_dict_value(src_data: dict, dst_data: dict):
"""Compare dict value
@@ -744,7 +561,6 @@ def exists_qlib_data(qlib_dir):
return False
# check calendar bin
for _calendar in calendars_dir.iterdir():
if ("_future" not in _calendar.name) and (
not list(features_dir.rglob(f"*.{_calendar.name.split('.')[0]}.bin"))
):
@@ -872,9 +688,9 @@ def get_item_from_obj(config: dict, name_path: str) -> object:
cur_cfg = config
for k in name_path.split("."):
if isinstance(cur_cfg, dict):
cur_cfg = cur_cfg[k]
cur_cfg = cur_cfg[k] # may raise KeyError
elif k.isdigit():
cur_cfg = cur_cfg[int(k)]
cur_cfg = cur_cfg[int(k)] # may raise IndexError
else:
raise ValueError(f"Error when getting {k} from cur_cfg")
return cur_cfg
@@ -910,6 +726,21 @@ def fill_placeholder(config: dict, config_extend: dict):
top = 0
tail = 1
item_queue = [config]
def try_replace_placeholder(value):
if value in config_extend.keys():
value = config_extend[value]
else:
m = re.match(r"<(?P<name_path>[^<>]+)>", value)
if m is not None:
try:
value = get_item_from_obj(config, m.groupdict()["name_path"])
except (KeyError, ValueError, IndexError):
get_module_logger("fill_placeholder").info(
f"{value} lookes like a placeholder, but it can't match to any given values"
)
return value
while top < tail:
now_item = item_queue[top]
top += 1
@@ -917,17 +748,13 @@ def fill_placeholder(config: dict, config_extend: dict):
item_keys = range(len(now_item))
elif isinstance(now_item, dict):
item_keys = now_item.keys()
for key in item_keys:
for key in item_keys: # noqa
if isinstance(now_item[key], (list, dict)):
item_queue.append(now_item[key])
tail += 1
elif isinstance(now_item[key], str):
if now_item[key] in config_extend.keys():
now_item[key] = config_extend[now_item[key]]
else:
m = re.match(r"<(?P<name_path>[^<>]+)>", now_item[key])
if m is not None:
now_item[key] = get_item_from_obj(config, m.groupdict()["name_path"])
# If it is a string, try to replace it with placeholder
now_item[key] = try_replace_placeholder(now_item[key])
return config
@@ -1049,6 +876,15 @@ def fname_to_code(fname: str):
return fname
from .mod import (
get_module_by_module_path,
split_module_path,
get_callable_kwargs,
get_cls_kwargs,
init_instance_by_config,
class_casting,
)
__all__ = [
"get_or_create_path",
"save_multiple_parts_file",
@@ -1056,4 +892,10 @@ __all__ = [
"get_tmp_file_with_buffer",
"set_log_with_config",
"init_instance_by_config",
"get_module_by_module_path",
"split_module_path",
"get_callable_kwargs",
"get_cls_kwargs",
"init_instance_by_config",
"class_casting",
]

1
qlib/utils/index_data.py
View File

@@ -351,7 +351,6 @@ class IndexData(metaclass=index_data_ops_creator):
loc_idx_cls = LocIndexer
def __init__(self, data: np.ndarray, *indices: Union[List, pd.Index, Index]):
self.data = data
self.indices = indices

235
qlib/utils/mod.py Normal file
View File

@@ -0,0 +1,235 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""
All module related class, e.g. :
- importing a module, class
- walkiing a module
- operations on class or module...
"""
import contextlib
import importlib
import os
from pathlib import Path
import pickle
import pkgutil
import re
import sys
from types import ModuleType
from typing import Any, Dict, List, Tuple, Union
from urllib.parse import urlparse
from qlib.typehint import InstConf
def get_module_by_module_path(module_path: Union[str, ModuleType]):
"""Load module path
:param module_path:
:return:
:raises: ModuleNotFoundError
"""
if module_path is None:
raise ModuleNotFoundError("None is passed in as parameters as module_path")
if isinstance(module_path, ModuleType):
module = module_path
else:
if module_path.endswith(".py"):
module_name = re.sub("^[^a-zA-Z_]+", "", re.sub("[^0-9a-zA-Z_]", "", module_path[:-3].replace("/", "_")))
module_spec = importlib.util.spec_from_file_location(module_name, module_path)
module = importlib.util.module_from_spec(module_spec)
sys.modules[module_name] = module
module_spec.loader.exec_module(module)
else:
module = importlib.import_module(module_path)
return module
def split_module_path(module_path: str) -> Tuple[str, str]:
"""
Parameters
----------
module_path : str
e.g. "a.b.c.ClassName"
Returns
-------
Tuple[str, str]
e.g. ("a.b.c", "ClassName")
"""
*m_path, cls = module_path.split(".")
m_path = ".".join(m_path)
return m_path, cls
def get_callable_kwargs(config: InstConf, default_module: Union[str, ModuleType] = None) -> (type, dict):
"""
extract class/func and kwargs from config info
Parameters
----------
config : [dict, str]
similar to config
please refer to the doc of init_instance_by_config
default_module : Python module or str
It should be a python module to load the class type
This function will load class from the config['module_path'] first.
If config['module_path'] doesn't exists, it will load the class from default_module.
Returns
-------
(type, dict):
the class/func object and it's arguments.
Raises
------
ModuleNotFoundError
"""
if isinstance(config, dict):
key = "class" if "class" in config else "func"
if isinstance(config[key], str):
# 1) get module and class
# - case 1): "a.b.c.ClassName"
# - case 2): {"class": "ClassName", "module_path": "a.b.c"}
m_path, cls = split_module_path(config[key])
if m_path == "":
m_path = config.get("module_path", default_module)
module = get_module_by_module_path(m_path)
# 2) get callable
_callable = getattr(module, cls) # may raise AttributeError
else:
_callable = config[key] # the class type itself is passed in
kwargs = config.get("kwargs", {})
elif isinstance(config, str):
# a.b.c.ClassName
m_path, cls = split_module_path(config)
module = get_module_by_module_path(default_module if m_path == "" else m_path)
_callable = getattr(module, cls)
kwargs = {}
else:
raise NotImplementedError(f"This type of input is not supported")
return _callable, kwargs
get_cls_kwargs = get_callable_kwargs # NOTE: this is for compatibility for the previous version
def init_instance_by_config(
config: InstConf,
default_module=None,
accept_types: Union[type, Tuple[type]] = (),
try_kwargs: Dict = {},
**kwargs,
) -> Any:
"""
get initialized instance with config
Parameters
----------
config : InstConf
default_module : Python module
Optional. It should be a python module.
NOTE: the "module_path" will be override by `module` arguments
This function will load class from the config['module_path'] first.
If config['module_path'] doesn't exists, it will load the class from default_module.
accept_types: Union[type, Tuple[type]]
Optional. If the config is a instance of specific type, return the config directly.
This will be passed into the second parameter of isinstance.
try_kwargs: Dict
Try to pass in kwargs in `try_kwargs` when initialized the instance
If error occurred, it will fail back to initialization without try_kwargs.
Returns
-------
object:
An initialized object based on the config info
"""
if isinstance(config, accept_types):
return config
if isinstance(config, (str, Path)):
if isinstance(config, str):
# path like 'file:///<path to pickle file>/obj.pkl'
pr = urlparse(config)
if pr.scheme == "file":
pr_path = os.path.join(pr.netloc, pr.path) if bool(pr.path) else pr.netloc
with open(os.path.normpath(pr_path), "rb") as f:
return pickle.load(f)
else:
with config.open("rb") as f:
return pickle.load(f)
klass, cls_kwargs = get_callable_kwargs(config, default_module=default_module)
try:
return klass(**cls_kwargs, **try_kwargs, **kwargs)
except (TypeError,):
# TypeError for handling errors like
# 1: `XXX() got multiple values for keyword argument 'YYY'`
# 2: `XXX() got an unexpected keyword argument 'YYY'
return klass(**cls_kwargs, **kwargs)
@contextlib.contextmanager
def class_casting(obj: object, cls: type):
"""
Python doesn't provide the downcasting mechanism.
We use the trick here to downcast the class
Parameters
----------
obj : object
the object to be cast
cls : type
the target class type
"""
orig_cls = obj.__class__
obj.__class__ = cls
yield
obj.__class__ = orig_cls
def find_all_classes(module_path: Union[str, ModuleType], cls: type) -> List[type]:
"""
Find all the classes recursively that inherit from `cls` in a given module.
- `cls` itself is also included
>>> from qlib.data.dataset.handler import DataHandler
>>> find_all_classes("qlib.contrib.data.handler", DataHandler)
[<class 'qlib.contrib.data.handler.Alpha158'>, <class 'qlib.contrib.data.handler.Alpha158vwap'>, <class 'qlib.contrib.data.handler.Alpha360'>, <class 'qlib.contrib.data.handler.Alpha360vwap'>, <class 'qlib.data.dataset.handler.DataHandlerLP'>]
TODO:
- skip import error
"""
if isinstance(module_path, ModuleType):
mod = module_path
else:
mod = importlib.import_module(module_path)
cls_list = []
def _append_cls(obj):
# Leverage the closure trick to reuse code
if isinstance(obj, type) and issubclass(obj, cls) and cls not in cls_list:
cls_list.append(obj)
for attr in dir(mod):
_append_cls(getattr(mod, attr))
if hasattr(mod, "__path__"):
# if the model is a package
for _, modname, _ in pkgutil.iter_modules(mod.__path__):
sub_mod = importlib.import_module(f"{mod.__package__}.{modname}")
for m_cls in find_all_classes(sub_mod, cls):
_append_cls(m_cls)
return cls_list

1
qlib/workflow/record_temp.py
View File

@@ -136,7 +136,6 @@ class RecordTemp:
whether the records are stored properly.
"""
if include_self:
# Some mlflow backend will not list the directly recursively.
# So we force to the directly
artifacts = {}

1
qlib/workflow/task/gen.py
View File

@@ -339,7 +339,6 @@ class MultiHorizonGenBase(TaskGen):
def generate(self, task: dict):
res = []
for hr in self.horizon:
# Add horizon
t = copy.deepcopy(task)
self.set_horizon(t, hr)

34
qlib/workflow/task/utils.py
View File

@@ -1,23 +1,25 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""
Some tools for task management.
"""
import bisect
from copy import deepcopy
import pandas as pd
from qlib.data import D
from qlib.utils import hash_args
from qlib.utils.mod import init_instance_by_config
from qlib.workflow import R
from qlib.config import C
from qlib.log import get_module_logger
from pymongo import MongoClient
from pymongo.database import Database
from typing import Union
from pathlib import Path
def get_mongodb() -> Database:
"""
Get database in MongoDB, which means you need to declare the address and the name of a database at first.
@@ -276,3 +278,31 @@ class TimeAdjuster:
return self.get(start_idx), self.get(end_idx)
else:
raise NotImplementedError(f"This type of input is not supported")
def replace_task_handler_with_cache(task: dict, cache_dir: Union[str, Path] = ".") -> dict:
"""
Replace the handler in task with a cache handler.
It will automatically cache the file and save it in cache_dir.
>>> import qlib
>>> qlib.auto_init()
>>> import datetime
>>> # it is simplified task
>>> task = {"dataset": {"kwargs":{'handler': {'class': 'Alpha158', 'module_path': 'qlib.contrib.data.handler', 'kwargs': {'start_time': datetime.date(2008, 1, 1), 'end_time': datetime.date(2020, 8, 1), 'fit_start_time': datetime.date(2008, 1, 1), 'fit_end_time': datetime.date(2014, 12, 31), 'instruments': 'CSI300'}}}}}
>>> new_task = replace_task_handler_with_cache(task)
>>> print(new_task)
{'dataset': {'kwargs': {'handler': 'file...Alpha158.3584f5f8b4.pkl'}}}
"""
cache_dir = Path(cache_dir)
task = deepcopy(task)
handler = task["dataset"]["kwargs"]["handler"]
if isinstance(handler, dict):
hash = hash_args(handler)
h_path = cache_dir / f"{handler['class']}.{hash[:10]}.pkl"
if not h_path.exists():
h = init_instance_by_config(handler)
h.to_pickle(h_path, dump_all=True)
task["dataset"]["kwargs"]["handler"] = f"file://{h_path}"
return task