liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-06 12:30:57 +08:00
Code Activity

Checking dataset empty (#647)

Browse Source 
* Checking dataset empty

* add dataset checker
This commit is contained in:
you-n-g
2021-10-14 23:35:12 +08:00
committed by GitHub
parent 9a175d7507
commit 334b92ace7
21 changed files with 47 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
qlib/contrib/model/catboost_model.py
View File

@@ -38,6 +38,8 @@ class CatBoostModel(Model, FeatureInt):
col_set=["feature", "label"],
data_key=DataHandlerLP.DK_L,
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]

2
qlib/contrib/model/double_ensemble.py
View File

@@ -64,6 +64,8 @@ class DEnsembleModel(Model, FeatureInt):
df_train, df_valid = dataset.prepare(
["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
x_train, y_train = df_train["feature"], df_train["label"]
# initialize the sample weights
N, F = x_train.shape

4
qlib/contrib/model/gbdt.py
View File

@@ -25,6 +25,8 @@ class LGBModel(ModelFT, LightGBMFInt):
df_train, df_valid = dataset.prepare(
["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]
@@ -83,6 +85,8 @@ class LGBModel(ModelFT, LightGBMFInt):
"""
# Based on existing model and finetune by train more rounds
dtrain, _ = self._prepare_data(dataset)
if dtrain.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
self.model = lgb.train(
self.params,
dtrain,

2
qlib/contrib/model/highfreq_gdbt_model.py
View File

@@ -82,6 +82,8 @@ class HFLGBModel(ModelFT, LightGBMFInt):
df_train, df_valid = dataset.prepare(
["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_train["feature"], df_valid["label"]

2
qlib/contrib/model/linear.py
View File

@@ -51,6 +51,8 @@ class LinearModel(Model):
def fit(self, dataset: DatasetH):
df_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
if df_train.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
X, y = df_train["feature"].values, np.squeeze(df_train["label"].values)
if self.estimator in [self.OLS, self.RIDGE, self.LASSO]:

2
qlib/contrib/model/pytorch_alstm.py
View File

@@ -224,6 +224,8 @@ class ALSTM(Model):
col_set=["feature", "label"],
data_key=DataHandlerLP.DK_L,
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]

2
qlib/contrib/model/pytorch_alstm_ts.py
View File

@@ -207,6 +207,8 @@ class ALSTM(Model):
):
dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
if dl_train.empty or dl_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader
dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader

2
qlib/contrib/model/pytorch_gats.py
View File

@@ -237,6 +237,8 @@ class GATs(Model):
col_set=["feature", "label"],
data_key=DataHandlerLP.DK_L,
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]

2
qlib/contrib/model/pytorch_gats_ts.py
View File

@@ -245,6 +245,8 @@ class GATs(Model):
dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
if dl_train.empty or dl_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader
dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader

2
qlib/contrib/model/pytorch_gru.py
View File

@@ -224,6 +224,8 @@ class GRU(Model):
col_set=["feature", "label"],
data_key=DataHandlerLP.DK_L,
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]

2
qlib/contrib/model/pytorch_gru_ts.py
View File

@@ -206,6 +206,8 @@ class GRU(Model):
):
dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
if dl_train.empty or dl_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader
dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader

2
qlib/contrib/model/pytorch_localformer.py
View File

@@ -176,6 +176,8 @@ class LocalformerModel(Model):
col_set=["feature", "label"],
data_key=DataHandlerLP.DK_L,
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]

2
qlib/contrib/model/pytorch_localformer_ts.py
View File

@@ -153,6 +153,8 @@ class LocalformerModel(Model):
dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
if dl_train.empty or dl_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader
dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader

2
qlib/contrib/model/pytorch_lstm.py
View File

@@ -219,6 +219,8 @@ class LSTM(Model):
col_set=["feature", "label"],
data_key=DataHandlerLP.DK_L,
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]

2
qlib/contrib/model/pytorch_lstm_ts.py
View File

@@ -201,6 +201,8 @@ class LSTM(Model):
):
dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
if dl_train.empty or dl_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader
dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader

2
qlib/contrib/model/pytorch_sfm.py
View File

@@ -374,6 +374,8 @@ class SFM(Model):
col_set=["feature", "label"],
data_key=DataHandlerLP.DK_L,
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]

2
qlib/contrib/model/pytorch_tabnet.py
View File

@@ -169,6 +169,8 @@ class TabnetModel(Model):
col_set=["feature", "label"],
data_key=DataHandlerLP.DK_L,
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
df_train.fillna(df_train.mean(), inplace=True)
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]

2
qlib/contrib/model/pytorch_tcts.py
View File

@@ -255,6 +255,8 @@ class TCTS(Model):
col_set=["feature", "label"],
data_key=DataHandlerLP.DK_L,
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]

2
qlib/contrib/model/pytorch_transformer.py
View File

@@ -175,6 +175,8 @@ class TransformerModel(Model):
col_set=["feature", "label"],
data_key=DataHandlerLP.DK_L,
)
if df_train.empty or df_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
x_train, y_train = df_train["feature"], df_train["label"]
x_valid, y_valid = df_valid["feature"], df_valid["label"]

3
qlib/contrib/model/pytorch_transformer_ts.py
View File

@@ -151,6 +151,9 @@ class TransformerModel(Model):
dl_train = dataset.prepare("train", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
dl_valid = dataset.prepare("valid", col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
if dl_train.empty or dl_valid.empty:
raise ValueError("Empty data from dataset, please check your dataset config.")
dl_train.config(fillna_type="ffill+bfill") # process nan brought by dataloader
dl_valid.config(fillna_type="ffill+bfill") # process nan brought by dataloader

4
qlib/data/dataset/__init__.py
View File

@@ -385,6 +385,10 @@ class TSDataSampler:
idx_map[real_idx] = (i, j)
return idx_df, idx_map
@property
def empty(self):
return self.__len__() == 0
def _get_indices(self, row: int, col: int) -> np.array:
"""
get series indices of self.data_arr from the row, col indices of self.idx_df