liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-17 17:34:35 +08:00
Code Activity

Remove batchsize and add daily-batch mode

Browse Source
This commit is contained in:
meng-ustc
2020-11-26 20:22:51 +08:00
parent 99efaadd38
commit ab98f44345
3 changed files with 30 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
examples/benchmarks/HATS/worflow_config_hats.yaml
View File

@@ -36,7 +36,6 @@ task:
n_epochs: 200 n_epochs: 200
lr: 1e-3 lr: 1e-3
early_stop: 20 early_stop: 20
batch_size: 800
metric: IC metric: IC
loss: mse loss: mse
base_model: GRU base_model: GRU

3
examples/workflow_by_code_hats.py
View File

@@ -62,12 +62,11 @@ if __name__ == "__main__":
"n_epochs": 200, "n_epochs": 200,
"lr": 1e-3, "lr": 1e-3,
"early_stop": 20, "early_stop": 20,
"batch_size": 800,
"metric": "IC", "metric": "IC",
"loss": "mse", "loss": "mse",
"base_model": "LSTM", "base_model": "LSTM",
"seed": 0, "seed": 0,
"GPU": "0", "GPU": "2",
}, },
}, },
"dataset": { "dataset": {

57
qlib/contrib/model/pytorch_hats.py
View File

@@ -54,7 +54,6 @@ class HATS(Model):
n_epochs=200, n_epochs=200,
lr=0.01, lr=0.01,
metric="IC", metric="IC",
batch_size=800,
early_stop=20, early_stop=20,
loss="mse", loss="mse",
base_model="GRU", base_model="GRU",
@@ -76,7 +75,6 @@ class HATS(Model):
self.n_epochs = n_epochs self.n_epochs = n_epochs
self.lr = lr self.lr = lr
self.metric = metric self.metric = metric
self.batch_size = batch_size
self.early_stop = early_stop self.early_stop = early_stop
self.optimizer = optimizer.lower() self.optimizer = optimizer.lower()
self.loss = loss self.loss = loss
@@ -95,7 +93,6 @@ class HATS(Model):
"\nn_epochs : {}" "\nn_epochs : {}"
"\nlr : {}" "\nlr : {}"
"\nmetric : {}" "\nmetric : {}"
"\nbatch_size : {}"
"\nearly_stop : {}" "\nearly_stop : {}"
"\noptimizer : {}" "\noptimizer : {}"
"\nloss_type : {}" "\nloss_type : {}"
@@ -111,7 +108,6 @@ class HATS(Model):
n_epochs, n_epochs,
lr, lr,
metric, metric,
batch_size,
early_stop, early_stop,
optimizer.lower(), optimizer.lower(),
loss, loss,
@@ -169,6 +165,18 @@ class HATS(Model):
def cal_ic(self, pred, label): def cal_ic(self, pred, label):
return torch.mean(pred * label) return torch.mean(pred * label)
def get_daily_inter(self, df, shuffle=False):
# organize the train data into daily inter as daily batches
daily_count = df.groupby(level=0).size().values
daily_index = np.roll(np.cumsum(daily_count), 1)
daily_index[0] = 0
if shuffle:
# shuffle the daily inter data
daily_shuffle = list(zip(daily_index, daily_count))
np.random.shuffle(daily_shuffle)
daily_index, daily_count = zip(*daily_shuffle)
return daily_index, daily_count
def train_epoch(self, x_train, y_train): def train_epoch(self, x_train, y_train):
x_train_values = x_train.values x_train_values = x_train.values
@@ -176,16 +184,13 @@ class HATS(Model):
self.HATS_model.train() self.HATS_model.train()
indices = np.arange(len(x_train_values)) # organize the train data into daily inter as daily batches
np.random.shuffle(indices) daily_index, daily_count = self.get_daily_inter(x_train, shuffle=True)
for i in range(len(indices))[:: self.batch_size]: for idx, count in zip(daily_index, daily_count):
batch = slice(idx, idx + count)
if len(indices) - i < self.batch_size: feature = torch.from_numpy(x_train_values[batch]).float()
break label = torch.from_numpy(y_train_values[batch]).float()
feature = torch.from_numpy(x_train_values[indices[i : i + self.batch_size]]).float()
label = torch.from_numpy(y_train_values[indices[i : i + self.batch_size]]).float()
if self.use_gpu: if self.use_gpu:
feature = feature.cuda() feature = feature.cuda()
@@ -210,15 +215,13 @@ class HATS(Model):
scores = [] scores = []
losses = [] losses = []
indices = np.arange(len(x_values)) # organize the test data into daily inter as daily batches
daily_index, daily_count = self.get_daily_inter(data_x, shuffle=False)
for i in range(len(indices))[:: self.batch_size]: for idx, count in zip(daily_index, daily_count):
batch = slice(idx, idx + count)
if len(indices) - i < self.batch_size: feature = torch.from_numpy(x_values[batch]).float()
break label = torch.from_numpy(y_values[batch]).float()
feature = torch.from_numpy(x_values[indices[i : i + self.batch_size]]).float()
label = torch.from_numpy(y_values[indices[i : i + self.batch_size]]).float()
if self.use_gpu: if self.use_gpu:
feature = feature.cuda() feature = feature.cuda()
@@ -317,14 +320,12 @@ class HATS(Model):
sample_num = x_values.shape[0] sample_num = x_values.shape[0]
preds = [] preds = []
for begin in range(sample_num)[:: self.batch_size]: # organize the data into daily inter as daily batches
daily_index, daily_count = self.get_daily_inter(x_test, shuffle=False)
if sample_num - begin < self.batch_size: for idx, count in zip(daily_index, daily_count):
end = sample_num batch = slice(idx, idx + count)
else: x_batch = torch.from_numpy(x_values[batch]).float()
end = begin + self.batch_size
x_batch = torch.from_numpy(x_values[begin:end]).float()
if self.use_gpu: if self.use_gpu:
x_batch = x_batch.cuda() x_batch = x_batch.cuda()