diff --git a/examples/benchmarks/DNN/workflow_config_dnn.yaml b/examples/benchmarks/DNN/workflow_config_dnn.yaml index bf5bd7c5f..023d1cd49 100644 --- a/examples/benchmarks/DNN/workflow_config_dnn.yaml +++ b/examples/benchmarks/DNN/workflow_config_dnn.yaml @@ -10,15 +10,30 @@ data_handler_config: &data_handler_config instruments: *market infer_processors: [ { - "class" : "CSZFillna", - "kwargs":{"fields_group": "feature"} + "class" : "DropCol", + "kwargs":{"col_list": ["VWAP0"]} }, { - "class" : "Fillna", - "kwargs":{"fields_group": "feature"} + "class" : "CSZFillna", + "kwargs":{"fields_group": "feature"} } ] - learn_processors: ["DropnaLabel", {"class": "CSZScoreNorm", "kwargs": {"fields_group": "label"}}] + learn_processors: [ + { + "class" : "DropCol", + "kwargs":{"col_list": ["VWAP0"]} + }, + { + "class" : "DropnaProcessor", + "kwargs":{"fields_group": "feature"} + }, + "DropnaLabel", + { + "class": "CSZScoreNorm", + "kwargs": {"fields_group": "label"} + } + ] + process_type: "independent" port_analysis_config: &port_analysis_config strategy: @@ -42,7 +57,7 @@ task: module_path: qlib.contrib.model.pytorch_nn kwargs: loss: mse - input_dim: 158 + input_dim: 157 output_dim: 1 lr: 0.002 lr_decay: 0.96 diff --git a/qlib/contrib/data/handler.py b/qlib/contrib/data/handler.py index 8cce92907..3668a0cc0 100644 --- a/qlib/contrib/data/handler.py +++ b/qlib/contrib/data/handler.py @@ -171,6 +171,7 @@ class Alpha158(DataHandlerLP): learn_processors=["DropnaLabel", {"class": "CSZScoreNorm", "kwargs": {"fields_group": "label"}}], fit_start_time=None, fit_end_time=None, + process_type=DataHandlerLP.PTYPE_A ): def check_transform_proc(proc_l): new_l = [] @@ -209,6 +210,7 @@ class Alpha158(DataHandlerLP): data_loader=data_loader, infer_processors=infer_processors, learn_processors=learn_processors, + process_type=process_type ) def get_feature_config(self): diff --git a/qlib/contrib/model/pytorch_nn.py b/qlib/contrib/model/pytorch_nn.py index e1b0736e2..47316ebf6 100644 --- a/qlib/contrib/model/pytorch_nn.py +++ b/qlib/contrib/model/pytorch_nn.py @@ -20,7 +20,7 @@ from ...data.dataset import DatasetH from ...data.dataset.handler import DataHandlerLP from ...utils import unpack_archive_with_buffer, save_multiple_parts_file, create_save_path, drop_nan_by_y_index from ...log import get_module_logger, TimeInspector - +from ...workflow import R class DNNModelPytorch(Model): """DNN Model @@ -151,7 +151,6 @@ class DNNModelPytorch(Model): verbose=True, save_path=None, ): - df_train, df_valid = dataset.prepare( ["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L ) @@ -170,7 +169,6 @@ class DNNModelPytorch(Model): best_loss = np.inf evals_result["train"] = [] evals_result["valid"] = [] - # train self.logger.info("training...") self._fitted = True @@ -184,9 +182,6 @@ class DNNModelPytorch(Model): x_val_auto = torch.from_numpy(x_valid.values).float() y_val_auto = torch.from_numpy(y_valid.values).float() w_val_auto = torch.from_numpy(w_valid.values).float() - #print('valiadationx:', x_val_auto) - #print('valiadationy:', y_val_auto) - #print('valiadationw:', w_val_auto) if self.use_GPU: x_val_auto = x_val_auto.cuda() y_val_auto = y_val_auto.cuda() @@ -200,7 +195,6 @@ class DNNModelPytorch(Model): loss = AverageMeter() self.dnn_model.train() self.train_optimizer.zero_grad() - choice = np.random.choice(train_num, self.batch_size) x_batch_auto = x_train_values[choice] y_batch_auto = y_train_values[choice] @@ -213,16 +207,14 @@ class DNNModelPytorch(Model): # forward preds = self.dnn_model(x_batch_auto) - #print('pred_train:', preds.detach().cpu().numpy()) - #print('label_train:', y_batch_auto.cpu().numpy()) cur_loss = self.get_loss(preds, w_batch_auto, y_batch_auto, self.loss_type) cur_loss.backward() self.train_optimizer.step() loss.update(cur_loss.item()) + R.log_metrics(train_loss=loss.avg, step=step) # validation train_loss += loss.val - # print(loss.val) if step and step % self.eval_steps == 0: stop_steps += 1 train_loss /= self.eval_steps @@ -232,10 +224,10 @@ class DNNModelPytorch(Model): loss_val = AverageMeter() # forward - preds = self.dnn_model(x_val_auto) cur_loss_val = self.get_loss(preds, w_val_auto, y_val_auto, self.loss_type) loss_val.update(cur_loss_val.item()) + R.log_metrics(val_loss=loss_val.val, step=step) if verbose: self.logger.info( "[Epoch {}]: train_loss {:.6f}, valid_loss {:.6f}".format(step, train_loss, loss_val.val) @@ -276,7 +268,6 @@ class DNNModelPytorch(Model): if not self._fitted: raise ValueError("model is not fitted yet!") x_test_pd = dataset.prepare("test", col_set="feature") - print(x_test_pd) x_test = torch.from_numpy(x_test_pd.values).float() if self.use_GPU: x_test = x_test.cuda() @@ -287,7 +278,6 @@ class DNNModelPytorch(Model): preds = self.dnn_model(x_test).detach().cpu().numpy() else: preds = self.dnn_model(x_test).detach().numpy() - print(preds) return pd.Series(np.squeeze(preds), index=x_test_pd.index) def save(self, filename, **kwargs): diff --git a/qlib/data/dataset/processor.py b/qlib/data/dataset/processor.py index 32b42462f..4a2d36e2f 100755 --- a/qlib/data/dataset/processor.py +++ b/qlib/data/dataset/processor.py @@ -90,7 +90,17 @@ class DropnaLabel(DropnaProcessor): return False +class DropCol(Processor): + def __init__(self, col_list=[]): + self.col_list = col_list + def __call__(self, df): + if isinstance(df.columns, pd.MultiIndex): + mask = df.columns.get_level_values(-1).isin(self.col_list) + else: + mask = df.columns.isin(self.col_list) + return df.loc[:, ~mask] + class TanhProcess(Processor): """ Use tanh to process noise data"""