liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-14 08:16:54 +08:00
Code Activity

performance mprovement (#921)

Browse Source 
* performance mprovement

* memory refine
This commit is contained in:
you-n-g
2022-02-19 18:36:23 +08:00
committed by GitHub
parent d482726f28
commit 528f74af09
2 changed files with 18 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
qlib/contrib/model/pytorch_nn.py
View File

@@ -7,6 +7,7 @@ from __future__ import print_function
from collections import defaultdict from collections import defaultdict
import os import os
import gc
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from typing import Callable, Optional, Text, Union from typing import Callable, Optional, Text, Union
@@ -32,7 +33,6 @@ from ...log import get_module_logger
from ...workflow import R from ...workflow import R
from qlib.contrib.meta.data_selection.utils import ICLoss from qlib.contrib.meta.data_selection.utils import ICLoss
from torch.nn import DataParallel from torch.nn import DataParallel
from torch.utils.data import DataLoader, SequentialSampler
class DNNModelPytorch(Model): class DNNModelPytorch(Model):
@@ -201,7 +201,7 @@ class DNNModelPytorch(Model):
seg, col_set=["feature", "label"], data_key=self.valid_key if seg == "valid" else DataHandlerLP.DK_L seg, col_set=["feature", "label"], data_key=self.valid_key if seg == "valid" else DataHandlerLP.DK_L
) )
all_df["x"][seg] = df["feature"] all_df["x"][seg] = df["feature"]
all_df["y"][seg] = df["label"] all_df["y"][seg] = df["label"].copy() # We have to use copy to remove the reference to release mem
if reweighter is None: if reweighter is None:
all_df["w"][seg] = pd.DataFrame(np.ones_like(all_df["y"][seg].values), index=df.index) all_df["w"][seg] = pd.DataFrame(np.ones_like(all_df["y"][seg].values), index=df.index)
elif isinstance(reweighter, Reweighter): elif isinstance(reweighter, Reweighter):
@@ -216,6 +216,10 @@ class DNNModelPytorch(Model):
all_t[v][seg] = all_t[v][seg].to(self.device) # This will consume a lot of memory !!!! all_t[v][seg] = all_t[v][seg].to(self.device) # This will consume a lot of memory !!!!
evals_result[seg] = [] evals_result[seg] = []
# free memory
del df
del all_df["x"]
gc.collect()
save_path = get_or_create_path(save_path) save_path = get_or_create_path(save_path)
stop_steps = 0 stop_steps = 0
@@ -266,7 +270,7 @@ class DNNModelPytorch(Model):
loss_val = cur_loss_val.item() loss_val = cur_loss_val.item()
metric_val = ( metric_val = (
self.get_metric( self.get_metric(
preds.reshape(-1), all_t["y"]["valid"].reshape(-1), all_df["x"]["valid"].index preds.reshape(-1), all_t["y"]["valid"].reshape(-1), all_df["y"]["valid"].index
) )
.detach() .detach()
.cpu() .cpu()
@@ -281,7 +285,7 @@ class DNNModelPytorch(Model):
self.get_metric( self.get_metric(
self._nn_predict(all_t["x"]["train"], return_cpu=False), self._nn_predict(all_t["x"]["train"], return_cpu=False),
all_t["y"]["train"].reshape(-1), all_t["y"]["train"].reshape(-1),
all_df["x"]["train"].index, all_df["y"]["train"].index,
) )
.detach() .detach()
.cpu() .cpu()
@@ -351,31 +355,17 @@ class DNNModelPytorch(Model):
1) test inference (data may come from CPU and expect the output data is on CPU) 1) test inference (data may come from CPU and expect the output data is on CPU)
2) evaluation on training (data may come from GPU) 2) evaluation on training (data may come from GPU)
""" """
if isinstance(data, torch.Tensor) and data.device.type != "cpu": if not isinstance(data, torch.Tensor):
# GPU data if isinstance(data, pd.DataFrame):
# CUDA data don't support pin_memory and multi-processing workers data = data.values
num_workers = 0 data = torch.Tensor(data)
pin_memory = False data = data.to(self.device)
else:
# CPU data
if not isinstance(data, torch.Tensor):
if isinstance(data, pd.DataFrame):
data = data.values
# else: CPU Tensor
num_workers = 8
pin_memory = True
data_loader = DataLoader(
data,
sampler=SequentialSampler(data),
batch_size=self.batch_size,
drop_last=False,
num_workers=num_workers,
pin_memory=pin_memory,
)
preds = [] preds = []
self.dnn_model.eval() self.dnn_model.eval()
with torch.no_grad(): with torch.no_grad():
for x in data_loader: batch_size = 8096
for i in range(0, len(data), batch_size):
x = data[i : i + batch_size]
preds.append(self.dnn_model(x.to(self.device)).detach().reshape(-1)) preds.append(self.dnn_model(x.to(self.device)).detach().reshape(-1))
if return_cpu: if return_cpu:
preds = np.concatenate([pr.cpu().numpy() for pr in preds]) preds = np.concatenate([pr.cpu().numpy() for pr in preds])

4
qlib/contrib/report/data/base.py
View File

@@ -8,14 +8,14 @@ Assumptions
""" """
import pandas as pd import pandas as pd
from blocks.utils.log import logt from qlib.log import TimeInspector
from qlib.contrib.report.utils import sub_fig_generator from qlib.contrib.report.utils import sub_fig_generator
class FeaAnalyser: class FeaAnalyser:
def __init__(self, dataset: pd.DataFrame): def __init__(self, dataset: pd.DataFrame):
self._dataset = dataset self._dataset = dataset
with logt("calc_stat_values"): with TimeInspector.logt("calc_stat_values"):
self.calc_stat_values() self.calc_stat_values()
def calc_stat_values(self): def calc_stat_values(self):