From 1ef8e61abdf0596fccfc49291eb427c0e6209a4b Mon Sep 17 00:00:00 2001 From: Linlang <30293408+SunsetWolf@users.noreply.github.com> Date: Thu, 9 Jun 2022 16:12:33 +0800 Subject: [PATCH] fix_pylint_for_CI (#1119) * fix_pylint_for_CI * reformat_with_black * fix_pylint_C3001 * fix_flake8_error --- .github/workflows/test.yml | 2 +- qlib/contrib/data/dataset.py | 15 ++++++++++++--- qlib/contrib/data/handler.py | 5 ++++- qlib/contrib/eva/alpha.py | 9 +++++++-- qlib/contrib/meta/data_selection/dataset.py | 2 +- qlib/contrib/model/highfreq_gdbt_model.py | 5 ++++- qlib/contrib/model/pytorch_hist.py | 4 +++- qlib/contrib/model/pytorch_tra.py | 4 ++-- qlib/data/dataset/__init__.py | 2 +- qlib/rl/utils/data_queue.py | 2 +- qlib/workflow/task/collect.py | 5 ++++- 11 files changed, 40 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f27359289..003cf8543 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,7 +72,7 @@ jobs: run: | pip install --upgrade pip pip install pylint - pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0201,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' qlib --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500" + pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' qlib --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500" # The following flake8 error codes were ignored: # E501 line too long diff --git a/qlib/contrib/data/dataset.py b/qlib/contrib/data/dataset.py index 60362130b..a8e734eed 100644 --- a/qlib/contrib/data/dataset.py +++ b/qlib/contrib/data/dataset.py @@ -63,11 +63,20 @@ def _get_date_parse_fn(target): get_date_parse_fn(20120101)('2017-01-01') => 20170101 """ if isinstance(target, int): - _fn = lambda x: int(str(x).replace("-", "")[:8]) # 20200201 + + def _fn(x): + return int(str(x).replace("-", "")[:8]) # 20200201 + elif isinstance(target, str) and len(target) == 8: - _fn = lambda x: str(x).replace("-", "")[:8] # '20200201' + + def _fn(x): + return str(x).replace("-", "")[:8] # '20200201' + else: - _fn = lambda x: x # '2021-01-01' + + def _fn(x): + return x # '2021-01-01' + return _fn diff --git a/qlib/contrib/data/handler.py b/qlib/contrib/data/handler.py index 81b7c7392..1862baaa9 100644 --- a/qlib/contrib/data/handler.py +++ b/qlib/contrib/data/handler.py @@ -255,7 +255,10 @@ class Alpha158(DataHandlerLP): exclude = config["rolling"].get("exclude", []) # `exclude` in dataset config unnecessary filed # `include` in dataset config necessary field - use = lambda x: x not in exclude and (include is None or x in include) + + def use(x): + return x not in exclude and (include is None or x in include) + if use("ROC"): fields += ["Ref($close, %d)/$close" % d for d in windows] names += ["ROC%d" % d for d in windows] diff --git a/qlib/contrib/eva/alpha.py b/qlib/contrib/eva/alpha.py index 398bd6f55..95ec9b91e 100644 --- a/qlib/contrib/eva/alpha.py +++ b/qlib/contrib/eva/alpha.py @@ -48,7 +48,9 @@ def calc_long_short_prec( group = df.groupby(level=date_col) - N = lambda x: int(len(x) * quantile) + def N(x): + return int(len(x) * quantile) + # find the top/low quantile of prediction and treat them as long and short target long = group.apply(lambda x: x.nlargest(N(x), columns="pred").label).reset_index(level=0, drop=True) short = group.apply(lambda x: x.nsmallest(N(x), columns="pred").label).reset_index(level=0, drop=True) @@ -98,7 +100,10 @@ def calc_long_short_return( if dropna: df.dropna(inplace=True) group = df.groupby(level=date_col) - N = lambda x: int(len(x) * quantile) + + def N(x): + return int(len(x) * quantile) + r_long = group.apply(lambda x: x.nlargest(N(x), columns="pred").label.mean()) r_short = group.apply(lambda x: x.nsmallest(N(x), columns="pred").label.mean()) r_avg = group.label.mean() diff --git a/qlib/contrib/meta/data_selection/dataset.py b/qlib/contrib/meta/data_selection/dataset.py index 235b4f49a..379588e85 100644 --- a/qlib/contrib/meta/data_selection/dataset.py +++ b/qlib/contrib/meta/data_selection/dataset.py @@ -290,7 +290,7 @@ class MetaDatasetDS(MetaTaskDataset): ic_df = self.internal_data.data_ic_df segs = task["dataset"]["kwargs"]["segments"] - end = max([segs[k][1] for k in ("train", "valid") if k in segs]) + end = max(segs[k][1] for k in ("train", "valid") if k in segs) ic_df_avail = ic_df.loc[:end, pd.IndexSlice[:, :end]] # meta data set focus on the **information** instead of preprocess diff --git a/qlib/contrib/model/highfreq_gdbt_model.py b/qlib/contrib/model/highfreq_gdbt_model.py index 8c0202d6e..324ea1e39 100644 --- a/qlib/contrib/model/highfreq_gdbt_model.py +++ b/qlib/contrib/model/highfreq_gdbt_model.py @@ -92,7 +92,10 @@ class HFLGBModel(ModelFT, LightGBMFInt): # Convert label into alpha df_train["label"][l_name] = df_train["label"][l_name] - df_train["label"][l_name].mean(level=0) df_valid["label"][l_name] = df_valid["label"][l_name] - df_valid["label"][l_name].mean(level=0) - mapping_fn = lambda x: 0 if x < 0 else 1 + + def mapping_fn(x): + return 0 if x < 0 else 1 + df_train["label_c"] = df_train["label"][l_name].apply(mapping_fn) df_valid["label_c"] = df_valid["label"][l_name].apply(mapping_fn) x_train, y_train = df_train["feature"], df_train["label_c"].values diff --git a/qlib/contrib/model/pytorch_hist.py b/qlib/contrib/model/pytorch_hist.py index 020534575..25445ba59 100644 --- a/qlib/contrib/model/pytorch_hist.py +++ b/qlib/contrib/model/pytorch_hist.py @@ -292,7 +292,9 @@ class HIST(Model): pretrained_model.load_state_dict(torch.load(self.model_path)) model_dict = self.HIST_model.state_dict() - pretrained_dict = {k: v for k, v in pretrained_model.state_dict().items() if k in model_dict} + pretrained_dict = { + k: v for k, v in pretrained_model.state_dict().items() if k in model_dict # pylint: disable=E1135 + } model_dict.update(pretrained_dict) self.HIST_model.load_state_dict(model_dict) self.logger.info("Loading pretrained model Done...") diff --git a/qlib/contrib/model/pytorch_tra.py b/qlib/contrib/model/pytorch_tra.py index 5c7484c32..46d362c68 100644 --- a/qlib/contrib/model/pytorch_tra.py +++ b/qlib/contrib/model/pytorch_tra.py @@ -167,8 +167,8 @@ class TRAModel(Model): for param in self.tra.predictors.parameters(): param.requires_grad_(False) - self.logger.info("# model params: %d" % sum([p.numel() for p in self.model.parameters() if p.requires_grad])) - self.logger.info("# tra params: %d" % sum([p.numel() for p in self.tra.parameters() if p.requires_grad])) + self.logger.info("# model params: %d" % sum(p.numel() for p in self.model.parameters() if p.requires_grad)) + self.logger.info("# tra params: %d" % sum(p.numel() for p in self.tra.parameters() if p.requires_grad)) self.optimizer = optim.Adam(list(self.model.parameters()) + list(self.tra.parameters()), lr=self.lr) diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index 7a65b3f3c..c74092de3 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -438,7 +438,7 @@ class TSDataSampler: @property def empty(self): - return self.__len__() == 0 + return len(self) == 0 def _get_indices(self, row: int, col: int) -> np.array: """ diff --git a/qlib/rl/utils/data_queue.py b/qlib/rl/utils/data_queue.py index 1e9dab24d..32041abef 100644 --- a/qlib/rl/utils/data_queue.py +++ b/qlib/rl/utils/data_queue.py @@ -145,7 +145,7 @@ class DataQueue(Generic[T]): def __iter__(self): if not self._activated: raise ValueError( - "Need to call activate() to launch a daemon worker " "to produce data into data queue before using it." + "Need to call activate() to launch a daemon worker to produce data into data queue before using it." ) return self._consumer() diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 1866344c4..16c5505fc 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -169,7 +169,10 @@ class RecorderCollector(Collector): self.experiment = experiment self.artifacts_path = artifacts_path if rec_key_func is None: - rec_key_func = lambda rec: rec.info["id"] + + def rec_key_func(rec): + return rec.info["id"] + if artifacts_key is None: artifacts_key = list(self.artifacts_path.keys()) self.rec_key_func = rec_key_func