Fix pylint (#888)

* add_pylint_to_workflow * fix-pylint * fix_pylinterror * fix-issue
2026-07-11 23:06:58 +08:00 · 2022-01-26 19:27:24 +08:00
parent 635632e4ed
commit 144e1e2459
103 changed files with 318 additions and 387 deletions
--- a/qlib/data/base.py
+++ b/qlib/data/base.py
@@ -6,7 +6,6 @@ from __future__ import division
 from __future__ import print_function

 import abc
-import pandas as pd

 from ..log import get_module_logger

@@ -21,107 +20,107 @@ class Expression(abc.ABC):
        return str(self)

    def __gt__(self, other):
-        from .ops import Gt
+        from .ops import Gt  # pylint: disable=C0415

        return Gt(self, other)

    def __ge__(self, other):
-        from .ops import Ge
+        from .ops import Ge  # pylint: disable=C0415

        return Ge(self, other)

    def __lt__(self, other):
-        from .ops import Lt
+        from .ops import Lt  # pylint: disable=C0415

        return Lt(self, other)

    def __le__(self, other):
-        from .ops import Le
+        from .ops import Le  # pylint: disable=C0415

        return Le(self, other)

    def __eq__(self, other):
-        from .ops import Eq
+        from .ops import Eq  # pylint: disable=C0415

        return Eq(self, other)

    def __ne__(self, other):
-        from .ops import Ne
+        from .ops import Ne  # pylint: disable=C0415

        return Ne(self, other)

    def __add__(self, other):
-        from .ops import Add
+        from .ops import Add  # pylint: disable=C0415

        return Add(self, other)

    def __radd__(self, other):
-        from .ops import Add
+        from .ops import Add  # pylint: disable=C0415

        return Add(other, self)

    def __sub__(self, other):
-        from .ops import Sub
+        from .ops import Sub  # pylint: disable=C0415

        return Sub(self, other)

    def __rsub__(self, other):
-        from .ops import Sub
+        from .ops import Sub  # pylint: disable=C0415

        return Sub(other, self)

    def __mul__(self, other):
-        from .ops import Mul
+        from .ops import Mul  # pylint: disable=C0415

        return Mul(self, other)

    def __rmul__(self, other):
-        from .ops import Mul
+        from .ops import Mul  # pylint: disable=C0415

        return Mul(self, other)

    def __div__(self, other):
-        from .ops import Div
+        from .ops import Div  # pylint: disable=C0415

        return Div(self, other)

    def __rdiv__(self, other):
-        from .ops import Div
+        from .ops import Div  # pylint: disable=C0415

        return Div(other, self)

    def __truediv__(self, other):
-        from .ops import Div
+        from .ops import Div  # pylint: disable=C0415

        return Div(self, other)

    def __rtruediv__(self, other):
-        from .ops import Div
+        from .ops import Div  # pylint: disable=C0415

        return Div(other, self)

    def __pow__(self, other):
-        from .ops import Power
+        from .ops import Power  # pylint: disable=C0415

        return Power(self, other)

    def __and__(self, other):
-        from .ops import And
+        from .ops import And  # pylint: disable=C0415

        return And(self, other)

    def __rand__(self, other):
-        from .ops import And
+        from .ops import And  # pylint: disable=C0415

        return And(other, self)

    def __or__(self, other):
-        from .ops import Or
+        from .ops import Or  # pylint: disable=C0415

        return Or(self, other)

    def __ror__(self, other):
-        from .ops import Or
+        from .ops import Or  # pylint: disable=C0415

        return Or(other, self)

@@ -144,7 +143,7 @@ class Expression(abc.ABC):
        pd.Series
            feature series: The index of the series is the calendar index
        """
-        from .cache import H
+        from .cache import H  # pylint: disable=C0415

        # cache
        args = str(self), instrument, start_index, end_index, freq
@@ -215,7 +214,7 @@ class Feature(Expression):

    def _load_internal(self, instrument, start_index, end_index, freq):
        # load
-        from .data import FeatureD
+        from .data import FeatureD  # pylint: disable=C0415

        return FeatureD.feature(instrument, str(self), start_index, end_index, freq)

@@ -232,5 +231,3 @@ class ExpressionOps(Expression):
    This kind of feature will use operator for feature
    construction on the fly.
    """
-
-    pass
--- a/qlib/data/cache.py
+++ b/qlib/data/cache.py
@@ -33,8 +33,7 @@ from ..utils import (

 from ..log import get_module_logger
 from .base import Feature
-
-from .ops import Operators
+from .ops import Operators  # pylint: disable=W0611


 class QlibCacheException(RuntimeError):
@@ -229,8 +228,8 @@ class CacheUtils:
                try:
                    d["meta"]["last_visit"] = str(time.time())
                    d["meta"]["visits"] = d["meta"]["visits"] + 1
-                except KeyError:
-                    raise KeyError("Unknown meta keyword")
+                except KeyError as key_e:
+                    raise KeyError("Unknown meta keyword") from key_e
                pickle.dump(d, f, protocol=C.dump_protocol_version)
        except Exception as e:
            get_module_logger("CacheUtils").warning(f"visit {cache_path} cache error: {e}")
@@ -239,7 +238,7 @@ class CacheUtils:
    def acquire(lock, lock_name):
        try:
            lock.acquire()
-        except redis_lock.AlreadyAcquired:
+        except redis_lock.AlreadyAcquired as lock_acquired:
            raise QlibCacheException(
                f"""It sees the key(lock:{repr(lock_name)[1:-1]}-wlock) of the redis lock has existed in your redis db now.
                    You can use the following command to clear your redis keys and rerun your commands:
@@ -249,7 +248,7 @@ class CacheUtils:
                    > quit
                    If the issue is not resolved, use "keys *" to find if multiple keys exist. If so, try using "flushall" to clear all the keys.
                """
-            )
+            ) from lock_acquired

    @staticmethod
    @contextlib.contextmanager
@@ -507,7 +506,7 @@ class DiskExpressionCache(ExpressionCache):
        _instrument_dir = self.get_cache_dir(freq).joinpath(instrument.lower())
        cache_path = _instrument_dir.joinpath(_cache_uri)
        # get calendar
-        from .data import Cal
+        from .data import Cal  # pylint: disable=C0415

        _calendar = Cal.calendar(freq=freq)

@@ -599,7 +598,7 @@ class DiskExpressionCache(ExpressionCache):
            last_update_time = d["info"]["last_update"]

            # get newest calendar
-            from .data import Cal, ExpressionD
+            from .data import Cal, ExpressionD  # pylint: disable=C0415

            whole_calendar = Cal.calendar(start_time=None, end_time=None, freq=freq)
            # calendar since last updated.
@@ -753,7 +752,7 @@ class DiskDatasetCache(DatasetCache):
        if disk_cache == 0:
            # In this case, server only checks the expression cache.
            # The client will load the cache data by itself.
-            from .data import LocalDatasetProvider
+            from .data import LocalDatasetProvider  # pylint: disable=C0415

            LocalDatasetProvider.multi_cache_walker(instruments, fields, start_time, end_time, freq)
            return ""
@@ -895,7 +894,7 @@ class DiskDatasetCache(DatasetCache):
        :return type pd.DataFrame; The fields of the returned DataFrame are consistent with the parameters of the function.
        """
        # get calendar
-        from .data import Cal
+        from .data import Cal  # pylint: disable=C0415

        cache_path = Path(cache_path)
        _calendar = Cal.calendar(freq=freq)
@@ -970,14 +969,14 @@ class DiskDatasetCache(DatasetCache):
            index_data = im.get_index()

            self.logger.debug("Updating dataset: {}".format(d))
-            from .data import Inst
+            from .data import Inst  # pylint: disable=C0415

            if Inst.get_inst_type(instruments) == Inst.DICT:
                self.logger.info(f"The file {cache_uri} has dict cache. Skip updating")
                return 1

            # get newest calendar
-            from .data import Cal
+            from .data import Cal  # pylint: disable=C0415

            whole_calendar = Cal.calendar(start_time=None, end_time=None, freq=freq)
            # The calendar since last updated
@@ -994,7 +993,7 @@ class DiskDatasetCache(DatasetCache):
                current_index = len(whole_calendar) - len(new_calendar) + 1

                # To avoid recursive import
-                from .data import ExpressionD
+                from .data import ExpressionD  # pylint: disable=C0415

                # The existing data length
                lft_etd = rght_etd = 0
--- a/qlib/data/data.py
+++ b/qlib/data/data.py
@@ -5,17 +5,13 @@
 from __future__ import division
 from __future__ import print_function

-import os
 import re
 import abc
-import time
 import copy
 import queue
 import bisect
 import numpy as np
 import pandas as pd
-from multiprocessing import Pool
-from typing import Iterable, Union
 from typing import List, Union

 # For supporting multiprocessing in outer code, joblib is used
@@ -23,13 +19,10 @@ from joblib import delayed

 from .cache import H
 from ..config import C
-from .base import Feature
-from .ops import Operators
 from .inst_processor import InstProcessor

 from ..log import get_module_logger
-from ..utils.time import Freq
-from .cache import DiskDatasetCache, DiskExpressionCache
+from .cache import DiskDatasetCache
 from ..utils import (
    Wrapper,
    init_instance_by_config,
@@ -43,6 +36,7 @@ from ..utils import (
    time_to_slc_point,
 )
 from ..utils.paral import ParallelExt
+from .ops import Operators  # pylint: disable=W0611


 class ProviderBackendMixin:
@@ -144,10 +138,10 @@ class CalendarProvider(abc.ABC):
        if start_time not in calendar_index:
            try:
                start_time = calendar[bisect.bisect_left(calendar, start_time)]
-            except IndexError:
+            except IndexError as index_e:
                raise IndexError(
                    "`start_time` uses a future date, if you want to get future trading days, you can use: `future=True`"
-                )
+                ) from index_e
        start_index = calendar_index[start_time]
        if end_time not in calendar_index:
            end_time = calendar[bisect.bisect_right(calendar, end_time) - 1]
@@ -246,7 +240,7 @@ class InstrumentProvider(abc.ABC):
        """
        if isinstance(market, list):
            return market
-        from .filter import SeriesDFilter
+        from .filter import SeriesDFilter  # pylint: disable=C0415

        if filter_pipe is None:
            filter_pipe = []
@@ -672,7 +666,7 @@ class LocalInstrumentProvider(InstrumentProvider, ProviderBackendMixin):
        # filter
        filter_pipe = instruments["filter_pipe"]
        for filter_config in filter_pipe:
-            from . import filter as F
+            from . import filter as F  # pylint: disable=C0415

            filter_t = getattr(F, filter_config["filter_type"]).from_config(filter_config)
            _instruments_filtered = filter_t(_instruments_filtered, start_time, end_time, freq)
@@ -1003,8 +997,8 @@ class ClientDatasetProvider(DatasetProvider):
                if return_uri:
                    return df, feature_uri
                return df
-            except AttributeError:
-                raise IOError("Unable to fetch instruments from remote server!")
+            except AttributeError as attribute_e:
+                raise IOError("Unable to fetch instruments from remote server!") from attribute_e


 class BaseProvider:
@@ -1110,7 +1104,7 @@ class ClientProvider(BaseProvider):

            return isinstance(instance, cls)

-        from .client import Client
+        from .client import Client  # pylint: disable=C0415

        self.client = Client(C.flask_server, C.flask_port)
        self.logger = get_module_logger(self.__class__.__name__)
--- a/qlib/data/dataset/init.py
+++ b/qlib/data/dataset/init.py
@@ -52,7 +52,6 @@ class Dataset(Serializable):

        - User prepare data for model based on previous status.
        """
-        pass

    def prepare(self, **kwargs) -> object:
        """
@@ -68,7 +67,6 @@ class Dataset(Serializable):
        object:
            return the object
        """
-        pass


 class DatasetH(Dataset):
@@ -348,7 +346,7 @@ class TSDataSampler:
            flt_data = flt_data.reindex(self.data_index).fillna(False).astype(np.bool)
            self.flt_data = flt_data.values
            self.idx_map = self.flt_idx_map(self.flt_data, self.idx_map)
-            self.data_index = self.data_index[np.where(self.flt_data == True)[0]]
+            self.data_index = self.data_index[np.where(self.flt_data is True)[0]]
        self.idx_map = self.idx_map2arr(self.idx_map)

        self.start_idx, self.end_idx = self.data_index.slice_locs(
--- a/qlib/data/dataset/handler.py
+++ b/qlib/data/dataset/handler.py
@@ -2,24 +2,16 @@
 # Licensed under the MIT License.

 # coding=utf-8
-import abc
-import bisect
-import logging
 import warnings
-from inspect import getfullargspec
 from typing import Callable, Union, Tuple, List, Iterator, Optional

 import pandas as pd
-import numpy as np

 from ...log import get_module_logger, TimeInspector
-from ...data import D
-from ...config import C
-from ...utils import parse_config, transform_end_date, init_instance_by_config
+from ...utils import init_instance_by_config
 from ...utils.serial import Serializable
 from .utils import fetch_df_by_index, fetch_df_by_col
 from ...utils import lazy_sort_index
-from pathlib import Path
 from .loader import DataLoader

 from . import processor as processor_module
@@ -228,7 +220,7 @@ class DataHandler(Serializable):
        proc_func: Callable = None,
    ):
        # This method is extracted for sharing in subclasses
-        from .storage import BaseHandlerStorage
+        from .storage import BaseHandlerStorage  # pylint: disable=C0415

        # Following conflictions may occurs
        # - Does [20200101", "20210101"] mean selecting this slice or these two days?
@@ -627,7 +619,6 @@ class DataHandlerLP(DataHandler):
        -------
        pd.DataFrame:
        """
-        from .storage import BaseHandlerStorage

        return self._fetch_data(
            data_storage=self._get_df_by_key(data_key),
--- a/qlib/data/dataset/loader.py
+++ b/qlib/data/dataset/loader.py
@@ -51,7 +51,6 @@ class DataLoader(abc.ABC):
        pd.DataFrame:
            data load from the under layer source
        """
-        pass


 class DLWParser(DataLoader):
@@ -129,7 +128,6 @@ class DLWParser(DataLoader):
        pd.DataFrame:
            the queried dataframe.
        """
-        pass

    def load(self, instruments=None, start_time=None, end_time=None) -> pd.DataFrame:
        if self.is_group:
@@ -308,7 +306,7 @@ class DataLoaderDH(DataLoader):
            is_group will be used to describe whether the key of handler_config is group

        """
-        from qlib.data.dataset.handler import DataHandler
+        from qlib.data.dataset.handler import DataHandler  # pylint: disable=C0415

        if is_group:
            self.handlers = {
--- a/qlib/data/dataset/processor.py
+++ b/qlib/data/dataset/processor.py
@@ -42,7 +42,6 @@ class Processor(Serializable):
            processor, i.e. `df`.

        """
-        pass

    @abc.abstractmethod
    def __call__(self, df: pd.DataFrame):
@@ -57,7 +56,6 @@ class Processor(Serializable):
        df : pd.DataFrame
            The raw_df of handler or result from previous processor.
        """
-        pass

    def is_for_infer(self) -> bool:
        """
@@ -201,7 +199,7 @@ class MinMaxNorm(Processor):
        self.fit_end_time = fit_end_time
        self.fields_group = fields_group

-    def fit(self, df):
+    def fit(self, df: pd.DataFrame = None):
        df = fetch_df_by_index(df, slice(self.fit_start_time, self.fit_end_time), level="datetime")
        cols = get_group_columns(df, self.fields_group)
        self.min_val = np.nanmin(df[cols].values, axis=0)
@@ -232,7 +230,7 @@ class ZScoreNorm(Processor):
        self.fit_end_time = fit_end_time
        self.fields_group = fields_group

-    def fit(self, df):
+    def fit(self, df: pd.DataFrame = None):
        df = fetch_df_by_index(df, slice(self.fit_start_time, self.fit_end_time), level="datetime")
        cols = get_group_columns(df, self.fields_group)
        self.mean_train = np.nanmean(df[cols].values, axis=0)
@@ -272,7 +270,7 @@ class RobustZScoreNorm(Processor):
        self.fields_group = fields_group
        self.clip_outlier = clip_outlier

-    def fit(self, df):
+    def fit(self, df: pd.DataFrame = None):
        df = fetch_df_by_index(df, slice(self.fit_start_time, self.fit_end_time), level="datetime")
        self.cols = get_group_columns(df, self.fields_group)
        X = df[self.cols].values
@@ -351,6 +349,6 @@ class HashStockFormat(Processor):
    """Process the storage of from df into hasing stock format"""

    def __call__(self, df: pd.DataFrame):
-        from .storage import HasingStockStorage
+        from .storage import HasingStockStorage  # pylint: disable=C0415

        return HasingStockStorage.from_df(df)
--- a/qlib/data/dataset/storage.py
+++ b/qlib/data/dataset/storage.py
@@ -2,7 +2,7 @@ import pandas as pd
 import numpy as np

 from .handler import DataHandler
-from typing import Tuple, Union, List, Callable
+from typing import Union, List, Callable

 from .utils import get_level_index, fetch_df_by_index, fetch_df_by_col

@@ -109,7 +109,7 @@ class HasingStockStorage(BaseHandlerStorage):
                stock_selector = selector[self.stock_level]
            elif isinstance(selector, (list, str)) and self.stock_level == 0:
                stock_selector = selector
-        elif level == "instrument" or level == self.stock_level:
+        elif level in ("instrument", self.stock_level):
            if isinstance(selector, tuple):
                stock_selector = selector[0]
            elif isinstance(selector, (list, str)):
--- a/qlib/data/dataset/utils.py
+++ b/qlib/data/dataset/utils.py
@@ -63,7 +63,7 @@ def fetch_df_by_index(
    Data of the given index.
    """
    # level = None -> use selector directly
-    if level == None:
+    if level is None:
        return df.loc(axis=0)[selector]
    # Try to get the right index
    idx_slc = (selector, slice(None, None))
@@ -75,7 +75,7 @@ def fetch_df_by_index(
                return df.loc[
                    pd.IndexSlice[idx_slc],
                ]
-        else:
+        else:  # pylint: disable=W0120
            return df
    else:
        return df.loc[
@@ -84,7 +84,7 @@ def fetch_df_by_index(


 def fetch_df_by_col(df: pd.DataFrame, col_set: Union[str, List[str]]) -> pd.DataFrame:
-    from .handler import DataHandler
+    from .handler import DataHandler  # pylint: disable=C0415

    if not isinstance(df.columns, pd.MultiIndex) or col_set == DataHandler.CS_RAW:
        return df
@@ -136,7 +136,7 @@ def init_task_handler(task: dict) -> Union[DataHandler, None]:
        returns
    """
    # avoid recursive import
-    from .handler import DataHandler
+    from .handler import DataHandler  # pylint: disable=C0415

    h_conf = task["dataset"]["kwargs"].get("handler")
    if h_conf is not None:
--- a/qlib/data/dataset/weight.py
+++ b/qlib/data/dataset/weight.py
@@ -1,13 +1,6 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.

-import pandas as pd
-import numpy as np
-from typing import Union, List, Tuple
-from ...data.dataset import TSDataSampler
-from ...data.dataset.utils import get_level_index
-from ...utils import lazy_sort_index
-

 class Reweighter:
    def __init__(self, *args, **kwargs):
--- a/qlib/data/filter.py
+++ b/qlib/data/filter.py
@@ -62,7 +62,7 @@ class SeriesDFilter(BaseDFilter):
    Override _getFilterSeries to use the rule to filter the series and get a dict of {inst => series}, or override filter_main for more advanced series filter rule
    """

-    def __init__(self, fstart_time=None, fend_time=None):
+    def __init__(self, fstart_time=None, fend_time=None, keep=False):
        """Init function for filter base class.
            Filter a set of instruments based on a certain rule within a certain period assigned by fstart_time and fend_time.

@@ -72,10 +72,13 @@ class SeriesDFilter(BaseDFilter):
            the time for the filter rule to start filter the instruments.
        fend_time: str
            the time for the filter rule to stop filter the instruments.
+        keep: bool
+            whether to keep the instruments of which features don't exist in the filter time span.
        """
        super(SeriesDFilter, self).__init__()
        self.filter_start_time = pd.Timestamp(fstart_time) if fstart_time else None
        self.filter_end_time = pd.Timestamp(fend_time) if fend_time else None
+        self.keep = keep

    def _getTimeBound(self, instruments):
        """Get time bound for all instruments.
@@ -330,12 +333,9 @@ class ExpressionDFilter(SeriesDFilter):
            filter the feature ending by this time.
        rule_expression: str
            an input expression for the rule.
-        keep: bool
-            whether to keep the instruments of which features don't exist in the filter time span.
        """
-        super(ExpressionDFilter, self).__init__(fstart_time, fend_time)
+        super(ExpressionDFilter, self).__init__(fstart_time, fend_time, keep=keep)
        self.rule_expression = rule_expression
-        self.keep = keep

    def _getFilterSeries(self, instruments, fstart, fend):
        # do not use dataset cache
--- a/qlib/data/inst_processor.py
+++ b/qlib/data/inst_processor.py
@@ -17,7 +17,6 @@ class InstProcessor:
        df : pd.DataFrame
            The raw_df of handler or result from previous processor.
        """
-        pass

    def __str__(self):
        return f"{self.__class__.__name__}:{json.dumps(self.__dict__, sort_keys=True, default=str)}"
--- a/qlib/data/ops.py
+++ b/qlib/data/ops.py
@@ -5,8 +5,6 @@
 from __future__ import division
 from __future__ import print_function

-import sys
-import abc
 import numpy as np
 import pandas as pd

@@ -15,7 +13,6 @@ from scipy.stats import percentileofscore

 from .base import Expression, ExpressionOps, Feature

-from ..config import C
 from ..log import get_module_logger
 from ..utils import get_callable_kwargs

@@ -331,7 +328,7 @@ class NpPairOperator(PairOperator):
            res = getattr(np, self.func)(series_left, series_right)
        except ValueError as e:
            get_module_logger("ops").debug(warning_info)
-            raise ValueError(f"{str(e)}. \n\t{warning_info}")
+            raise ValueError(f"{str(e)}. \n\t{warning_info}") from e
        else:
            if check_length and len(series_left) != len(series_right):
                get_module_logger("ops").debug(warning_info)
@@ -1430,21 +1427,20 @@ class PairRolling(ExpressionOps):
        return max(left_br, right_br)

    def get_extended_window_size(self):
+        if isinstance(self.feature_left, Expression):
+            ll, lr = self.feature_left.get_extended_window_size()
+        else:
+            ll, lr = 0, 0
+        if isinstance(self.feature_right, Expression):
+            rl, rr = self.feature_right.get_extended_window_size()
+        else:
+            rl, rr = 0, 0
        if self.N == 0:
            get_module_logger(self.__class__.__name__).warning(
                "The PairRolling(ATTR, 0) will not be accurately calculated"
            )
            return -np.inf, max(lr, rr)
        else:
-            if isinstance(self.feature_left, Expression):
-                ll, lr = self.feature_left.get_extended_window_size()
-            else:
-                ll, lr = 0, 0
-
-            if isinstance(self.feature_right, Expression):
-                rl, rr = self.feature_right.get_extended_window_size()
-            else:
-                rl, rr = 0, 0
            return max(ll, rl) + self.N - 1, max(lr, rr)