add_pre-commit_and_flake8_to_CI

qlib/__init__.py

@@ -12,6 +12,7 @@ import platform
 import subprocess
 from .log import get_module_logger
 
+
 # init qlib
 def init(default_conf="client", **kwargs):
     """

qlib/backtest/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
+
 from __future__ import annotations
 import copy
 from typing import List, Tuple, Union, TYPE_CHECKING
@@ -323,3 +324,6 @@ def format_decisions(
             last_dec_idx = i
     res[1].append((decisions[last_dec_idx], format_decisions(decisions[last_dec_idx + 1 :])))
     return res
+
+
+__all__ = ["Order"]

qlib/backtest/executor.py

@@ -242,7 +242,7 @@ class BaseExecutor:
         if self.track_data:
             yield trade_decision
 
-        atomic = not issubclass(self.__class__, NestedExecutor)  #  issubclass(A, A) is True
+        atomic = not issubclass(self.__class__, NestedExecutor)  # issubclass(A, A) is True
 
         if atomic and trade_decision.get_range_limit(default_value=None) is not None:
             raise ValueError("atomic executor doesn't support specify `range_limit`")

qlib/contrib/data/utils/sepdf.py

@@ -164,14 +164,14 @@ import builtins
 
 
 def _isinstance(instance, cls):
-    if isinstance_orig(instance, SepDataFrame):  # pylint: disable=E0602
+    if isinstance_orig(instance, SepDataFrame):  # pylint: disable=E0602  # noqa: F821
         if isinstance(cls, Iterable):
             for c in cls:
                 if c is pd.DataFrame:
                     return True
         elif cls is pd.DataFrame:
             return True
-    return isinstance_orig(instance, cls)  # pylint: disable=E0602
+    return isinstance_orig(instance, cls)  # pylint: disable=E0602  # noqa: F821
 
 
 builtins.isinstance_orig = builtins.isinstance

qlib/contrib/meta/__init__.py

@@ -2,3 +2,6 @@
 # Licensed under the MIT License.
 
 from .data_selection import MetaTaskDS, MetaDatasetDS, MetaModelDS
+
+
+__all__ = ["MetaTaskDS", "MetaDatasetDS", "MetaModelDS"]

qlib/contrib/meta/data_selection/__init__.py

@@ -3,3 +3,6 @@
 
 from .dataset import MetaDatasetDS, MetaTaskDS
 from .model import MetaModelDS
+
+
+__all__ = ["MetaDatasetDS", "MetaTaskDS", "MetaModelDS"]

qlib/contrib/meta/data_selection/model.py

@@ -10,7 +10,6 @@ from tqdm.auto import tqdm
 import copy
 from typing import Union, List
 
-from ....data.dataset.weight import Reweighter
 from ....model.meta.dataset import MetaTaskDataset
 from ....model.meta.model import MetaTaskModel
 from ....workflow import R
@@ -18,8 +17,8 @@ from .utils import ICLoss
 from .dataset import MetaDatasetDS
 
 from qlib.log import get_module_logger
-from qlib.data.dataset.weight import Reweighter
 from qlib.model.meta.task import MetaTask
+from qlib.data.dataset.weight import Reweighter
 from qlib.contrib.meta.data_selection.net import PredNet
 
 logger = get_module_logger("data selection")
@@ -98,7 +97,6 @@ class MetaModelDS(MetaTaskModel):
 
             if phase == "train":
                 opt.zero_grad()
-                norm_loss = nn.MSELoss()
                 loss.backward()
                 opt.step()
             elif phase == "test":

qlib/contrib/model/double_ensemble.py

@@ -249,7 +249,7 @@ class DEnsembleModel(Model, FeatureInt):
         return pred
 
     def predict_sub(self, submodel, df_data, features):
-        x_data, y_data = df_data["feature"].loc[:, features], df_data["label"]
+        x_data = df_data["feature"].loc[:, features]
         pred_sub = pd.Series(submodel.predict(x_data.values), index=x_data.index)
         return pred_sub
 

qlib/contrib/model/pytorch_sfm.py

@@ -84,7 +84,7 @@ class SFM_Model(nn.Module):
             if len(self.states) == 0:  # hasn't initialized yet
                 self.init_states(x)
             self.get_constants(x)
-            p_tm1 = self.states[0]
+            p_tm1 = self.states[0]  # noqa: F841
             h_tm1 = self.states[1]
             S_re_tm1 = self.states[2]
             S_im_tm1 = self.states[3]

qlib/contrib/model/pytorch_tabnet.py

@@ -477,10 +477,10 @@ class TabNet(nn.Module):
         sparse_loss = []
         out = torch.zeros(x.size(0), self.n_d).to(x.device)
         for step in self.steps:
-            x_te, l = step(x, x_a, priors)
+            x_te, loss = step(x, x_a, priors)
             out += F.relu(x_te[:, : self.n_d])  # split the feature from feat_transformer
             x_a = x_te[:, self.n_d :]
-            sparse_loss.append(l)
+            sparse_loss.append(loss)
         return self.fc(out), sum(sparse_loss)
 
 

qlib/contrib/online/__init__.py

@@ -1,4 +1,5 @@
 # pylint: skip-file
+# flake8: noqa
 
 '''
 TODO:

qlib/contrib/online/manager.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import yaml
 import pathlib

qlib/contrib/online/online_model.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import random
 import pandas as pd

qlib/contrib/online/operator.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import fire
 import pandas as pd

qlib/contrib/online/user.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import logging
 

qlib/contrib/online/utils.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import pathlib
 import pickle

qlib/contrib/report/analysis_model/__init__.py

@@ -2,3 +2,6 @@
 # Licensed under the MIT License.
 
 from .analysis_model_performance import model_performance_graph
+
+
+__all__ = ["model_performance_graph"]

qlib/contrib/report/analysis_position/__init__.py

@@ -6,3 +6,6 @@ from .score_ic import score_ic_graph
 from .report import report_graph
 from .rank_label import rank_label_graph
 from .risk_analysis import risk_analysis_graph
+
+
+__all__ = ["cumulative_return_graph", "score_ic_graph", "report_graph", "rank_label_graph", "risk_analysis_graph"]

qlib/contrib/strategy/__init__.py

@@ -15,3 +15,14 @@ from .rule_strategy import (
 )
 
 from .cost_control import SoftTopkStrategy
+
+
+__all__ = [
+    "TopkDropoutStrategy",
+    "WeightStrategyBase",
+    "EnhancedIndexingStrategy",
+    "TWAPStrategy",
+    "SBBStrategyBase",
+    "SBBStrategyEMA",
+    "SoftTopkStrategy",
+]

qlib/contrib/strategy/optimizer/__init__.py

@@ -4,3 +4,6 @@
 from .base import BaseOptimizer
 from .optimizer import PortfolioOptimizer
 from .enhanced_indexing import EnhancedIndexingOptimizer
+
+
+__all__ = ["BaseOptimizer", "PortfolioOptimizer", "EnhancedIndexingOptimizer"]

qlib/contrib/strategy/signal_strategy.py

@@ -131,10 +131,10 @@ class TopkDropoutStrategy(BaseSignalStrategy):
         if self.only_tradable:
             # If The strategy only consider tradable stock when make decision
             # It needs following actions to filter stocks
-            def get_first_n(l, n, reverse=False):
+            def get_first_n(li, n, reverse=False):
                 cur_n = 0
                 res = []
-                for si in reversed(l) if reverse else l:
+                for si in reversed(li) if reverse else li:
                     if self.trade_exchange.is_stock_tradable(
                         stock_id=si, start_time=trade_start_time, end_time=trade_end_time
                     ):
@@ -144,13 +144,13 @@ class TopkDropoutStrategy(BaseSignalStrategy):
                             break
                 return res[::-1] if reverse else res
 
-            def get_last_n(l, n):
-                return get_first_n(l, n, reverse=True)
+            def get_last_n(li, n):
+                return get_first_n(li, n, reverse=True)
 
-            def filter_stock(l):
+            def filter_stock(li):
                 return [
                     si
-                    for si in l
+                    for si in li
                     if self.trade_exchange.is_stock_tradable(
                         stock_id=si, start_time=trade_start_time, end_time=trade_end_time
                     )
@@ -158,14 +158,14 @@ class TopkDropoutStrategy(BaseSignalStrategy):
 
         else:
             # Otherwise, the stock will make decision with out the stock tradable info
-            def get_first_n(l, n):
-                return list(l)[:n]
+            def get_first_n(li, n):
+                return list(li)[:n]
 
-            def get_last_n(l, n):
-                return list(l)[-n:]
+            def get_last_n(li, n):
+                return list(li)[-n:]
 
-            def filter_stock(l):
-                return l
+            def filter_stock(li):
+                return li
 
         current_temp = copy.deepcopy(self.trade_position)
         # generate order list for this adjust date
@@ -203,7 +203,7 @@ class TopkDropoutStrategy(BaseSignalStrategy):
             candi = filter_stock(last)
             try:
                 sell = pd.Index(np.random.choice(candi, self.n_drop, replace=False) if len(last) else [])
-            except ValueError:  #  No enough candidates
+            except ValueError:  # No enough candidates
                 sell = candi
         else:
             raise NotImplementedError(f"This type of input is not supported")

qlib/contrib/tuner/__init__.py

@@ -1 +1,2 @@
 # pylint: skip-file
+# flake8: noqa

qlib/contrib/tuner/config.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import yaml
 import copy

qlib/contrib/tuner/launcher.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 # coding=utf-8
 

qlib/contrib/tuner/pipeline.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import os
 import json

qlib/contrib/tuner/space.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 from hyperopt import hp
 

qlib/contrib/tuner/tuner.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import os
 import yaml

qlib/contrib/workflow/__init__.py

@@ -2,3 +2,6 @@
 #  Licensed under the MIT License.
 from .record_temp import MultiSegRecord
 from .record_temp import SignalMseRecord
+
+
+__all__ = ["MultiSegRecord", "SignalMseRecord"]

qlib/data/__init__.py

@@ -35,3 +35,32 @@ from .cache import (
     DatasetURICache,
     MemoryCalendarCache,
 )
+
+
+__all__ = [
+    "D",
+    "CalendarProvider",
+    "InstrumentProvider",
+    "FeatureProvider",
+    "ExpressionProvider",
+    "DatasetProvider",
+    "LocalCalendarProvider",
+    "LocalInstrumentProvider",
+    "LocalFeatureProvider",
+    "LocalPITProvider",
+    "LocalExpressionProvider",
+    "LocalDatasetProvider",
+    "ClientCalendarProvider",
+    "ClientInstrumentProvider",
+    "ClientDatasetProvider",
+    "BaseProvider",
+    "LocalProvider",
+    "ClientProvider",
+    "ExpressionCache",
+    "DatasetCache",
+    "DiskExpressionCache",
+    "DiskDatasetCache",
+    "SimpleDatasetCache",
+    "DatasetURICache",
+    "MemoryCalendarCache",
+]

qlib/data/cache.py

@@ -33,7 +33,7 @@ from ..utils import (
 
 from ..log import get_module_logger
 from .base import Feature
-from .ops import Operators  # pylint: disable=W0611
+from .ops import Operators  # pylint: disable=W0611  # noqa: F401
 
 
 class QlibCacheException(RuntimeError):
@@ -528,7 +528,7 @@ class DiskExpressionCache(ExpressionCache):
                     CacheUtils.visit(cache_path)
                 series = read_bin(cache_path, start_index, end_index)
                 return series
-            except Exception as e:
+            except Exception:
                 series = None
                 self.logger.error("reading %s file error : %s" % (cache_path, traceback.format_exc()))
             return series
@@ -1068,7 +1068,7 @@ class SimpleDatasetCache(DatasetCache):
         super(SimpleDatasetCache, self).__init__(provider)
         try:
             self.local_cache_path: Path = Path(C["local_cache_path"]).expanduser().resolve()
-        except (KeyError, TypeError) as e:
+        except (KeyError, TypeError):
             self.logger.error("Assign a local_cache_path in config if you want to use this cache mechanism")
             raise
         self.logger.info(

qlib/data/data.py

@@ -38,7 +38,7 @@ from ..utils import (
     get_period_list,
 )
 from ..utils.paral import ParallelExt
-from .ops import Operators  # pylint: disable=W0611
+from .ops import Operators  # pylint: disable=W0611  # noqa: F401
 
 
 class ProviderBackendMixin:

qlib/data/dataset/__init__.py

@@ -609,3 +609,6 @@ class TSDatasetH(DatasetH):
 
         tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len, dtype=dtype, flt_data=flt_data)
         return tsds
+
+
+__all__ = ["Optional"]

qlib/data/ops.py

@@ -22,7 +22,7 @@ except ImportError:
         "#### Do not import qlib package in the repository directory in case of importing qlib from . without compiling #####"
     )
     raise
-except ValueError as e:
+except ValueError:
     print("!!!!!!!! A error occurs when importing operators implemented based on Cython.!!!!!!!!")
     print("!!!!!!!! They will be disabled. Please Upgrade your numpy to enable them     !!!!!!!!")
     # We catch this error because some platform can't upgrade there package (e.g. Kaggle)

qlib/data/storage/__init__.py

@@ -2,3 +2,6 @@
 # Licensed under the MIT License.
 
 from .storage import CalendarStorage, InstrumentStorage, FeatureStorage, CalVT, InstVT, InstKT
+
+
+__all__ = ["CalendarStorage", "InstrumentStorage", "FeatureStorage", "CalVT", "InstVT", "InstKT"]

qlib/model/__init__.py

@@ -4,3 +4,6 @@
 import warnings
 
 from .base import Model
+
+
+__all__ = ["Model", "warnings"]

qlib/model/meta/__init__.py

@@ -3,3 +3,6 @@
 
 from .task import MetaTask
 from .dataset import MetaTaskDataset
+
+
+__all__ = ["MetaTask", "MetaTaskDataset"]

qlib/model/riskmodel/__init__.py

@@ -5,3 +5,11 @@ from .base import RiskModel
 from .poet import POETCovEstimator
 from .shrink import ShrinkCovEstimator
 from .structured import StructuredCovEstimator
+
+
+__all__ = [
+    "RiskModel",
+    "POETCovEstimator",
+    "ShrinkCovEstimator",
+    "StructuredCovEstimator",
+]

qlib/utils/__init__.py

@@ -14,22 +14,19 @@ import json
 import yaml
 import redis
 import bisect
-import shutil
 import struct
 import difflib
 import inspect
 import hashlib
-import warnings
 import datetime
 import requests
-import tempfile
 import importlib
 import contextlib
 import collections
 import numpy as np
 import pandas as pd
 from pathlib import Path
-from typing import List, Dict, Union, Tuple, Any, Text, Optional, Callable
+from typing import List, Dict, Union, Tuple, Any, Optional, Callable
 from types import ModuleType
 from urllib.parse import urlparse
 from packaging import version
@@ -1047,3 +1044,12 @@ def fname_to_code(fname: str):
     if fname.startswith(prefix):
         fname = fname.lstrip(prefix)
     return fname
+
+
+__all__ = [
+    "get_or_create_path",
+    "save_multiple_parts_file",
+    "unpack_archive_with_buffer",
+    "get_tmp_file_with_buffer",
+    "set_log_with_config",
+]

qlib/utils/time.py

@@ -199,10 +199,8 @@ class Freq:
         """
         base_freq = Freq(base_freq)
         # use the nearest freq greater than 0
-        _freq_minutes = []
         min_freq = None
         for _freq in freq_list:
-            freq = Freq(_freq)
             _min_delta = Freq.get_min_delta(base_freq, _freq)
             if _min_delta < 0:
                 continue

qlib/workflow/expm.py

@@ -170,12 +170,9 @@ class ExpManager:
             experiment_name = self._default_exp_name
 
         if create:
-            exp, is_new = self._get_or_create_exp(experiment_id=experiment_id, experiment_name=experiment_name)
+            exp, _ = self._get_or_create_exp(experiment_id=experiment_id, experiment_name=experiment_name)
         else:
-            exp, is_new = (
-                self._get_exp(experiment_id=experiment_id, experiment_name=experiment_name),
-                False,
-            )
+            exp = self._get_exp(experiment_id=experiment_id, experiment_name=experiment_name)
         if self.active_experiment is None and start:
             self.active_experiment = exp
             # start the recorder
@@ -201,7 +198,7 @@ class ExpManager:
             # So we supported it in the interface wrapper
             pr = urlparse(self.uri)
             if pr.scheme == "file":
-                with FileLock(os.path.join(pr.netloc, pr.path, "filelock")) as f:  # pylint: disable=E0110
+                with FileLock(os.path.join(pr.netloc, pr.path, "filelock")):  # pylint: disable=E0110
                     return self.create_exp(experiment_name), True
             # NOTE: for other schemes like http, we double check to avoid create exp conflicts
             try:

qlib/workflow/record_temp.py

@@ -146,7 +146,7 @@ class RecordTemp:
 
             for item in self.list():
                 ps = self.get_path(item).split("/")
-                dirn, fn = "/".join(ps[:-1]), ps[-1]
+                dirn = "/".join(ps[:-1])
                 if self.get_path(item) not in _get_arts(dirn):
                     raise FileNotFoundError
         if parents: