add_pre-commit_and_flake8_to_CI

.github/workflows/test.yml

@@ -65,6 +65,31 @@ jobs:
         pip install pylint
         pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0201,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' qlib --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500"
 
+    # The following flake8 error codes were ignored:
+      # E501 line too long
+        # Description: We have used black to limit the length of each line to 120.
+      # F541 f-string is missing placeholders
+        # Description: The same thing is done when using pylint for detection.
+      # E266 too many leading '#' for block comment
+        # Description: To make the code more readable, a lot of "#" is used.
+        # This error code appears centrally in:
+          # qlib/backtest/executor.py
+          # qlib/data/ops.py
+          # qlib/utils/__init__.py
+      # E402 module level import not at top of file
+        # Description: There are times when module level import is not available at the top of the file.
+      # W503 line break before binary operator
+        # Description: Since black formats the length of each line of code, it has to perform a line break when a line of arithmetic is too long.
+      # E731 do not assign a lambda expression, use a def
+        # Description: Restricts the use of lambda expressions, but at some point lambda expressions are required.
+      # E203 whitespace before ':'
+        # Description: If there is whitespace before ":", it cannot pass the black check.
+    - name: Check Qlib with flake8
+      run: |
+        pip install --upgrade pip
+        pip install flake8
+        flake8 --ignore=E501,F541,E266,E402,W503,E731,E203 qlib
+
     - name: Test data downloads
       run: |
         python scripts/get_data.py qlib_data --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn

.github/workflows/test_macos.yml

@@ -34,6 +34,13 @@ jobs:
         python -m black qlib -l 120 --check --diff
     # Test Qlib installed with pip
 
+    - name: Check Qlib with flake8
+        run: |
+          pip install --upgrade pip
+          pip install flake8
+          cd ..
+          flake8 --ignore=E501,F541,E266,E402,W503,E731,E203 qlib
+
     - name: Install Qlib with pip
       run: |
           python -m pip install numpy==1.19.5

.pre-commit-config.yaml

@@ -3,3 +3,10 @@ repos:
     rev: 22.1.0
     hooks:
     -   id: black
+        args: ["qlib", "-l 120"]
+
+-   repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+        - id: flake8
+          args: ["--ignore=E501,F541,E266,E402,W503,E731,E203"]

docs/developer/code_standard.rst

@@ -17,7 +17,7 @@ When you submit a PR request, you can check whether your code passes the CI test
 1. Qlib will check the code format with black. The PR will raise error if your code does not align to the standard of Qlib(e.g. a common error is the mixed use of space and tab).
  You can fix the bug by inputing the following code in the command line.
 
-.. code-block:: python
+.. code-block:: bash
 
     pip install black
     python -m black . -l 120
@@ -30,3 +30,19 @@ When you submit a PR request, you can check whether your code passes the CI test
 
     return -ICLoss()(pred, target, index)  # pylint: disable=E1130
 
+
+3. Qlib will check your code style flake8. The checking command is implemented in [github action workflow](https://github.com/microsoft/qlib/blob/0e8b94a552f1c457cfa6cd2c1bb3b87ebb3fb279/.github/workflows/test.yml#L73).
+ You can fix the bug by inputing the following code in the command line.
+
+.. code-block:: bash
+
+    flake8 --ignore E501,F541,E402,F401,W503,E741,E266,E203,E302,E731,E262,F523,F821,F811,F841,E713,E265,W291,E712,E722,W293 qlib
+
+
+4. Qlib has integrated pre-commit, which will make it easier for developers to format their code.
+ Just run the following two commands, and the code will be automatically formatted using black and flake8 when the git commit command is executed.
+
+.. code-block:: bash
+
+    pip install -e .[dev]
+    pre-commit install

qlib/__init__.py

@@ -12,6 +12,7 @@ import platform
 import subprocess
 from .log import get_module_logger
 
+
 # init qlib
 def init(default_conf="client", **kwargs):
     """

qlib/backtest/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
+
 from __future__ import annotations
 import copy
 from typing import List, Tuple, Union, TYPE_CHECKING
@@ -323,3 +324,6 @@ def format_decisions(
             last_dec_idx = i
     res[1].append((decisions[last_dec_idx], format_decisions(decisions[last_dec_idx + 1 :])))
     return res
+
+
+__all__ = ["Order"]

qlib/contrib/data/utils/sepdf.py

@@ -164,14 +164,14 @@ import builtins
 
 
 def _isinstance(instance, cls):
-    if isinstance_orig(instance, SepDataFrame):  # pylint: disable=E0602
+    if isinstance_orig(instance, SepDataFrame):  # pylint: disable=E0602  # noqa: F821
         if isinstance(cls, Iterable):
             for c in cls:
                 if c is pd.DataFrame:
                     return True
         elif cls is pd.DataFrame:
             return True
-    return isinstance_orig(instance, cls)  # pylint: disable=E0602
+    return isinstance_orig(instance, cls)  # pylint: disable=E0602  # noqa: F821
 
 
 builtins.isinstance_orig = builtins.isinstance

qlib/contrib/meta/__init__.py

@@ -2,3 +2,6 @@
 # Licensed under the MIT License.
 
 from .data_selection import MetaTaskDS, MetaDatasetDS, MetaModelDS
+
+
+__all__ = ["MetaTaskDS", "MetaDatasetDS", "MetaModelDS"]

qlib/contrib/meta/data_selection/__init__.py

@@ -3,3 +3,6 @@
 
 from .dataset import MetaDatasetDS, MetaTaskDS
 from .model import MetaModelDS
+
+
+__all__ = ["MetaDatasetDS", "MetaTaskDS", "MetaModelDS"]

qlib/contrib/meta/data_selection/model.py

@@ -10,7 +10,6 @@ from tqdm.auto import tqdm
 import copy
 from typing import Union, List
 
-from ....data.dataset.weight import Reweighter
 from ....model.meta.dataset import MetaTaskDataset
 from ....model.meta.model import MetaTaskModel
 from ....workflow import R
@@ -18,8 +17,8 @@ from .utils import ICLoss
 from .dataset import MetaDatasetDS
 
 from qlib.log import get_module_logger
-from qlib.data.dataset.weight import Reweighter
 from qlib.model.meta.task import MetaTask
+from qlib.data.dataset.weight import Reweighter
 from qlib.contrib.meta.data_selection.net import PredNet
 
 logger = get_module_logger("data selection")
@@ -98,7 +97,6 @@ class MetaModelDS(MetaTaskModel):
 
             if phase == "train":
                 opt.zero_grad()
-                norm_loss = nn.MSELoss()
                 loss.backward()
                 opt.step()
             elif phase == "test":

qlib/contrib/model/double_ensemble.py

@@ -249,7 +249,7 @@ class DEnsembleModel(Model, FeatureInt):
         return pred
 
     def predict_sub(self, submodel, df_data, features):
-        x_data, y_data = df_data["feature"].loc[:, features], df_data["label"]
+        x_data = df_data["feature"].loc[:, features]
         pred_sub = pd.Series(submodel.predict(x_data.values), index=x_data.index)
         return pred_sub
 

qlib/contrib/model/pytorch_sfm.py

@@ -84,7 +84,7 @@ class SFM_Model(nn.Module):
             if len(self.states) == 0:  # hasn't initialized yet
                 self.init_states(x)
             self.get_constants(x)
-            p_tm1 = self.states[0]
+            p_tm1 = self.states[0]  # noqa: F841
             h_tm1 = self.states[1]
             S_re_tm1 = self.states[2]
             S_im_tm1 = self.states[3]

qlib/contrib/model/pytorch_tabnet.py

@@ -477,10 +477,10 @@ class TabNet(nn.Module):
         sparse_loss = []
         out = torch.zeros(x.size(0), self.n_d).to(x.device)
         for step in self.steps:
-            x_te, l = step(x, x_a, priors)
+            x_te, loss = step(x, x_a, priors)
             out += F.relu(x_te[:, : self.n_d])  # split the feature from feat_transformer
             x_a = x_te[:, self.n_d :]
-            sparse_loss.append(l)
+            sparse_loss.append(loss)
         return self.fc(out), sum(sparse_loss)
 
 

qlib/contrib/online/__init__.py

@@ -1,4 +1,5 @@
 # pylint: skip-file
+# flake8: noqa
 
 '''
 TODO:

qlib/contrib/online/manager.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import yaml
 import pathlib

qlib/contrib/online/online_model.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import random
 import pandas as pd

qlib/contrib/online/operator.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import fire
 import pandas as pd

qlib/contrib/online/user.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import logging
 

qlib/contrib/online/utils.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import pathlib
 import pickle

qlib/contrib/report/analysis_model/__init__.py

@@ -2,3 +2,6 @@
 # Licensed under the MIT License.
 
 from .analysis_model_performance import model_performance_graph
+
+
+__all__ = ["model_performance_graph"]

qlib/contrib/report/analysis_position/__init__.py

@@ -6,3 +6,6 @@ from .score_ic import score_ic_graph
 from .report import report_graph
 from .rank_label import rank_label_graph
 from .risk_analysis import risk_analysis_graph
+
+
+__all__ = ["cumulative_return_graph", "score_ic_graph", "report_graph", "rank_label_graph", "risk_analysis_graph"]

qlib/contrib/strategy/__init__.py

@@ -15,3 +15,14 @@ from .rule_strategy import (
 )
 
 from .cost_control import SoftTopkStrategy
+
+
+__all__ = [
+    "TopkDropoutStrategy",
+    "WeightStrategyBase",
+    "EnhancedIndexingStrategy",
+    "TWAPStrategy",
+    "SBBStrategyBase",
+    "SBBStrategyEMA",
+    "SoftTopkStrategy",
+]

qlib/contrib/strategy/optimizer/__init__.py

@@ -4,3 +4,6 @@
 from .base import BaseOptimizer
 from .optimizer import PortfolioOptimizer
 from .enhanced_indexing import EnhancedIndexingOptimizer
+
+
+__all__ = ["BaseOptimizer", "PortfolioOptimizer", "EnhancedIndexingOptimizer"]

qlib/contrib/strategy/signal_strategy.py

@@ -131,10 +131,10 @@ class TopkDropoutStrategy(BaseSignalStrategy):
         if self.only_tradable:
             # If The strategy only consider tradable stock when make decision
             # It needs following actions to filter stocks
-            def get_first_n(l, n, reverse=False):
+            def get_first_n(li, n, reverse=False):
                 cur_n = 0
                 res = []
-                for si in reversed(l) if reverse else l:
+                for si in reversed(li) if reverse else li:
                     if self.trade_exchange.is_stock_tradable(
                         stock_id=si, start_time=trade_start_time, end_time=trade_end_time
                     ):
@@ -144,13 +144,13 @@ class TopkDropoutStrategy(BaseSignalStrategy):
                             break
                 return res[::-1] if reverse else res
 
-            def get_last_n(l, n):
+            def get_last_n(li, n):
-                return get_first_n(l, n, reverse=True)
+                return get_first_n(li, n, reverse=True)
 
-            def filter_stock(l):
+            def filter_stock(li):
                 return [
                     si
-                    for si in l
+                    for si in li
                     if self.trade_exchange.is_stock_tradable(
                         stock_id=si, start_time=trade_start_time, end_time=trade_end_time
                     )
@@ -158,14 +158,14 @@ class TopkDropoutStrategy(BaseSignalStrategy):
 
         else:
             # Otherwise, the stock will make decision with out the stock tradable info
-            def get_first_n(l, n):
+            def get_first_n(li, n):
-                return list(l)[:n]
+                return list(li)[:n]
 
-            def get_last_n(l, n):
+            def get_last_n(li, n):
-                return list(l)[-n:]
+                return list(li)[-n:]
 
-            def filter_stock(l):
+            def filter_stock(li):
-                return l
+                return li
 
         current_temp = copy.deepcopy(self.trade_position)
         # generate order list for this adjust date

qlib/contrib/tuner/__init__.py

@@ -1 +1,2 @@
 # pylint: skip-file
+# flake8: noqa

qlib/contrib/tuner/config.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import yaml
 import copy

qlib/contrib/tuner/launcher.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 # coding=utf-8
 

qlib/contrib/tuner/pipeline.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import os
 import json

qlib/contrib/tuner/space.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 from hyperopt import hp
 

qlib/contrib/tuner/tuner.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT License.
 
 # pylint: skip-file
+# flake8: noqa
 
 import os
 import yaml

qlib/contrib/workflow/__init__.py

@@ -2,3 +2,6 @@
 #  Licensed under the MIT License.
 from .record_temp import MultiSegRecord
 from .record_temp import SignalMseRecord
+
+
+__all__ = ["MultiSegRecord", "SignalMseRecord"]

qlib/data/__init__.py

@@ -35,3 +35,32 @@ from .cache import (
     DatasetURICache,
     MemoryCalendarCache,
 )
+
+
+__all__ = [
+    "D",
+    "CalendarProvider",
+    "InstrumentProvider",
+    "FeatureProvider",
+    "ExpressionProvider",
+    "DatasetProvider",
+    "LocalCalendarProvider",
+    "LocalInstrumentProvider",
+    "LocalFeatureProvider",
+    "LocalPITProvider",
+    "LocalExpressionProvider",
+    "LocalDatasetProvider",
+    "ClientCalendarProvider",
+    "ClientInstrumentProvider",
+    "ClientDatasetProvider",
+    "BaseProvider",
+    "LocalProvider",
+    "ClientProvider",
+    "ExpressionCache",
+    "DatasetCache",
+    "DiskExpressionCache",
+    "DiskDatasetCache",
+    "SimpleDatasetCache",
+    "DatasetURICache",
+    "MemoryCalendarCache",
+]

qlib/data/cache.py

@@ -33,7 +33,7 @@ from ..utils import (
 
 from ..log import get_module_logger
 from .base import Feature
-from .ops import Operators  # pylint: disable=W0611
+from .ops import Operators  # pylint: disable=W0611  # noqa: F401
 
 
 class QlibCacheException(RuntimeError):
@@ -528,7 +528,7 @@ class DiskExpressionCache(ExpressionCache):
                     CacheUtils.visit(cache_path)
                 series = read_bin(cache_path, start_index, end_index)
                 return series
-            except Exception as e:
+            except Exception:
                 series = None
                 self.logger.error("reading %s file error : %s" % (cache_path, traceback.format_exc()))
             return series
@@ -1068,7 +1068,7 @@ class SimpleDatasetCache(DatasetCache):
         super(SimpleDatasetCache, self).__init__(provider)
         try:
             self.local_cache_path: Path = Path(C["local_cache_path"]).expanduser().resolve()
-        except (KeyError, TypeError) as e:
+        except (KeyError, TypeError):
             self.logger.error("Assign a local_cache_path in config if you want to use this cache mechanism")
             raise
         self.logger.info(

qlib/data/data.py

@@ -38,7 +38,7 @@ from ..utils import (
     get_period_list,
 )
 from ..utils.paral import ParallelExt
-from .ops import Operators  # pylint: disable=W0611
+from .ops import Operators  # pylint: disable=W0611  # noqa: F401
 
 
 class ProviderBackendMixin:

qlib/data/dataset/__init__.py

@@ -609,3 +609,6 @@ class TSDatasetH(DatasetH):
 
         tsds = TSDataSampler(data=data, start=start, end=end, step_len=self.step_len, dtype=dtype, flt_data=flt_data)
         return tsds
+
+
+__all__ = ["Optional"]

qlib/data/ops.py

@@ -22,7 +22,7 @@ except ImportError:
         "#### Do not import qlib package in the repository directory in case of importing qlib from . without compiling #####"
     )
     raise
-except ValueError as e:
+except ValueError:
     print("!!!!!!!! A error occurs when importing operators implemented based on Cython.!!!!!!!!")
     print("!!!!!!!! They will be disabled. Please Upgrade your numpy to enable them     !!!!!!!!")
     # We catch this error because some platform can't upgrade there package (e.g. Kaggle)

qlib/data/storage/__init__.py

@@ -2,3 +2,6 @@
 # Licensed under the MIT License.
 
 from .storage import CalendarStorage, InstrumentStorage, FeatureStorage, CalVT, InstVT, InstKT
+
+
+__all__ = ["CalendarStorage", "InstrumentStorage", "FeatureStorage", "CalVT", "InstVT", "InstKT"]

qlib/model/__init__.py

@@ -4,3 +4,6 @@
 import warnings
 
 from .base import Model
+
+
+__all__ = ["Model", "warnings"]

qlib/model/meta/__init__.py

@@ -3,3 +3,6 @@
 
 from .task import MetaTask
 from .dataset import MetaTaskDataset
+
+
+__all__ = ["MetaTask", "MetaTaskDataset"]

qlib/model/riskmodel/__init__.py

@@ -5,3 +5,11 @@ from .base import RiskModel
 from .poet import POETCovEstimator
 from .shrink import ShrinkCovEstimator
 from .structured import StructuredCovEstimator
+
+
+__all__ = [
+    "RiskModel",
+    "POETCovEstimator",
+    "ShrinkCovEstimator",
+    "StructuredCovEstimator",
+]

qlib/utils/__init__.py

@@ -14,22 +14,19 @@ import json
 import yaml
 import redis
 import bisect
-import shutil
 import struct
 import difflib
 import inspect
 import hashlib
-import warnings
 import datetime
 import requests
-import tempfile
 import importlib
 import contextlib
 import collections
 import numpy as np
 import pandas as pd
 from pathlib import Path
-from typing import List, Dict, Union, Tuple, Any, Text, Optional, Callable
+from typing import List, Dict, Union, Tuple, Any, Optional, Callable
 from types import ModuleType
 from urllib.parse import urlparse
 from packaging import version
@@ -1047,3 +1044,12 @@ def fname_to_code(fname: str):
     if fname.startswith(prefix):
         fname = fname.lstrip(prefix)
     return fname
+
+
+__all__ = [
+    "get_or_create_path",
+    "save_multiple_parts_file",
+    "unpack_archive_with_buffer",
+    "get_tmp_file_with_buffer",
+    "set_log_with_config",
+]

qlib/utils/time.py

@@ -199,10 +199,8 @@ class Freq:
         """
         base_freq = Freq(base_freq)
         # use the nearest freq greater than 0
-        _freq_minutes = []
         min_freq = None
         for _freq in freq_list:
-            freq = Freq(_freq)
             _min_delta = Freq.get_min_delta(base_freq, _freq)
             if _min_delta < 0:
                 continue

qlib/workflow/expm.py

@@ -170,12 +170,9 @@ class ExpManager:
             experiment_name = self._default_exp_name
 
         if create:
-            exp, is_new = self._get_or_create_exp(experiment_id=experiment_id, experiment_name=experiment_name)
+            exp, _ = self._get_or_create_exp(experiment_id=experiment_id, experiment_name=experiment_name)
         else:
-            exp, is_new = (
+            exp = self._get_exp(experiment_id=experiment_id, experiment_name=experiment_name)
-                self._get_exp(experiment_id=experiment_id, experiment_name=experiment_name),
-                False,
-            )
         if self.active_experiment is None and start:
             self.active_experiment = exp
             # start the recorder
@@ -201,7 +198,7 @@ class ExpManager:
             # So we supported it in the interface wrapper
             pr = urlparse(self.uri)
             if pr.scheme == "file":
-                with FileLock(os.path.join(pr.netloc, pr.path, "filelock")) as f:  # pylint: disable=E0110
+                with FileLock(os.path.join(pr.netloc, pr.path, "filelock")):  # pylint: disable=E0110
                     return self.create_exp(experiment_name), True
             # NOTE: for other schemes like http, we double check to avoid create exp conflicts
             try:

qlib/workflow/record_temp.py

@@ -146,7 +146,7 @@ class RecordTemp:
 
             for item in self.list():
                 ps = self.get_path(item).split("/")
-                dirn, fn = "/".join(ps[:-1]), ps[-1]
+                dirn = "/".join(ps[:-1])
                 if self.get_path(item) not in _get_arts(dirn):
                     raise FileNotFoundError
         if parents: