Fix pylint (#888)

* add_pylint_to_workflow

* fix-pylint

* fix_pylinterror

* fix-issue

qlib/workflow/__init__.py

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 
 from contextlib import contextmanager
-from typing import Text, Optional, Any, Dict, Text, Optional
+from typing import Text, Optional, Any, Dict
 from .expm import ExpManager
 from .exp import Experiment
 from .recorder import Recorder

qlib/workflow/cli.py

@@ -1,7 +1,8 @@
 #  Copyright (c) Microsoft Corporation.
 #  Licensed under the MIT License.
 
-import sys, os
+import sys
+import os
 from pathlib import Path
 
 import qlib

qlib/workflow/exp.py

@@ -2,10 +2,10 @@
 # Licensed under the MIT License.
 
 from typing import Dict, List, Union
-import mlflow, logging
+import mlflow
+import logging
 from mlflow.entities import ViewType
 from mlflow.exceptions import MlflowException
-from pathlib import Path
 from .recorder import Recorder, MLflowRecorder
 from ..log import get_module_logger
 
@@ -271,7 +271,7 @@ class MLflowExperiment(Experiment):
 
         return self.active_recorder
 
-    def end(self, recorder_status):
+    def end(self, recorder_status=Recorder.STATUS_S):
         if self.active_recorder is not None:
             self.active_recorder.end_run(recorder_status)
             self.active_recorder = None
@@ -299,8 +299,10 @@ class MLflowExperiment(Experiment):
                 run = self._client.get_run(recorder_id)
                 recorder = MLflowRecorder(self.id, self._uri, mlflow_run=run)
                 return recorder
-            except MlflowException:
-                raise ValueError("No valid recorder has been found, please make sure the input recorder id is correct.")
+            except MlflowException as mlflow_exp:
+                raise ValueError(
+                    "No valid recorder has been found, please make sure the input recorder id is correct."
+                ) from mlflow_exp
         elif recorder_name is not None:
             logger.warning(
                 f"Please make sure the recorder name {recorder_name} is unique, we will only return the latest recorder if there exist several matched the given name."
@@ -332,7 +334,7 @@ class MLflowExperiment(Experiment):
         except MlflowException as e:
             raise Exception(
                 f"Error: {e}. Something went wrong when deleting recorder. Please check if the name/id of the recorder is correct."
-            )
+            ) from e
 
     UNLIMITED = 50000  # FIXME: Mlflow can only list 50000 records at most!!!!!!!
 
@@ -362,10 +364,10 @@ class MLflowExperiment(Experiment):
         )
         rids = []
         recorders = []
-        for i in range(len(runs)):
-            recorder = MLflowRecorder(self.id, self._uri, mlflow_run=runs[i])
+        for i, n in enumerate(runs):
+            recorder = MLflowRecorder(self.id, self._uri, mlflow_run=n)
             if status is None or recorder.status == status:
-                rids.append(runs[i].info.run_id)
+                rids.append(n.info.run_id)
                 recorders.append(recorder)
 
         if rtype == Experiment.RT_D:

qlib/workflow/expm.py

@@ -6,9 +6,7 @@ import mlflow
 from filelock import FileLock
 from mlflow.exceptions import MlflowException, RESOURCE_ALREADY_EXISTS, ErrorCode
 from mlflow.entities import ViewType
-import os, logging
-from pathlib import Path
-from contextlib import contextmanager
+import os
 from typing import Optional, Text
 
 from .exp import MLflowExperiment, Experiment
@@ -203,7 +201,7 @@ class ExpManager:
             # So we supported it in the interface wrapper
             pr = urlparse(self.uri)
             if pr.scheme == "file":
-                with FileLock(os.path.join(pr.netloc, pr.path, "filelock")) as f:
+                with FileLock(os.path.join(pr.netloc, pr.path, "filelock")) as f:  # pylint: disable=E0110
                     return self.create_exp(experiment_name), True
             # NOTE: for other schemes like http, we double check to avoid create exp conflicts
             try:
@@ -363,7 +361,7 @@ class MLflowExpManager(ExpManager):
             experiment_id = self.client.create_experiment(experiment_name)
         except MlflowException as e:
             if e.error_code == ErrorCode.Name(RESOURCE_ALREADY_EXISTS):
-                raise ExpAlreadyExistError()
+                raise ExpAlreadyExistError() from e
             raise e
 
         experiment = MLflowExperiment(experiment_id, experiment_name, self.uri)
@@ -387,10 +385,10 @@ class MLflowExpManager(ExpManager):
                     raise MlflowException("No valid experiment has been found.")
                 experiment = MLflowExperiment(exp.experiment_id, exp.name, self.uri)
                 return experiment
-            except MlflowException:
+            except MlflowException as e:
                 raise ValueError(
                     "No valid experiment has been found, please make sure the input experiment id is correct."
-                )
+                ) from e
         elif experiment_name is not None:
             try:
                 exp = self.client.get_experiment_by_name(experiment_name)
@@ -401,9 +399,9 @@ class MLflowExpManager(ExpManager):
             except MlflowException as e:
                 raise ValueError(
                     "No valid experiment has been found, please make sure the input experiment name is correct."
-                )
+                ) from e
 
-    def search_records(self, experiment_ids, **kwargs):
+    def search_records(self, experiment_ids=None, **kwargs):
         filter_string = "" if kwargs.get("filter_string") is None else kwargs.get("filter_string")
         run_view_type = 1 if kwargs.get("run_view_type") is None else kwargs.get("run_view_type")
         max_results = 100000 if kwargs.get("max_results") is None else kwargs.get("max_results")
@@ -425,7 +423,7 @@ class MLflowExpManager(ExpManager):
         except MlflowException as e:
             raise Exception(
                 f"Error: {e}. Something went wrong when deleting experiment. Please check if the name/id of the experiment is correct."
-            )
+            ) from e
 
     def list_experiments(self):
         # retrieve all the existing experiments

qlib/workflow/online/manager.py

@@ -83,15 +83,14 @@ For simplicity
 """
 
 import logging
-from typing import Callable, Dict, List, Union
+from typing import Callable, List, Union
 
 import pandas as pd
 from qlib import get_module_logger
 from qlib.data.data import D
 from qlib.log import set_global_logger_level
 from qlib.model.ens.ensemble import AverageEnsemble
-from qlib.model.trainer import DelayTrainerR, Trainer, TrainerR
-from qlib.utils import flatten_dict
+from qlib.model.trainer import Trainer, TrainerR
 from qlib.utils.serial import Serializable
 from qlib.workflow.online.strategy import OnlineStrategy
 from qlib.workflow.task.collect import MergeCollector

qlib/workflow/online/strategy.py

@@ -5,9 +5,7 @@
 OnlineStrategy module is an element of online serving.
 """
 
-from copy import deepcopy
-from typing import List, Tuple, Union
-from qlib.data.data import D
+from typing import List, Union
 from qlib.log import get_module_logger
 from qlib.model.ens.group import RollingGroup
 from qlib.utils import transform_end_date

qlib/workflow/online/update.py

@@ -148,7 +148,7 @@ class DSBasedUpdater(RecordUpdater, metaclass=ABCMeta):
         self.rmdl = loader_cls(rec=record)
 
         latest_date = D.calendar(freq=freq)[-1]
-        if to_date == None:
+        if to_date is None:
             to_date = latest_date
         to_date = pd.Timestamp(to_date)
 
@@ -191,7 +191,9 @@ class DSBasedUpdater(RecordUpdater, metaclass=ABCMeta):
         else:
             hist_ref = self.hist_ref
 
-        start_time_buffer = get_date_by_shift(self.last_end, -hist_ref + 1, clip_shift=False, freq=self.freq)
+        start_time_buffer = get_date_by_shift(
+            self.last_end, -hist_ref + 1, clip_shift=False, freq=self.freq  # pylint: disable=E1130
+        )
         start_time = get_date_by_shift(self.last_end, 1, freq=self.freq)
         seg = {"test": (start_time, self.to_date)}
         return self.rmdl.get_dataset(

qlib/workflow/online/utils.py

@@ -8,10 +8,8 @@ This allows us to use efficient submodels as the market-style changing.
 """
 
 from typing import List, Union
-from qlib.data.dataset import TSDatasetH
 
 from qlib.log import get_module_logger
-from qlib.utils import get_callable_kwargs
 from qlib.utils.exceptions import LoadObjectError
 from qlib.workflow.online.update import PredUpdater
 from qlib.workflow.recorder import Recorder

qlib/workflow/recorder.py

@@ -2,15 +2,18 @@
 # Licensed under the MIT License.
 
 import os
-from qlib.utils.serial import Serializable
-import mlflow, logging
-import shutil, os, pickle, tempfile, codecs, pickle
+import mlflow
+import logging
+import shutil
+import pickle
+import tempfile
 from pathlib import Path
 from datetime import datetime
 
+from qlib.utils.serial import Serializable
 from qlib.utils.exceptions import LoadObjectError
 from qlib.utils.paral import AsyncCaller
-from ..utils.objm import FileManager
+
 from ..log import TimeInspector, get_module_logger
 from mlflow.store.artifact.azure_blob_artifact_repo import AzureBlobArtifactRepository
 
@@ -355,7 +358,7 @@ class MLflowRecorder(Recorder):
                 shutil.rmtree(Path(path).absolute().parent)
             return data
         except Exception as e:
-            raise LoadObjectError(str(e))
+            raise LoadObjectError(str(e)) from e
 
     @AsyncCaller.async_dec(ac_attr="async_log")
     def log_params(self, **kwargs):

qlib/workflow/task/gen.py

@@ -82,7 +82,6 @@ class TaskGen(metaclass=abc.ABCMeta):
         typing.List[dict]:
             A list of tasks
         """
-        pass
 
     def __call__(self, *args, **kwargs):
         """

qlib/workflow/task/utils.py

@@ -189,7 +189,7 @@ class TimeAdjuster:
         """
         if isinstance(segment, dict):
             return {k: self.align_seg(seg) for k, seg in segment.items()}
-        elif isinstance(segment, tuple) or isinstance(segment, list):
+        elif isinstance(segment, (tuple, list)):
             return self.align_time(segment[0], tp_type="start"), self.align_time(segment[1], tp_type="end")
         else:
             raise NotImplementedError(f"This type of input is not supported")