update python version (#1868)

* update python version

* fix: Correct selector handling and add time filtering in storage.py

* fix: convert index and columns to list in repr methods

* feat: Add Makefile for managing project prerequisites

* feat: Add Cython extensions for rolling and expanding operations

* resolve install error

* fix lint error

* fix lint error

* fix lint error

* fix lint error

* fix lint error

* update build package

* update makefile

* update ci yaml

* fix docs build error

* fix ubuntu install error

* fix docs build error

* fix install error

* fix install error

* fix install error

* fix install error

* fix pylint error

* fix pylint error

* fix pylint error

* fix pylint error

* fix pylint error E1123

* fix pylint error R0917

* fix pytest error

* fix pytest error

* fix pytest error

* update code

* update code

* fix ci error

* fix pylint error

* fix black error

* fix pytest error

* fix CI error

* fix CI error

* add python version to CI

* add python version to CI

* add python version to CI

* fix pylint error

* fix pytest general nn error

* fix CI error

* optimize code

* add coments

* Extended macos version

* remove build package

---------

Co-authored-by: Young <afe.young@gmail.com>

qlib/__init__.py

@@ -6,7 +6,7 @@ __version__ = "0.9.5.99"
 __version__bak = __version__  # This version is backup for QlibConfig.reset_qlib_version
 import os
 from typing import Union
-import yaml
+from ruamel.yaml import YAML
 import logging
 import platform
 import subprocess
@@ -176,7 +176,8 @@ def init_from_yaml_conf(conf_path, **kwargs):
         config = {}
     else:
         with open(conf_path) as f:
-            config = yaml.safe_load(f)
+            yaml = YAML(typ="safe", pure=True)
+            config = yaml.load(f)
     config.update(kwargs)
     default_conf = config.pop("default_conf", "client")
     init(default_conf, **config)
@@ -272,7 +273,8 @@ def auto_init(**kwargs):
         logger = get_module_logger("Initialization")
         conf_pp = pp / "config.yaml"
         with conf_pp.open() as f:
-            conf = yaml.safe_load(f)
+            yaml = YAML(typ="safe", pure=True)
+            conf = yaml.load(f)
 
         conf_type = conf.get("conf_type", "origin")
         if conf_type == "origin":

qlib/backtest/high_performance_ds.py

@@ -278,7 +278,7 @@ class BaseSingleMetric:
         raise NotImplementedError(f"Please implement the `empty` method")
 
     def add(self, other: BaseSingleMetric, fill_value: float = None) -> BaseSingleMetric:
-        """Replace np.NaN with fill_value in two metrics and add them."""
+        """Replace np.nan with fill_value in two metrics and add them."""
 
         raise NotImplementedError(f"Please implement the `add` method")
 
@@ -412,7 +412,7 @@ class BaseOrderIndicator:
         metrics : Union[str, List[str]]
             all metrics needs to be sumed.
         fill_value : float, optional
-            fill np.NaN with value. By default None.
+            fill np.nan with value. By default None.
         """
 
         raise NotImplementedError(f"Please implement the 'sum_all_indicators' method")

qlib/backtest/report.py

@@ -325,9 +325,9 @@ class Indicator:
 
     def _update_order_fulfill_rate(self) -> None:
         def func(deal_amount, amount):
-            # deal_amount is np.NaN or None when there is no inner decision. So full fill rate is 0.
+            # deal_amount is np.nan or None when there is no inner decision. So full fill rate is 0.
             tmp_deal_amount = deal_amount.reindex(amount.index, 0)
-            tmp_deal_amount = tmp_deal_amount.replace({np.NaN: 0})
+            tmp_deal_amount = tmp_deal_amount.replace({np.nan: 0})
             return tmp_deal_amount / amount
 
         self.order_indicator.transfer(func, "ffr")
@@ -354,8 +354,8 @@ class Indicator:
         )
 
         def func(trade_price, deal_amount):
-            # trade_price is np.NaN instead of inf when deal_amount is zero.
-            tmp_deal_amount = deal_amount.replace({0: np.NaN})
+            # trade_price is np.nan instead of inf when deal_amount is zero.
+            tmp_deal_amount = deal_amount.replace({0: np.nan})
             return trade_price / tmp_deal_amount
 
         self.order_indicator.transfer(func, "trade_price")
@@ -425,7 +425,7 @@ class Indicator:
         assert isinstance(price_s, idd.SingleData)
         price_s = price_s.loc[(price_s > 1e-08).data.astype(bool)]
         # NOTE ~(price_s < 1e-08) is different from price_s >= 1e-8
-        #   ~(np.NaN < 1e-8) -> ~(False)  -> True
+        #   ~(np.nan < 1e-8) -> ~(False)  -> True
 
         assert isinstance(price_s, idd.SingleData)
         if agg == "vwap":

qlib/contrib/data/handler.py

@@ -58,7 +58,7 @@ class Alpha360(DataHandlerLP):
         fit_end_time=None,
         filter_pipe=None,
         inst_processors=None,
-        **kwargs
+        **kwargs,
     ):
         infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time)
         learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time)
@@ -83,7 +83,7 @@ class Alpha360(DataHandlerLP):
             data_loader=data_loader,
             learn_processors=learn_processors,
             infer_processors=infer_processors,
-            **kwargs
+            **kwargs,
         )
 
     def get_label_config(self):
@@ -109,7 +109,7 @@ class Alpha158(DataHandlerLP):
         process_type=DataHandlerLP.PTYPE_A,
         filter_pipe=None,
         inst_processors=None,
-        **kwargs
+        **kwargs,
     ):
         infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time)
         learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time)
@@ -134,7 +134,7 @@ class Alpha158(DataHandlerLP):
             infer_processors=infer_processors,
             learn_processors=learn_processors,
             process_type=process_type,
-            **kwargs
+            **kwargs,
         )
 
     def get_feature_config(self):

qlib/contrib/model/catboost_model.py

@@ -33,7 +33,7 @@ class CatBoostModel(Model, FeatureInt):
         verbose_eval=20,
         evals_result=dict(),
         reweighter=None,
-        **kwargs
+        **kwargs,
     ):
         df_train, df_valid = dataset.prepare(
             ["train", "valid"],

qlib/contrib/model/double_ensemble.py

@@ -31,7 +31,7 @@ class DEnsembleModel(Model, FeatureInt):
         sub_weights=None,
         epochs=100,
         early_stopping_rounds=None,
-        **kwargs
+        **kwargs,
     ):
         self.base_model = base_model  # "gbm" or "mlp", specifically, we use lgbm for "gbm"
         self.num_models = num_models  # the number of sub-models

qlib/contrib/model/pytorch_adarnn.py

@@ -56,7 +56,7 @@ class ADARNN(Model):
         n_splits=2,
         GPU=0,
         seed=None,
-        **_
+        **_,
     ):
         # Set logger.
         self.logger = get_module_logger("ADARNN")
@@ -154,10 +154,7 @@ class ADARNN(Model):
         self.model.train()
         criterion = nn.MSELoss()
         dist_mat = torch.zeros(self.num_layers, self.len_seq).to(self.device)
-        len_loader = np.inf
-        for loader in train_loader_list:
-            if len(loader) < len_loader:
-                len_loader = len(loader)
+        out_weight_list = None
         for data_all in zip(*train_loader_list):
             #  for data_all in zip(*train_loader_list):
             self.train_optimizer.zero_grad()
@@ -571,6 +568,7 @@ class TransferLoss:
         Returns:
             [tensor] -- transfer loss
         """
+        loss = None
         if self.loss_type in ("mmd_lin", "mmd"):
             mmdloss = MMD_loss(kernel_type="linear")
             loss = mmdloss(X, Y)

qlib/contrib/model/pytorch_add.py

@@ -63,7 +63,7 @@ class ADD(Model):
         mu=0.05,
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("ADD")

qlib/contrib/model/pytorch_alstm.py

@@ -52,7 +52,7 @@ class ALSTM(Model):
         optimizer="adam",
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("ALSTM")

qlib/contrib/model/pytorch_alstm_ts.py

@@ -56,7 +56,7 @@ class ALSTM(Model):
         n_jobs=10,
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("ALSTM")

qlib/contrib/model/pytorch_gats.py

@@ -56,7 +56,7 @@ class GATs(Model):
         optimizer="adam",
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("GATs")

qlib/contrib/model/pytorch_gats_ts.py

@@ -73,7 +73,7 @@ class GATs(Model):
         GPU=0,
         n_jobs=10,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("GATs")

qlib/contrib/model/pytorch_general_nn.py

@@ -319,7 +319,12 @@ class GeneralPTNN(Model):
         if self.use_gpu:
             torch.cuda.empty_cache()
 
-    def predict(self, dataset: Union[DatasetH, TSDatasetH]):
+    def predict(
+        self,
+        dataset: Union[DatasetH, TSDatasetH],
+        batch_size=None,
+        n_jobs=None,
+    ):
         if not self.fitted:
             raise ValueError("model is not fitted yet!")
 

qlib/contrib/model/pytorch_gru.py

@@ -52,7 +52,7 @@ class GRU(Model):
         optimizer="adam",
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("GRU")

qlib/contrib/model/pytorch_gru_ts.py

@@ -54,7 +54,7 @@ class GRU(Model):
         n_jobs=10,
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("GRU")

qlib/contrib/model/pytorch_hist.py

@@ -59,7 +59,7 @@ class HIST(Model):
         optimizer="adam",
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("HIST")

qlib/contrib/model/pytorch_igmtf.py

@@ -55,7 +55,7 @@ class IGMTF(Model):
         optimizer="adam",
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("IGMTF")

qlib/contrib/model/pytorch_krnn.py

@@ -255,7 +255,7 @@ class KRNN(Model):
         optimizer="adam",
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("KRNN")

qlib/contrib/model/pytorch_localformer.py

@@ -44,7 +44,7 @@ class LocalformerModel(Model):
         n_jobs=10,
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # set hyper-parameters.
         self.d_model = d_model

qlib/contrib/model/pytorch_localformer_ts.py

@@ -42,7 +42,7 @@ class LocalformerModel(Model):
         n_jobs=10,
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # set hyper-parameters.
         self.d_model = d_model

qlib/contrib/model/pytorch_lstm.py

@@ -51,7 +51,7 @@ class LSTM(Model):
         optimizer="adam",
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("LSTM")

qlib/contrib/model/pytorch_lstm_ts.py

@@ -53,7 +53,7 @@ class LSTM(Model):
         n_jobs=10,
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("LSTM")

qlib/contrib/model/pytorch_sandwich.py

@@ -35,7 +35,7 @@ class SandwichModel(nn.Module):
         rnn_layers,
         dropout,
         device,
-        **params
+        **params,
     ):
         """Build a Sandwich model
 
@@ -129,7 +129,7 @@ class Sandwich(Model):
         optimizer="adam",
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("Sandwich")

qlib/contrib/model/pytorch_sfm.py

@@ -212,7 +212,7 @@ class SFM(Model):
         optimizer="gd",
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("SFM")

qlib/contrib/model/pytorch_tcn.py

@@ -56,7 +56,7 @@ class TCN(Model):
         optimizer="adam",
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("TCN")

qlib/contrib/model/pytorch_tcn_ts.py

@@ -54,7 +54,7 @@ class TCN(Model):
         n_jobs=10,
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("TCN")

qlib/contrib/model/pytorch_tcts.py

@@ -58,7 +58,7 @@ class TCTS(Model):
         mode="soft",
         seed=None,
         lowest_valid_performance=0.993,
-        **kwargs
+        **kwargs,
     ):
         # Set logger.
         self.logger = get_module_logger("TCTS")

qlib/contrib/model/pytorch_transformer.py

@@ -43,7 +43,7 @@ class TransformerModel(Model):
         n_jobs=10,
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # set hyper-parameters.
         self.d_model = d_model

qlib/contrib/model/pytorch_transformer_ts.py

@@ -41,7 +41,7 @@ class TransformerModel(Model):
         n_jobs=10,
         GPU=0,
         seed=None,
-        **kwargs
+        **kwargs,
     ):
         # set hyper-parameters.
         self.d_model = d_model

qlib/contrib/model/xgboost.py

@@ -28,7 +28,7 @@ class XGBModel(Model, FeatureInt):
         verbose_eval=20,
         evals_result=dict(),
         reweighter=None,
-        **kwargs
+        **kwargs,
     ):
         df_train, df_valid = dataset.prepare(
             ["train", "valid"],
@@ -63,7 +63,7 @@ class XGBModel(Model, FeatureInt):
             early_stopping_rounds=early_stopping_rounds,
             verbose_eval=verbose_eval,
             evals_result=evals_result,
-            **kwargs
+            **kwargs,
         )
         evals_result["train"] = list(evals_result["train"].values())[0]
         evals_result["valid"] = list(evals_result["valid"].values())[0]

qlib/contrib/online/manager.py

@@ -4,10 +4,10 @@
 # pylint: skip-file
 # flake8: noqa
 
-import yaml
 import pathlib
 import pandas as pd
 import shutil
+from ruamel.yaml import YAML
 from ...backtest.account import Account
 from .user import User
 from .utils import load_instance, save_instance
@@ -110,7 +110,8 @@ class UserManager:
             raise ValueError("User data for {} already exists".format(user_id))
 
         with config_file.open("r") as fp:
-            config = yaml.safe_load(fp)
+            yaml = YAML(typ="safe", pure=True)
+            config = yaml.load(fp)
         # load model
         model = init_instance_by_config(config["model"])
 

qlib/contrib/online/utils.py

@@ -6,8 +6,8 @@
 
 import pathlib
 import pickle
-import yaml
 import pandas as pd
+from ruamel.yaml import YAML
 from ...data import D
 from ...config import C
 from ...log import get_module_logger
@@ -91,7 +91,8 @@ def prepare(um, today, user_id, exchange_config=None):
     dates.append(get_next_trading_date(dates[-1], future=True))
     if exchange_config:
         with pathlib.Path(exchange_config).open("r") as fp:
-            exchange_paras = yaml.safe_load(fp)
+            yaml = YAML(typ="safe", pure=True)
+            exchange_paras = yaml.load(fp)
     else:
         exchange_paras = {}
     trade_exchange = Exchange(trade_dates=dates, **exchange_paras)

qlib/contrib/report/graph.py

@@ -176,7 +176,7 @@ class HeatmapGraph(BaseGraph):
                 x=self._df.columns,
                 y=self._df.index,
                 z=self._df.values.tolist(),
-                **self._graph_kwargs
+                **self._graph_kwargs,
             )
         ]
         return _data
@@ -213,7 +213,7 @@ class SubplotsGraph:
         sub_graph_layout: dict = None,
         sub_graph_data: list = None,
         subplots_kwargs: dict = None,
-        **kwargs
+        **kwargs,
     ):
         """
 
@@ -355,7 +355,7 @@ class SubplotsGraph:
                         df=self._df.loc[:, [column_name]],
                         name_dict={column_name: temp_name},
                         graph_kwargs=_graph_kwargs,
-                    )
+                    ),
                 )
             else:
                 raise TypeError()

qlib/contrib/rolling/base.py

@@ -2,11 +2,11 @@
 # Licensed under the MIT License.
 from copy import deepcopy
 from pathlib import Path
+from ruamel.yaml import YAML
 from typing import List, Optional, Union
 
 import fire
 import pandas as pd
-import yaml
 
 from qlib import auto_init
 from qlib.log import get_module_logger
@@ -117,7 +117,8 @@ class Rolling:
 
     def _raw_conf(self) -> dict:
         with self.conf_path.open("r") as f:
-            return yaml.safe_load(f)
+            yaml = YAML(typ="safe", pure=True)
+            return yaml.load(f)
 
     def _replace_handler_with_cache(self, task: dict):
         """

qlib/contrib/tuner/config.py

@@ -4,9 +4,9 @@
 # pylint: skip-file
 # flake8: noqa
 
-import yaml
 import copy
 import os
+from ruamel.yaml import YAML
 
 
 class TunerConfigManager:
@@ -16,7 +16,8 @@ class TunerConfigManager:
         self.config_path = config_path
 
         with open(config_path) as fp:
-            config = yaml.safe_load(fp)
+            yaml = YAML(typ="safe", pure=True)
+            config = yaml.load(fp)
         self.config = copy.deepcopy(config)
 
         self.pipeline_ex_config = PipelineExperimentConfig(config.get("experiment", dict()), self)

qlib/data/dataset/storage.py

@@ -104,15 +104,24 @@ class HashingStockStorage(BaseHandlerStorage):
         """
 
         stock_selector = slice(None)
+        time_selector = slice(None)  # by default not filter by time.
 
         if level is None:
+            # For directly applying.
             if isinstance(selector, tuple) and self.stock_level < len(selector):
+                # full selector format
                 stock_selector = selector[self.stock_level]
+                time_selector = selector[1 - self.stock_level]
             elif isinstance(selector, (list, str)) and self.stock_level == 0:
+                # only stock selector
                 stock_selector = selector
         elif level in ("instrument", self.stock_level):
             if isinstance(selector, tuple):
+                # NOTE: How could the stock level selector be a tuple?
                 stock_selector = selector[0]
+                raise TypeError(
+                    "I forget why would this case appear. But I think it does not make sense. So we raise a error for that case."
+                )
             elif isinstance(selector, (list, str)):
                 stock_selector = selector
 
@@ -120,7 +129,7 @@ class HashingStockStorage(BaseHandlerStorage):
             raise TypeError(f"stock selector must be type str|list, or slice(None), rather than {stock_selector}")
 
         if stock_selector == slice(None):
-            return self.hash_df
+            return self.hash_df, time_selector
 
         if isinstance(stock_selector, str):
             stock_selector = [stock_selector]
@@ -129,7 +138,7 @@ class HashingStockStorage(BaseHandlerStorage):
         for each_stock in sorted(stock_selector):
             if each_stock in self.hash_df:
                 select_dict[each_stock] = self.hash_df[each_stock]
-        return select_dict
+        return select_dict, time_selector
 
     def fetch(
         self,
@@ -138,10 +147,13 @@ class HashingStockStorage(BaseHandlerStorage):
         col_set: Union[str, List[str]] = DataHandler.CS_ALL,
         fetch_orig: bool = True,
     ) -> pd.DataFrame:
-        fetch_stock_df_list = list(self._fetch_hash_df_by_stock(selector=selector, level=level).values())
+        fetch_stock_df_list, time_selector = self._fetch_hash_df_by_stock(selector=selector, level=level)
+        fetch_stock_df_list = list(fetch_stock_df_list.values())
         for _index, stock_df in enumerate(fetch_stock_df_list):
             fetch_col_df = fetch_df_by_col(df=stock_df, col_set=col_set)
-            fetch_index_df = fetch_df_by_index(df=fetch_col_df, selector=selector, level=level, fetch_orig=fetch_orig)
+            fetch_index_df = fetch_df_by_index(
+                df=fetch_col_df, selector=time_selector, level="datetime", fetch_orig=fetch_orig
+            )
             fetch_stock_df_list[_index] = fetch_index_df
         if len(fetch_stock_df_list) == 0:
             index_names = ("instrument", "datetime") if self.stock_level == 0 else ("datetime", "instrument")

qlib/data/filter.py

@@ -164,6 +164,7 @@ class SeriesDFilter(BaseDFilter):
         timestamp = []
         _lbool = None
         _ltime = None
+        _cur_start = None
         for _ts, _bool in timestamp_series.items():
             # there is likely to be NAN when the filter series don't have the
             # bool value, so we just change the NAN into False

qlib/rl/contrib/naive_config_parser.py

@@ -7,8 +7,7 @@ import shutil
 import sys
 import tempfile
 from importlib import import_module
-
-import yaml
+from ruamel.yaml import YAML
 
 
 DELETE_KEY = "_delete_"
@@ -57,7 +56,8 @@ def parse_backtest_config(path: str) -> dict:
                 del sys.modules[tmp_module_name]
             else:
                 with open(tmp_config_file.name) as input_stream:
-                    config = yaml.safe_load(input_stream)
+                    yaml = YAML(typ="safe", pure=True)
+                    config = yaml.load(input_stream)
 
     if "_base_" in config:
         base_file_name = config.pop("_base_")

qlib/rl/contrib/train_onpolicy.py

@@ -8,12 +8,12 @@ import random
 import sys
 import warnings
 from pathlib import Path
+from ruamel.yaml import YAML
 from typing import cast, List, Optional
 
 import numpy as np
 import pandas as pd
 import torch
-import yaml
 from qlib.backtest import Order
 from qlib.backtest.decision import OrderDir
 from qlib.constant import ONE_MIN
@@ -263,6 +263,7 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     with open(args.config_path, "r") as input_stream:
-        config = yaml.safe_load(input_stream)
+        yaml = YAML(typ="safe", pure=True)
+        config = yaml.load(input_stream)
 
     main(config, run_training=not args.no_training, run_backtest=args.run_backtest)

qlib/utils/__init__.py

@@ -10,7 +10,6 @@ import os
 import re
 import copy
 import json
-import yaml
 import redis
 import bisect
 import struct
@@ -25,6 +24,7 @@ import pandas as pd
 from pathlib import Path
 from typing import List, Union, Optional, Callable
 from packaging import version
+from ruamel.yaml import YAML
 from .file import (
     get_or_create_path,
     save_multiple_parts_file,
@@ -244,12 +244,13 @@ def parse_config(config):
     if not isinstance(config, str):
         return config
     # Check whether config is file
+    yaml = YAML(typ="safe", pure=True)
     if os.path.exists(config):
         with open(config, "r") as f:
-            return yaml.safe_load(f)
+            return yaml.load(f)
     # Check whether the str can be parsed
     try:
-        return yaml.safe_load(config)
+        return yaml.load(config)
     except BaseException as base_exp:
         raise ValueError("cannot parse config!") from base_exp
 
@@ -799,6 +800,7 @@ def fill_placeholder(config: dict, config_extend: dict):
                     )
         return value
 
+    item_keys = None
     while top < tail:
         now_item = item_queue[top]
         top += 1

qlib/utils/index_data.py

@@ -44,7 +44,7 @@ def concat(data_list: Union[SingleData], axis=0) -> MultiData:
         all_index_map = dict(zip(all_index, range(len(all_index))))
 
         # concat all
-        tmp_data = np.full((len(all_index), len(data_list)), np.NaN)
+        tmp_data = np.full((len(all_index), len(data_list)), np.nan)
         for data_id, index_data in enumerate(data_list):
             assert isinstance(index_data, SingleData)
             now_data_map = [all_index_map[index] for index in index_data.index]
@@ -64,7 +64,7 @@ def sum_by_index(data_list: Union[SingleData], new_index: list, fill_value=0) ->
     new_index : list
         the new_index of new SingleData.
     fill_value : float
-        fill the missing values or replace np.NaN.
+        fill the missing values or replace np.nan.
 
     Returns
     -------
@@ -444,7 +444,7 @@ class IndexData(metaclass=index_data_ops_creator):
         return self.__class__(~self.data.astype(bool), *self.indices)
 
     def abs(self):
-        """get the abs of data except np.NaN."""
+        """get the abs of data except np.nan."""
         tmp_data = np.absolute(self.data)
         return self.__class__(tmp_data, *self.indices)
 
@@ -566,8 +566,8 @@ class SingleData(IndexData):
                 f"The indexes of self and other do not meet the requirements of the four arithmetic operations"
             )
 
-    def reindex(self, index: Index, fill_value=np.NaN) -> SingleData:
-        """reindex data and fill the missing value with np.NaN.
+    def reindex(self, index: Index, fill_value=np.nan) -> SingleData:
+        """reindex data and fill the missing value with np.nan.
 
         Parameters
         ----------
@@ -615,7 +615,7 @@ class SingleData(IndexData):
         return pd.Series(self.data, index=self.index)
 
     def __repr__(self) -> str:
-        return str(pd.Series(self.data, index=self.index))
+        return str(pd.Series(self.data, index=self.index.tolist()))
 
 
 class MultiData(IndexData):
@@ -651,4 +651,4 @@ class MultiData(IndexData):
             )
 
     def __repr__(self) -> str:
-        return str(pd.DataFrame(self.data, index=self.index, columns=self.columns))
+        return str(pd.DataFrame(self.data, index=self.index.tolist(), columns=self.columns.tolist()))

qlib/workflow/cli.py

@@ -7,7 +7,7 @@ import sys
 
 import fire
 from jinja2 import Template, meta
-import ruamel.yaml as yaml
+from ruamel.yaml import YAML
 
 import qlib
 from qlib.config import C
@@ -104,7 +104,8 @@ def workflow(config_path, experiment_name="workflow", uri_folder="mlruns"):
     """
     # Render the template
     rendered_yaml = render_template(config_path)
-    config = yaml.safe_load(rendered_yaml)
+    yaml = YAML(typ="safe", pure=True)
+    config = yaml.load(rendered_yaml)
 
     base_config_path = config.get("BASE_CONFIG_PATH", None)
     if base_config_path:
@@ -126,7 +127,8 @@ def workflow(config_path, experiment_name="workflow", uri_folder="mlruns"):
                 raise FileNotFoundError(f"Can't find the BASE_CONFIG file: {base_config_path}")
 
         with open(path) as fp:
-            base_config = yaml.safe_load(fp)
+            yaml = YAML(typ="safe", pure=True)
+            base_config = yaml.load(fp)
         logger.info(f"Load BASE_CONFIG_PATH succeed: {path.resolve()}")
         config = update_config(base_config, config)
 

qlib/workflow/expm.py

@@ -8,6 +8,7 @@ from mlflow.exceptions import MlflowException, RESOURCE_ALREADY_EXISTS, ErrorCod
 from mlflow.entities import ViewType
 import os
 from typing import Optional, Text
+from pathlib import Path
 
 from .exp import MLflowExperiment, Experiment
 from ..config import C
@@ -233,7 +234,7 @@ class ExpManager:
             # So we supported it in the interface wrapper
             pr = urlparse(self.uri)
             if pr.scheme == "file":
-                with FileLock(os.path.join(pr.netloc, pr.path, "filelock")):  # pylint: disable=E0110
+                with FileLock(Path(os.path.join(pr.netloc, pr.path.lstrip("/"), "filelock"))):  # pylint: disable=E0110
                     return self.create_exp(experiment_name), True
             # NOTE: for other schemes like http, we double check to avoid create exp conflicts
             try:
@@ -421,7 +422,11 @@ class MLflowExpManager(ExpManager):
 
     def list_experiments(self):
         # retrieve all the existing experiments
-        exps = self.client.list_experiments(view_type=ViewType.ACTIVE_ONLY)
+        mlflow_version = int(mlflow.__version__.split(".", maxsplit=1)[0])
+        if mlflow_version >= 2:
+            exps = self.client.search_experiments(view_type=ViewType.ACTIVE_ONLY)
+        else:
+            exps = self.client.list_experiments(view_type=ViewType.ACTIVE_ONLY)  # pylint: disable=E1101
         experiments = dict()
         for exp in exps:
             experiment = MLflowExperiment(exp.experiment_id, exp.name, self.uri)

qlib/workflow/recorder.py

@@ -9,6 +9,7 @@ import shutil
 import pickle
 import tempfile
 import subprocess
+import platform
 from pathlib import Path
 from datetime import datetime
 
@@ -316,7 +317,10 @@ class MLflowRecorder(Recorder):
         This function will return the directory path of this recorder.
         """
         if self.artifact_uri is not None:
-            local_dir_path = Path(self.artifact_uri.lstrip("file:")) / ".."
+            if platform.system() == "Windows":
+                local_dir_path = Path(self.artifact_uri.lstrip("file:").lstrip("/")).parent
+            else:
+                local_dir_path = Path(self.artifact_uri.lstrip("file:")).parent
             local_dir_path = str(local_dir_path.resolve())
             if os.path.isdir(local_dir_path):
                 return local_dir_path