Fix backtest (#719)

* modify FileStorage to support multiple freqs

* modify backtest's sample documentation

* change the logging level of read data exception from error to debug

* fix the backtest exception when volume is 0 or np.nan

* fix test_storage.py

* add backtest_daily

* modify backtest_daily's docstring

* add __repr__/__str__ to Position

* fix the bug of nested_decision_execution example

Co-authored-by: Young <afe.young@gmail.com>
Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>

qlib/data/base.py

@@ -155,7 +155,7 @@ class Expression(abc.ABC):
         try:
             series = self._load_internal(instrument, start_index, end_index, freq)
         except Exception as e:
-            get_module_logger("data").error(
+            get_module_logger("data").debug(
                 f"Loading data error: instrument={instrument}, expression={str(self)}, "
                 f"start_index={start_index}, end_index={end_index}, freq={freq}. "
                 f"error info: {str(e)}"

qlib/data/data.py

@@ -58,34 +58,13 @@ class ProviderBackendMixin:
         backend = copy.deepcopy(backend)
 
         # set default storage kwargs
+        # NOTE: provider_uri priority：
+        #   1. backend_config: backend_obj["kwargs"]["provider_uri"]
+        #   2. qlib.init: provider_uri
         backend_kwargs = backend.setdefault("kwargs", {})
-        # default provider_uri map
-        if "provider_uri" not in backend_kwargs:
-            # if the user has no uri configured, use: uri = uri_map[freq]
-            # NOTE: provider_uri priority：
-            #   1. backend_config: backend_obj["kwargs"]["provider_uri"]
-            #   2. backend_config: backend_obj["kwargs"]["provider_uri_map"]
-            #   3. qlib.init: provider_uri
-            provider_uri_map = backend_kwargs.setdefault("provider_uri_map", {})
-            freq = kwargs.get("freq", "day")
-            if freq not in provider_uri_map:
-                # NOTE: uri
-                #   1. If `freq` in C.dpm.provider_uri.keys(), uri = C.dpm.provider_uri[freq]
-                #   2. If `freq` not in C.dpm.provider_uri.keys()
-                #       - Get the `min_freq` closest to `freq` from C.dpm.provider_uri.keys(), uri = C.dpm.provider_uri[min_freq]
-                # NOTE: In Storage, only CalendarStorage is supported
-                #   1. If `uri` does not exist
-                #       - Get the `min_uri` of the closest `freq` under the same "directory" as the `uri`
-                #       - Read data from `min_uri` and resample to `freq`
-                try:
-                    _uri = C.dpm.get_data_uri(freq)
-                except KeyError:
-                    # provider_uri is dict and freq not in list(provider_uri.keys())
-                    # use the nearest freq greater than 0
-                    min_freq = Freq.get_recent_freq(freq, C.dpm.provider_uri.keys())
-                    _uri = C.dpm.get_data_uri(freq) if min_freq is None else C.dpm.get_data_uri(min_freq)
-                provider_uri_map[freq] = _uri
-            backend_kwargs["provider_uri"] = provider_uri_map[freq]
+        provider_uri = backend_kwargs.get("provider_uri", None)
+        provider_uri = C.dpm.provider_uri if provider_uri is None else C.dpm.format_provider_uri(provider_uri)
+        backend_kwargs["provider_uri"] = provider_uri
         backend.setdefault("kwargs", {}).update(**kwargs)
         return init_instance_by_config(backend)
 
@@ -730,7 +709,7 @@ class LocalExpressionProvider(ExpressionProvider):
         try:
             series = expression.load(instrument, max(0, start_index - lft_etd), end_index + rght_etd, freq)
         except Exception as e:
-            get_module_logger("data").error(
+            get_module_logger("data").debug(
                 f"Loading expression error: "
                 f"instrument={instrument}, field=({field}), start_time={start_time}, end_time={end_time}, freq={freq}. "
                 f"error info: {str(e)}"

qlib/data/ops.py

@@ -324,11 +324,11 @@ class NpPairOperator(PairOperator):
         try:
             res = getattr(np, self.func)(series_left, series_right)
         except ValueError as e:
-            get_module_logger("ops").error(warning_info)
+            get_module_logger("ops").debug(warning_info)
             raise ValueError(f"{str(e)}. \n\t{warning_info}")
         else:
-            if check_length and len(series_left) != len(series_right) and C.ops_warning_log:
-                get_module_logger("ops").warning(warning_info)
+            if check_length and len(series_left) != len(series_right):
+                get_module_logger("ops").debug(warning_info)
         return res
 
 

qlib/data/storage/file_storage.py

@@ -10,23 +10,44 @@ import pandas as pd
 
 from qlib.utils.time import Freq
 from qlib.utils.resam import resam_calendar
+from qlib.config import C
 from qlib.log import get_module_logger
 from qlib.data.storage import CalendarStorage, InstrumentStorage, FeatureStorage, CalVT, InstKT, InstVT
-from qlib.data.cache import H
 
 logger = get_module_logger("file_storage")
 
 
 class FileStorageMixin:
+    """FileStorageMixin, applicable to FileXXXStorage
+    Subclasses need to have provider_uri, freq, storage_name, file_name attributes
+
+    """
+
+    @property
+    def dpm(self):
+        return C.DataPathManager(self.provider_uri, None)
+
+    @property
+    def support_freq(self) -> List[str]:
+        _v = "_support_freq"
+        if hasattr(self, _v):
+            return getattr(self, _v)
+        if len(self.provider_uri) == 1 and C.DEFAULT_FREQ in self.provider_uri:
+            freq = filter(
+                lambda _freq: not _freq.endswith("_future"),
+                map(lambda x: x.stem, self.dpm.get_data_uri(C.DEFAULT_FREQ).joinpath("calendars").glob("*.txt")),
+            )
+        else:
+            freq = self.provider_uri.keys()
+        freq = list(freq)
+        setattr(self, _v, freq)
+        return freq
+
     @property
     def uri(self) -> Path:
-        _provider_uri = self.kwargs.get("provider_uri", None)
-        if _provider_uri is None:
-            raise ValueError(
-                f"The `provider_uri` parameter is not found in {self.__class__.__name__}, "
-                f'please specify `provider_uri` in the "provider\'s backend"'
-            )
-        return Path(_provider_uri).expanduser().joinpath(f"{self.storage_name}s", self.file_name)
+        if self.freq not in self.support_freq:
+            raise ValueError(f"{self.storage_name}: {self.provider_uri} does not contain data for {self.freq}")
+        return self.dpm.get_data_uri(self.freq).joinpath(f"{self.storage_name}s", self.file_name)
 
     def check(self):
         """check self.uri
@@ -40,10 +61,19 @@ class FileStorageMixin:
 
 
 class FileCalendarStorage(FileStorageMixin, CalendarStorage):
-    def __init__(self, freq: str, future: bool, **kwargs):
+    def __init__(self, freq: str, future: bool, provider_uri: dict, **kwargs):
         super(FileCalendarStorage, self).__init__(freq, future, **kwargs)
         self.future = future
-        self.file_name = f"{freq}_future.txt" if future else f"{freq}.txt".lower()
+        self.provider_uri = C.DataPathManager.format_provider_uri(provider_uri)
+        self.resample_freq = None
+
+    @property
+    def file_name(self) -> str:
+        return f"{self.use_freq}_future.txt" if self.future else f"{self.use_freq}.txt".lower()
+
+    @property
+    def use_freq(self) -> str:
+        return self.freq if self.resample_freq is None else self.resample_freq
 
     def _read_calendar(self, skip_rows: int = 0, n_rows: int = None) -> List[CalVT]:
         if not self.uri.exists():
@@ -59,28 +89,26 @@ class FileCalendarStorage(FileStorageMixin, CalendarStorage):
             np.savetxt(fp, values, fmt="%s", encoding="utf-8")
 
     @property
-    def data(self) -> List[CalVT]:
-        # NOTE: uri
-        #   1. If `uri` does not exist
-        #       - Get the `min_uri` of the closest `freq` under the same "directory" as the `uri`
-        #       - Read data from `min_uri` and resample to `freq`
-        try:
-            self.check()
-            _calendar = self._read_calendar()
-        except ValueError:
-            freq_list = self._get_storage_freq()
-            _freq = Freq.get_recent_freq(self.freq, freq_list)
-            if _freq is None:
-                raise ValueError(f"can't find a freq from {freq_list} that can resample to {self.freq}!")
-            self.file_name = f"{_freq}_future.txt" if self.future else f"{_freq}.txt".lower()
-            # The cache is useful for the following cases
-            # - multiple frequencies are sampled from the same calendar
-            cache_key = self.uri
-            if cache_key not in H["c"]:
-                H["c"][cache_key] = self._read_calendar()
-            _calendar = H["c"][cache_key]
-            _calendar = resam_calendar(np.array(list(map(pd.Timestamp, _calendar))), _freq, self.freq)
+    def uri(self) -> Path:
+        freq = self.freq
+        if freq not in self.support_freq:
+            # NOTE: uri
+            #   1. If `uri` does not exist
+            #       - Get the `min_uri` of the closest `freq` under the same "directory" as the `uri`
+            #       - Read data from `min_uri` and resample to `freq`
 
+            freq = Freq.get_recent_freq(freq, self.support_freq)
+            if freq is None:
+                raise ValueError(f"can't find a freq from {self.support_freq} that can resample to {self.freq}!")
+            self.resample_freq = freq
+        return self.dpm.get_data_uri(self.use_freq).joinpath(f"{self.storage_name}s", self.file_name)
+
+    @property
+    def data(self) -> List[CalVT]:
+        self.check()
+        _calendar = self._read_calendar()
+        if self.resample_freq is not None:
+            _calendar = resam_calendar(np.array(list(map(pd.Timestamp, _calendar))), self.resample_freq, self.freq)
         return _calendar
 
     def _get_storage_freq(self) -> List[str]:
@@ -135,8 +163,9 @@ class FileInstrumentStorage(FileStorageMixin, InstrumentStorage):
     INSTRUMENT_END_FIELD = "end_datetime"
     SYMBOL_FIELD_NAME = "instrument"
 
-    def __init__(self, market: str, **kwargs):
-        super(FileInstrumentStorage, self).__init__(market, **kwargs)
+    def __init__(self, market: str, freq: str, provider_uri: dict, **kwargs):
+        super(FileInstrumentStorage, self).__init__(market, freq, **kwargs)
+        self.provider_uri = C.DataPathManager.format_provider_uri(provider_uri)
         self.file_name = f"{market.lower()}.txt"
 
     def _read_instrument(self) -> Dict[InstKT, InstVT]:
@@ -223,8 +252,9 @@ class FileInstrumentStorage(FileStorageMixin, InstrumentStorage):
 
 
 class FileFeatureStorage(FileStorageMixin, FeatureStorage):
-    def __init__(self, instrument: str, field: str, freq: str, **kwargs):
+    def __init__(self, instrument: str, field: str, freq: str, provider_uri: dict, **kwargs):
         super(FileFeatureStorage, self).__init__(instrument, field, freq, **kwargs)
+        self.provider_uri = C.DataPathManager.format_provider_uri(provider_uri)
         self.file_name = f"{instrument.lower()}/{field.lower()}.{freq.lower()}.bin"
 
     def clear(self):

qlib/data/storage/storage.py

@@ -195,8 +195,9 @@ class CalendarStorage(BaseStorage):
 
 
 class InstrumentStorage(BaseStorage):
-    def __init__(self, market: str, **kwargs):
+    def __init__(self, market: str, freq: str, **kwargs):
         self.market = market
+        self.freq = freq
         self.kwargs = kwargs
 
     @property