liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-10 22:36:55 +08:00
Code Activity

refactor qlib conf&init. Fix test bug

Browse Source
This commit is contained in:
Young
2020-09-29 11:38:48 +00:00
committed by you-n-g
parent 34ce3ad9bf
commit 5c2f218cfb
5 changed files with 267 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
qlib/data/cache.py
View File

@@ -388,10 +388,7 @@ class DiskExpressionCache(ExpressionCache):
self.r = get_redis_connection()
# remote==True means client is using this module, writing behaviour will not be allowed.
self.remote = kwargs.get("remote", False)
if self.remote:
self.expr_cache_path = os.path.join(C.mount_path, C.features_cache_dir_name)
else:
self.expr_cache_path = os.path.join(C.provider_uri, C.features_cache_dir_name)
self.expr_cache_path = os.path.join(C.get_data_path(), C.features_cache_dir_name)
os.makedirs(self.expr_cache_path, exist_ok=True)
def _uri(self, instrument, field, start_time, end_time, freq):
@@ -562,10 +559,7 @@ class DiskDatasetCache(DatasetCache):
super(DiskDatasetCache, self).__init__(provider)
self.r = get_redis_connection()
self.remote = kwargs.get("remote", False)
if self.remote:
self.dtst_cache_path = os.path.join(C.mount_path, C.dataset_cache_dir_name)
else:
self.dtst_cache_path = os.path.join(C.provider_uri, C.dataset_cache_dir_name)
self.dtst_cache_path = os.path.join(C.get_data_path(), C.dataset_cache_dir_name)
os.makedirs(self.dtst_cache_path, exist_ok=True)
@staticmethod
@@ -1003,7 +997,7 @@ class DatasetURICache(DatasetCache):
# use ClientDatasetProvider
feature_uri = self._uri(instruments, fields, None, None, freq, disk_cache=disk_cache)
value, expire = MemCacheExpire.get_cache(H["f"], feature_uri)
mnt_feature_uri = os.path.join(C.mount_path, C.dataset_cache_dir_name, feature_uri)
mnt_feature_uri = os.path.join(C.get_data_path(), C.dataset_cache_dir_name, feature_uri)
if value is None or expire or not os.path.exists(mnt_feature_uri):
df, uri = self.provider.dataset(
instruments, fields, start_time, end_time, freq, disk_cache, return_uri=True
@@ -1014,7 +1008,7 @@ class DatasetURICache(DatasetCache):
# HZ['f'][uri] = df.copy()
get_module_logger("cache").debug(f"get feature from {C.dataset_provider}")
else:
mnt_feature_uri = os.path.join(C.mount_path, C.dataset_cache_dir_name, feature_uri)
mnt_feature_uri = os.path.join(C.get_data_path(), C.dataset_cache_dir_name, feature_uri)
df = DiskDatasetCache.read_data_from_cache(mnt_feature_uri, start_time, end_time, fields)
get_module_logger("cache").debug("get feature from uri cache")

14
qlib/data/data.py
View File

@@ -502,10 +502,7 @@ class LocalCalendarProvider(CalendarProvider):
@property
def _uri_cal(self):
"""Calendar file uri."""
if self.remote:
return os.path.join(C.mount_path, "calendars", "{}.txt")
else:
return os.path.join(C.provider_uri, "calendars", "{}.txt")
return os.path.join(C.get_data_path(), "calendars", "{}.txt")
def _load_calendar(self, freq, future):
"""Load original calendar timestamp from file.
@@ -568,7 +565,7 @@ class LocalInstrumentProvider(InstrumentProvider):
@property
def _uri_inst(self):
"""Instrument file uri."""
return os.path.join(C.provider_uri, "instruments", "{}.txt")
return os.path.join(C.get_data_path(), "instruments", "{}.txt")
def _load_instruments(self, market):
fname = self._uri_inst.format(market)
@@ -637,10 +634,7 @@ class LocalFeatureProvider(FeatureProvider):
@property
def _uri_data(self):
"""Static feature file uri."""
if self.remote:
return os.path.join(C.mount_path, "features", "{}", "{}.{}.bin")
else:
return os.path.join(C.provider_uri, "features", "{}", "{}.{}.bin")
return os.path.join(C.get_data_path(), "features", "{}", "{}.{}.bin")
def feature(self, instrument, field, start_index, end_index, freq):
# validate
@@ -914,7 +908,7 @@ class ClientDatasetProvider(DatasetProvider):
get_module_logger("data").debug("get result")
try:
# pre-mound nfs, used for demo
mnt_feature_uri = os.path.join(C.mount_path, C.dataset_cache_dir_name, feature_uri)
mnt_feature_uri = os.path.join(C.get_data_path(), C.dataset_cache_dir_name, feature_uri)
df = DiskDatasetCache.read_data_from_cache(mnt_feature_uri, start_time, end_time, fields)
get_module_logger("data").debug("finish slicing data")
if return_uri: