liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-06 04:20:57 +08:00
Code Activity

Merge branch 'main' into online_srv

Browse Source
This commit is contained in:
lzh222333
2021-03-23 11:20:02 +08:00
committed by GitHub
parent 84d5318bda 2b74b4dfa4
commit d66d4ec93d
74 changed files with 3035 additions and 1020 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
qlib/data/cache.py
View File

@@ -1045,9 +1045,6 @@ class SimpleDatasetCache(DatasetCache):
class DatasetURICache(DatasetCache):
"""Prepared cache mechanism for server."""
def __init__(self, provider):
super(DatasetURICache, self).__init__(provider)
def _uri(self, instruments, fields, start_time, end_time, freq, disk_cache=1, **kwargs):
return hash_args(*self.normalize_uri_args(instruments, fields, freq), disk_cache)

3
qlib/data/data.py
View File

@@ -654,9 +654,6 @@ class LocalExpressionProvider(ExpressionProvider):
Provide expression data from local data source.
"""
def __init__(self):
super().__init__()
def expression(self, instrument, field, start_time=None, end_time=None, freq="day"):
expression = self.get_expression_instance(field)
start_time = pd.Timestamp(start_time)

29
qlib/data/dataset/__init__.py
View File

@@ -1,5 +1,5 @@
from ...utils.serial import Serializable
from typing import Union, List, Tuple
from typing import Union, List, Tuple, Dict, Text, Optional
from ...utils import init_instance_by_config, np_ffill
from ...log import get_module_logger
from .handler import DataHandler, DataHandlerLP
@@ -76,17 +76,6 @@ class DatasetH(Dataset):
- The processing is related to data split.
"""
def __init__(self, handler: Union[dict, DataHandler], segments: dict):
"""
Parameters
----------
handler : Union[dict, DataHandler]
handler will be passed into setup_data.
segments : dict
handler will be passed into setup_data.
"""
super().__init__(handler, segments)
def init(self, handler_kwargs: dict = None, segment_kwargs: dict = None):
"""
Initialize the DatasetH
@@ -124,7 +113,7 @@ class DatasetH(Dataset):
raise TypeError(f"param handler_kwargs must be type dict, not {type(segment_kwargs)}")
self.segments = segment_kwargs.copy()
def setup_data(self, handler: Union[dict, DataHandler], segments: dict):
def setup_data(self, handler: Union[Dict, DataHandler], segments: Dict[Text, Tuple]):
"""
Setup the underlying data.
@@ -156,6 +145,11 @@ class DatasetH(Dataset):
self.handler = init_instance_by_config(handler, accept_types=DataHandler)
self.segments = segments.copy()
def __repr__(self):
return "{name}(handler={handler}, segments={segments})".format(
name=self.__class__.__name__, handler=self.handler, segments=self.segments
)
def _prepare_seg(self, slc: slice, **kwargs):
"""
Give a slice, retrieve the according data
@@ -168,7 +162,7 @@ class DatasetH(Dataset):
def prepare(
self,
segments: Union[List[str], Tuple[str], str, slice],
segments: Union[List[Text], Tuple[Text], Text, slice],
col_set=DataHandler.CS_ALL,
data_key=DataHandlerLP.DK_I,
**kwargs,
@@ -178,7 +172,7 @@ class DatasetH(Dataset):
Parameters
----------
segments : Union[List[str], Tuple[str], str, slice]
segments : Union[List[Text], Tuple[Text], Text, slice]
Describe the scope of the data to be prepared
Here are some examples:
@@ -265,7 +259,7 @@ class TSDataSampler:
self.fillna_type = fillna_type
assert get_level_index(data, "datetime") == 0
self.data = lazy_sort_index(data)
self.data_arr = np.array(self.data) # Get index from numpy.array will much faster than DataFrame.values! But
self.data_arr = np.array(self.data) # Get index from numpy.array will much faster than DataFrame.values!
# NOTE: append last line with full NaN for better performance in `__getitem__`
self.data_arr = np.append(self.data_arr, np.full((1, self.data_arr.shape[1]), np.nan), axis=0)
self.nan_idx = -1 # The last line is all NaN
@@ -273,7 +267,6 @@ class TSDataSampler:
# the data type will be changed
# The index of usable data is between start_idx and end_idx
self.start_idx, self.end_idx = self.data.index.slice_locs(start=pd.Timestamp(start), end=pd.Timestamp(end))
# self.index_link = self.build_link(self.data)
self.idx_df, self.idx_map = self.build_index(self.data)
self.idx_arr = np.array(self.idx_df.values, dtype=np.float64) # for better performance
@@ -408,7 +401,7 @@ class TSDataSampler:
# 1) for better performance, use the last nan line for padding the lost date
# 2) In case of precision problems. We use np.float64. # TODO: I'm not sure if whether np.float64 will result in
# precision problems. It will not cause any problems in my tests at least
indices = np.nan_to_num(indices.astype(np.float64), nan=self.nan_idx).astype(np.int)
indices = np.nan_to_num(indices.astype(np.float64), nan=self.nan_idx).astype(int)
data = self.data_arr[indices]
if isinstance(idx, mtit):

6
qlib/data/dataset/handler.py
View File

@@ -35,7 +35,7 @@ class DataHandler(Serializable):
The data handler try to maintain a handler with 2 level.
`datetime` & `instruments`.
Any order of the index level can be suported(The order will implied in the data).
Any order of the index level can be suported (The order will be implied in the data).
The order <`datetime`, `instruments`> will be used when the dataframe index name is missed.
Example of the data:
@@ -47,8 +47,8 @@ class DataHandler(Serializable):
$close $volume Ref($close, 1) Mean($close, 3) $high-$low LABEL0
datetime instrument
2010-01-04 SH600000 81.807068 17145150.0 83.737389 83.016739 2.741058 0.0032
SH600004 13.313329 11800983.0 13.313329 13.317701 0.183632 0.0042
SH600005 37.796539 12231662.0 38.258602 37.919757 0.970325 0.0289
SH600004 13.313329 11800983.0 13.313329 13.317701 0.183632 0.0042
SH600005 37.796539 12231662.0 38.258602 37.919757 0.970325 0.0289
Tips for improving the performance of datahandler

9
qlib/data/ops.py
View File

@@ -74,7 +74,6 @@ class NpElemOperator(ElemOperator):
"""
def __init__(self, feature, func):
self.feature = feature
self.func = func
super(NpElemOperator, self).__init__(feature)
@@ -289,8 +288,6 @@ class NpPairOperator(PairOperator):
"""
def __init__(self, feature_left, feature_right, func):
self.feature_left = feature_left
self.feature_right = feature_right
self.func = func
super(NpPairOperator, self).__init__(feature_left, feature_right)
@@ -1182,7 +1179,7 @@ class Slope(Rolling):
Returns
----------
Expression
a feature instance with regression slope of given window
a feature instance with linear regression slope of given window
"""
def __init__(self, feature, N):
@@ -1210,7 +1207,7 @@ class Rsquare(Rolling):
Returns
----------
Expression
a feature instance with regression r-value square of given window
a feature instance with linear regression r-value square of given window
"""
def __init__(self, feature, N):
@@ -1489,7 +1486,7 @@ OpsList = [
]
class OpsWrapper(object):
class OpsWrapper:
"""Ops Wrapper"""
def __init__(self):