mirror of
https://github.com/microsoft/qlib.git
synced 2026-07-11 06:46:56 +08:00
split code into core and contrib for data&model
This commit is contained in:
Young
0
qlib/data/dataset/__init__.py
Normal file
0
qlib/data/dataset/__init__.py
Normal file
@@ -513,73 +513,3 @@ class ConfigQLibDataHandler(QLibDataHandler):
|
||||
if "labels" not in kwargs:
|
||||
kwargs["labels"] = ["Ref($vwap, -2)/Ref($vwap, -1) - 1"]
|
||||
super()._init_kwargs(**kwargs)
|
||||
|
||||
|
||||
class ALPHA360(ConfigQLibDataHandler):
|
||||
config_template = {
|
||||
"price": {"windows": range(60)},
|
||||
"volume": {"windows": range(60)},
|
||||
}
|
||||
|
||||
|
||||
class QLibDataHandlerV1(ConfigQLibDataHandler):
|
||||
config_template = {
|
||||
"kbar": {},
|
||||
"price": {
|
||||
"windows": [0],
|
||||
"feature": ["OPEN", "HIGH", "LOW", "VWAP"],
|
||||
},
|
||||
"rolling": {},
|
||||
}
|
||||
|
||||
def __init__(self, start_date, end_date, processors=None, **kwargs):
|
||||
if processors is None:
|
||||
processors = ["PanelProcessor"] # V1 default processor
|
||||
super().__init__(start_date, end_date, processors, **kwargs)
|
||||
|
||||
def setup_label(self):
|
||||
"""
|
||||
load the labels df
|
||||
:return: df_labels
|
||||
"""
|
||||
TimeInspector.set_time_mark()
|
||||
|
||||
df_labels = super().setup_label()
|
||||
|
||||
## calculate new labels
|
||||
df_labels["LABEL1"] = df_labels["LABEL0"].groupby(level="datetime").apply(lambda x: (x - x.mean()) / x.std())
|
||||
|
||||
df_labels = df_labels.drop(["LABEL0"], axis=1)
|
||||
|
||||
TimeInspector.log_cost_time("Finished loading labels.")
|
||||
|
||||
return df_labels
|
||||
|
||||
|
||||
class Alpha158(QLibDataHandlerV1):
|
||||
config_template = {
|
||||
"kbar": {},
|
||||
"price": {
|
||||
"windows": [0],
|
||||
"feature": ["OPEN", "HIGH", "LOW", "CLOSE"],
|
||||
},
|
||||
"rolling": {},
|
||||
}
|
||||
|
||||
def _init_kwargs(self, **kwargs):
|
||||
kwargs["labels"] = ["Ref($close, -2)/Ref($close, -1) - 1"]
|
||||
super(Alpha158, self)._init_kwargs(**kwargs)
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# import qlib
|
||||
#
|
||||
# qlib.init()
|
||||
#
|
||||
# handler = ALPHA80('2010-01-01', '2018-12-31')
|
||||
# data = handler.get_split_data(
|
||||
# pd.Timestamp('2010-01-01'), pd.Timestamp('2014-01-01'),
|
||||
# pd.Timestamp('2015-01-01'), pd.Timestamp('2016-01-01'),
|
||||
# pd.Timestamp('2017-01-01'), pd.Timestamp('2018-01-01'))
|
||||
# print(data[0])
|
||||
# data[0].to_pickle('alpha80.pkl')
|
||||
|
||||
249
qlib/data/dataset/processor.py
Normal file
249
qlib/data/dataset/processor.py
Normal file
@@ -0,0 +1,249 @@
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import abc
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from ...log import TimeInspector
|
||||
|
||||
EPS = 1e-12
|
||||
|
||||
|
||||
class Processor(abc.ABC):
|
||||
def __init__(self, feature_names, label_names, **kwargs):
|
||||
self.feature_names = feature_names
|
||||
self.label_names = label_names
|
||||
|
||||
@abc.abstractmethod
|
||||
def __call__(self, df_train, df_valid, df_test):
|
||||
pass
|
||||
|
||||
|
||||
class PanelProcessor(Processor):
|
||||
"""Panel Preprocessor"""
|
||||
|
||||
STD_NORM = "Std"
|
||||
MINMAX_NORM = "MinMax"
|
||||
|
||||
def __init__(self, feature_names, label_names, **kwargs):
|
||||
super().__init__(feature_names, label_names)
|
||||
# Options.
|
||||
self.dropna_label = kwargs.get("dropna_label", True)
|
||||
self.dropna_feature = kwargs.get("dropna_feature", False)
|
||||
self.normalize_method = kwargs.get("normalize_method", None)
|
||||
self.replace_inf = kwargs.get("replace_inf_feature", False)
|
||||
|
||||
def __call__(self, df_train, df_valid, df_test):
|
||||
"""
|
||||
Preprocess the data
|
||||
:param df: the dataframe to process data.
|
||||
"""
|
||||
# Drop null labels.
|
||||
if self.dropna_label:
|
||||
df_train, df_valid, df_test = self._process_drop_null_label(df_train, df_valid, df_test)
|
||||
|
||||
# Dropna if need.
|
||||
if self.dropna_feature:
|
||||
df_train, df_valid, df_test = self._process_drop_null_feature(df_train, df_valid, df_test)
|
||||
|
||||
# replace the 'inf' with the mean the corresponding dimension
|
||||
if self.replace_inf:
|
||||
df_train, df_valid, df_test = self._process_replace_inf_feature(df_train, df_valid, df_test)
|
||||
|
||||
# normalize data in given method.
|
||||
if self.normalize_method is not None:
|
||||
df_train, df_valid, df_test = self._process_normalize_feature(df_train, df_valid, df_test)
|
||||
|
||||
return df_train, df_valid, df_test
|
||||
|
||||
def _process_drop_null_label(self, df_train, df_valid, df_test):
|
||||
"""
|
||||
Drop null labels.
|
||||
"""
|
||||
TimeInspector.set_time_mark()
|
||||
df_train = df_train.dropna(subset=self.label_names)
|
||||
df_valid = df_valid.dropna(subset=self.label_names)
|
||||
# The test data's label is Unkown. They can not be seen when preprocessing
|
||||
TimeInspector.log_cost_time("Finished dropping null labels.")
|
||||
|
||||
return df_train, df_valid, df_test
|
||||
|
||||
def _process_drop_null_feature(self, df_train, df_valid, df_test):
|
||||
"""
|
||||
Drop data which contain null features if needed.
|
||||
"""
|
||||
# TODO - `Pandas.dropna` is a low performance method.
|
||||
TimeInspector.set_time_mark()
|
||||
df_train = df_train.dropna(subset=self.feature_names)
|
||||
df_valid = df_valid.dropna(subset=self.feature_names)
|
||||
df_test = df_test.dropna(subset=self.feature_names)
|
||||
TimeInspector.log_cost_time("Finished dropping nan.")
|
||||
|
||||
return df_train, df_valid, df_test
|
||||
|
||||
def _process_replace_inf_feature(self, df_train, df_valid, df_test):
|
||||
"""
|
||||
replace the 'inf' in feature with the mean of this dimension.
|
||||
"""
|
||||
TimeInspector.set_time_mark()
|
||||
|
||||
def replace_inf(data):
|
||||
def process_inf(df):
|
||||
for col in df.columns:
|
||||
df[col] = df[col].replace([np.inf, -np.inf], df[col][~np.isinf(df[col])].mean())
|
||||
return df
|
||||
|
||||
data = data.groupby("datetime").apply(process_inf)
|
||||
data.sort_index(inplace=True)
|
||||
return data
|
||||
|
||||
df_train = replace_inf(df_train)
|
||||
df_valid = replace_inf(df_valid)
|
||||
df_test = replace_inf(df_test)
|
||||
TimeInspector.log_cost_time("Finished replace inf.")
|
||||
|
||||
return df_train, df_valid, df_test
|
||||
|
||||
def _process_normalize_feature(self, df_train, df_valid, df_test):
|
||||
"""
|
||||
Normalize data if needed, we provide two method now: min-max normalization and standard normalization.
|
||||
"""
|
||||
TimeInspector.set_time_mark()
|
||||
|
||||
if self.normalize_method == self.MINMAX_NORM:
|
||||
min_train = np.nanmin(df_train[self.feature_names].values, axis=0)
|
||||
max_train = np.nanmax(df_train[self.feature_names].values, axis=0)
|
||||
ignore = min_train == max_train
|
||||
|
||||
def normalize(x, min_train=min_train, max_train=max_train, ignore=ignore):
|
||||
if (~ignore).all():
|
||||
return (x - min_train) / (max_train - min_train)
|
||||
for i in range(ignore.size):
|
||||
if not ignore[i]:
|
||||
x[i] = (x[i] - min_train) / (max_train - min_train)
|
||||
return x
|
||||
|
||||
elif self.normalize_method == self.STD_NORM:
|
||||
mean_train = np.nanmean(df_train[self.feature_names].values, axis=0)
|
||||
std_train = np.nanstd(df_train[self.feature_names].values, axis=0)
|
||||
ignore = std_train == 0
|
||||
|
||||
def normalize(x, mean_train=mean_train, std_train=std_train, ignore=ignore):
|
||||
if (~ignore).all():
|
||||
return (x - mean_train) / std_train
|
||||
for i in range(ignore.size):
|
||||
if not ignore[i]:
|
||||
x[i] = (x[i] - mean_train) / std_train
|
||||
return x
|
||||
|
||||
else:
|
||||
raise ValueError("Normalize method {} is not allowed".format(self.normalize_method))
|
||||
|
||||
df_train.loc(axis=1)[self.feature_names] = normalize(df_train[self.feature_names].values)
|
||||
df_valid.loc(axis=1)[self.feature_names] = normalize(df_valid[self.feature_names].values)
|
||||
df_test.loc(axis=1)[self.feature_names] = normalize(df_test[self.feature_names].values)
|
||||
|
||||
TimeInspector.log_cost_time("Finished normalizing data.")
|
||||
|
||||
return df_train, df_valid, df_test
|
||||
|
||||
|
||||
class ConfigSectionProcessor(Processor):
|
||||
def __init__(self, feature_names, label_names, **kwargs):
|
||||
super().__init__(feature_names, label_names)
|
||||
# Options
|
||||
self.fillna_feature = kwargs.get("fillna_feature", True)
|
||||
self.fillna_label = kwargs.get("fillna_label", True)
|
||||
self.clip_feature_outlier = kwargs.get("clip_feature_outlier", False)
|
||||
self.shrink_feature_outlier = kwargs.get("shrink_feature_outlier", True)
|
||||
self.clip_label_outlier = kwargs.get("clip_label_outlier", False)
|
||||
|
||||
def __call__(self, *args):
|
||||
return [self._transform(x) for x in args]
|
||||
|
||||
def _transform(self, df):
|
||||
def _label_norm(x):
|
||||
x = x - x.mean() # copy
|
||||
x /= x.std()
|
||||
if self.clip_label_outlier:
|
||||
x.clip(-3, 3, inplace=True)
|
||||
if self.fillna_label:
|
||||
x.fillna(0, inplace=True)
|
||||
return x
|
||||
|
||||
def _feature_norm(x):
|
||||
x = x - x.median() # copy
|
||||
x /= x.abs().median() * 1.4826
|
||||
if self.clip_feature_outlier:
|
||||
x.clip(-3, 3, inplace=True)
|
||||
if self.shrink_feature_outlier:
|
||||
x.where(x <= 3, 3 + (x - 3).div(x.max() - 3) * 0.5, inplace=True)
|
||||
x.where(x >= -3, -3 - (x + 3).div(x.min() + 3) * 0.5, inplace=True)
|
||||
if self.fillna_feature:
|
||||
x.fillna(0, inplace=True)
|
||||
return x
|
||||
|
||||
TimeInspector.set_time_mark()
|
||||
|
||||
# Copy
|
||||
df_new = df.copy()
|
||||
|
||||
# Label
|
||||
cols = df.columns[df.columns.str.contains("^LABEL")]
|
||||
df_new[cols] = df[cols].groupby(level="datetime").apply(_label_norm)
|
||||
|
||||
# Features
|
||||
cols = df.columns[df.columns.str.contains("^KLEN|^KLOW|^KUP")]
|
||||
df_new[cols] = df[cols].apply(lambda x: x ** 0.25).groupby(level="datetime").apply(_feature_norm)
|
||||
|
||||
cols = df.columns[df.columns.str.contains("^KLOW2|^KUP2")]
|
||||
df_new[cols] = df[cols].apply(lambda x: x ** 0.5).groupby(level="datetime").apply(_feature_norm)
|
||||
|
||||
_cols = [
|
||||
"KMID",
|
||||
"KSFT",
|
||||
"OPEN",
|
||||
"HIGH",
|
||||
"LOW",
|
||||
"CLOSE",
|
||||
"VWAP",
|
||||
"ROC",
|
||||
"MA",
|
||||
"BETA",
|
||||
"RESI",
|
||||
"QTLU",
|
||||
"QTLD",
|
||||
"RSV",
|
||||
"SUMP",
|
||||
"SUMN",
|
||||
"SUMD",
|
||||
"VSUMP",
|
||||
"VSUMN",
|
||||
"VSUMD",
|
||||
]
|
||||
pat = "|".join(["^" + x for x in _cols])
|
||||
cols = df.columns[df.columns.str.contains(pat) & (~df.columns.isin(["HIGH0", "LOW0"]))]
|
||||
df_new[cols] = df[cols].groupby(level="datetime").apply(_feature_norm)
|
||||
|
||||
cols = df.columns[df.columns.str.contains("^STD|^VOLUME|^VMA|^VSTD")]
|
||||
df_new[cols] = df[cols].apply(np.log).groupby(level="datetime").apply(_feature_norm)
|
||||
|
||||
cols = df.columns[df.columns.str.contains("^RSQR")]
|
||||
df_new[cols] = df[cols].fillna(0).groupby(level="datetime").apply(_feature_norm)
|
||||
|
||||
cols = df.columns[df.columns.str.contains("^MAX|^HIGH0")]
|
||||
df_new[cols] = df[cols].apply(lambda x: (x - 1) ** 0.5).groupby(level="datetime").apply(_feature_norm)
|
||||
|
||||
cols = df.columns[df.columns.str.contains("^MIN|^LOW0")]
|
||||
df_new[cols] = df[cols].apply(lambda x: (1 - x) ** 0.5).groupby(level="datetime").apply(_feature_norm)
|
||||
|
||||
cols = df.columns[df.columns.str.contains("^CORR|^CORD")]
|
||||
df_new[cols] = df[cols].apply(np.exp).groupby(level="datetime").apply(_feature_norm)
|
||||
|
||||
cols = df.columns[df.columns.str.contains("^WVMA")]
|
||||
df_new[cols] = df[cols].apply(np.log1p).groupby(level="datetime").apply(_feature_norm)
|
||||
|
||||
TimeInspector.log_cost_time("Finished preprocessing data.")
|
||||
|
||||
return df_new
|
||||
Reference in New Issue
Block a user