liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-15 08:46:56 +08:00
Code Activity

Fix ZScoreNorm processor bug (#1398)

Browse Source 
* fix_ZScoreNorm_bug

* fix_CI_error

* fix_CI_error

* add_test_processor

* fix_pylint_error

* fix_some_error_and_optimize_code

* modify_terrible_code

* optimize_code

* optimize_code
This commit is contained in:
Linlang
2022-12-30 20:42:37 +08:00
committed by GitHub
parent 667fb0e4d9
commit 756bd0f65b
4 changed files with 105 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
qlib/data/dataset/processor.py
View File

@@ -211,16 +211,19 @@ class MinMaxNorm(Processor):
self.min_val = np.nanmin(df[cols].values, axis=0)
self.max_val = np.nanmax(df[cols].values, axis=0)
self.ignore = self.min_val == self.max_val
# To improve the speed, we set the value of `min_val` to `0` for the columns that do not need to be processed,
# and the value of `max_val` to `1`, when using `(x - min_val) / (max_val - min_val)` for uniform calculation,
# the columns that do not need to be processed will be calculated by `(x - 0) / (1 - 0)`,
# as you can see, the columns that do not need to be processed, will not be affected.
for _i, _con in enumerate(self.ignore):
if _con:
self.min_val[_i] = 0
self.max_val[_i] = 1
self.cols = cols
def __call__(self, df):
def normalize(x, min_val=self.min_val, max_val=self.max_val, ignore=self.ignore):
if (~ignore).all():
return (x - min_val) / (max_val - min_val)
for i in range(ignore.size):
if not ignore[i]:
x[i] = (x[i] - min_val) / (max_val - min_val)
return x
def normalize(x, min_val=self.min_val, max_val=self.max_val):
return (x - min_val) / (max_val - min_val)
df.loc(axis=1)[self.cols] = normalize(df[self.cols].values)
return df
@@ -242,16 +245,19 @@ class ZScoreNorm(Processor):
self.mean_train = np.nanmean(df[cols].values, axis=0)
self.std_train = np.nanstd(df[cols].values, axis=0)
self.ignore = self.std_train == 0
# To improve the speed, we set the value of `std_train` to `1` for the columns that do not need to be processed,
# and the value of `mean_train` to `0`, when using `(x - mean_train) / std_train` for uniform calculation,
# the columns that do not need to be processed will be calculated by `(x - 0) / 1`,
# as you can see, the columns that do not need to be processed, will not be affected.
for _i, _con in enumerate(self.ignore):
if _con:
self.std_train[_i] = 1
self.mean_train[_i] = 0
self.cols = cols
def __call__(self, df):
def normalize(x, mean_train=self.mean_train, std_train=self.std_train, ignore=self.ignore):
if (~ignore).all():
return (x - mean_train) / std_train
for i in range(ignore.size):
if not ignore[i]:
x[i] = (x[i] - mean_train) / std_train
return x
def normalize(x, mean_train=self.mean_train, std_train=self.std_train):
return (x - mean_train) / std_train
df.loc(axis=1)[self.cols] = normalize(df[self.cols].values)
return df
@@ -361,7 +367,7 @@ class CSZFillna(Processor):
def __call__(self, df):
cols = get_group_columns(df, self.fields_group)
df[cols] = df[cols].groupby("datetime").apply(lambda x: x.fillna(x.mean()))
df[cols] = df[cols].groupby("datetime", group_keys=False).apply(lambda x: x.fillna(x.mean()))
return df