liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-10 22:36:55 +08:00
Code Activity

Add files via upload

Browse Source
This commit is contained in:
Wendi Li
2020-12-09 12:07:59 +08:00
committed by you-n-g
parent 1bbd026195
commit 15cdfeb121
1 changed files with 229 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

448
examples/benchmarks/TFT/data_formatters/qlib_Alpha158.py
View File

@@ -1,219 +1,229 @@
# coding=utf-8 # coding=utf-8
# Copyright 2020 The Google Research Authors. # Copyright 2020 The Google Research Authors.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# Lint as: python3 # Lint as: python3
"""Custom formatting functions for Alpha158 dataset. """Custom formatting functions for Alpha158 dataset.
Defines dataset specific column definitions and data transformations. Defines dataset specific column definitions and data transformations.
""" """
import data_formatters.base import data_formatters.base
import libs.utils as utils import libs.utils as utils
import sklearn.preprocessing import sklearn.preprocessing
GenericDataFormatter = data_formatters.base.GenericDataFormatter GenericDataFormatter = data_formatters.base.GenericDataFormatter
DataTypes = data_formatters.base.DataTypes DataTypes = data_formatters.base.DataTypes
InputTypes = data_formatters.base.InputTypes InputTypes = data_formatters.base.InputTypes
class Alpha158Formatter(GenericDataFormatter): class Alpha158Formatter(GenericDataFormatter):
"""Defines and formats data for the Alpha158 dataset. """Defines and formats data for the Alpha158 dataset.
Attributes: Attributes:
column_definition: Defines input and data type of column used in the column_definition: Defines input and data type of column used in the
experiment. experiment.
identifiers: Entity identifiers used in experiments. identifiers: Entity identifiers used in experiments.
""" """
_column_definition = [ _column_definition = [
("instrument", DataTypes.CATEGORICAL, InputTypes.ID), ("instrument", DataTypes.CATEGORICAL, InputTypes.ID),
("LABEL0", DataTypes.REAL_VALUED, InputTypes.TARGET), ("LABEL0", DataTypes.REAL_VALUED, InputTypes.TARGET),
("date", DataTypes.DATE, InputTypes.TIME), ("date", DataTypes.DATE, InputTypes.TIME),
("month", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT), ("month", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
("day_of_week", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT), ("day_of_week", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
# Selected 10 features # Selected 10 features
("RESI5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT), ("RESI5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
("WVMA5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT), ("WVMA5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
("RSQR5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT), ("RSQR5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
("KLEN", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT), ("KLEN", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
("RSQR10", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT), ("RSQR10", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
("CORR5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT), ("CORR5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
("CORD5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT), ("CORD5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
("CORR10", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT), ("CORR10", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
("ROC60", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT), ("ROC60", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
("RESI10", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT), ("RESI10", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
("const", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT), ("VSTD5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
] ("RSQR60", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
("CORR60", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
def __init__(self): ("WVMA60", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
"""Initialises formatter.""" ("STD5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
("RSQR20", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
self.identifiers = None ("CORD60", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
self._real_scalers = None ("CORD10", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
self._cat_scalers = None ("CORR20", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
self._target_scaler = None ("KLOW", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
self._num_classes_per_cat_input = None ("const", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
]
def split_data(self, df, valid_boundary=2016, test_boundary=2018):
"""Splits data frame into training-validation-test data frames. def __init__(self):
"""Initialises formatter."""
This also calibrates scaling object, and transforms data for each split.
self.identifiers = None
Args: self._real_scalers = None
df: Source data frame to split. self._cat_scalers = None
valid_boundary: Starting year for validation data self._target_scaler = None
test_boundary: Starting year for test data self._num_classes_per_cat_input = None
Returns: def split_data(self, df, valid_boundary=2016, test_boundary=2018):
Tuple of transformed (train, valid, test) data. """Splits data frame into training-validation-test data frames.
"""
This also calibrates scaling object, and transforms data for each split.
print("Formatting train-valid-test splits.")
Args:
index = df["year"] df: Source data frame to split.
train = df.loc[index < valid_boundary] valid_boundary: Starting year for validation data
valid = df.loc[(index >= valid_boundary) & (index < test_boundary)] test_boundary: Starting year for test data
test = df.loc[index >= test_boundary]
Returns:
self.set_scalers(train) Tuple of transformed (train, valid, test) data.
"""
return (self.transform_inputs(data) for data in [train, valid, test])
print("Formatting train-valid-test splits.")
def set_scalers(self, df):
"""Calibrates scalers using the data supplied. index = df["year"]
train = df.loc[index < valid_boundary]
Args: valid = df.loc[(index >= valid_boundary) & (index < test_boundary)]
df: Data to use to calibrate scalers. test = df.loc[index >= test_boundary]
"""
print("Setting scalers with training data...") self.set_scalers(train)
column_definitions = self.get_column_definition() return (self.transform_inputs(data) for data in [train, valid, test])
id_column = utils.get_single_col_by_input_type(InputTypes.ID, column_definitions)
target_column = utils.get_single_col_by_input_type(InputTypes.TARGET, column_definitions) def set_scalers(self, df):
"""Calibrates scalers using the data supplied.
# Extract identifiers in case required
self.identifiers = list(df[id_column].unique()) Args:
df: Data to use to calibrate scalers.
# Format real scalers """
real_inputs = utils.extract_cols_from_data_type( print("Setting scalers with training data...")
DataTypes.REAL_VALUED, column_definitions, {InputTypes.ID, InputTypes.TIME}
) column_definitions = self.get_column_definition()
id_column = utils.get_single_col_by_input_type(InputTypes.ID, column_definitions)
data = df[real_inputs].values target_column = utils.get_single_col_by_input_type(InputTypes.TARGET, column_definitions)
self._real_scalers = sklearn.preprocessing.StandardScaler().fit(data)
self._target_scaler = sklearn.preprocessing.StandardScaler().fit( # Extract identifiers in case required
df[[target_column]].values self.identifiers = list(df[id_column].unique())
) # used for predictions
# Format real scalers
# Format categorical scalers real_inputs = utils.extract_cols_from_data_type(
categorical_inputs = utils.extract_cols_from_data_type( DataTypes.REAL_VALUED, column_definitions, {InputTypes.ID, InputTypes.TIME}
DataTypes.CATEGORICAL, column_definitions, {InputTypes.ID, InputTypes.TIME} )
)
data = df[real_inputs].values
categorical_scalers = {} self._real_scalers = sklearn.preprocessing.StandardScaler().fit(data)
num_classes = [] self._target_scaler = sklearn.preprocessing.StandardScaler().fit(
for col in categorical_inputs: df[[target_column]].values
# Set all to str so that we don't have mixed integer/string columns ) # used for predictions
srs = df[col].apply(str)
categorical_scalers[col] = sklearn.preprocessing.LabelEncoder().fit(srs.values) # Format categorical scalers
num_classes.append(srs.nunique()) categorical_inputs = utils.extract_cols_from_data_type(
DataTypes.CATEGORICAL, column_definitions, {InputTypes.ID, InputTypes.TIME}
# Set categorical scaler outputs )
self._cat_scalers = categorical_scalers
self._num_classes_per_cat_input = num_classes categorical_scalers = {}
num_classes = []
def transform_inputs(self, df): for col in categorical_inputs:
"""Performs feature transformations. # Set all to str so that we don't have mixed integer/string columns
srs = df[col].apply(str)
This includes both feature engineering, preprocessing and normalisation. categorical_scalers[col] = sklearn.preprocessing.LabelEncoder().fit(srs.values)
num_classes.append(srs.nunique())
Args:
df: Data frame to transform. # Set categorical scaler outputs
self._cat_scalers = categorical_scalers
Returns: self._num_classes_per_cat_input = num_classes
Transformed data frame.
def transform_inputs(self, df):
""" """Performs feature transformations.
output = df.copy()
This includes both feature engineering, preprocessing and normalisation.
if self._real_scalers is None and self._cat_scalers is None:
raise ValueError("Scalers have not been set!") Args:
df: Data frame to transform.
column_definitions = self.get_column_definition()
Returns:
real_inputs = utils.extract_cols_from_data_type( Transformed data frame.
DataTypes.REAL_VALUED, column_definitions, {InputTypes.ID, InputTypes.TIME}
) """
categorical_inputs = utils.extract_cols_from_data_type( output = df.copy()
DataTypes.CATEGORICAL, column_definitions, {InputTypes.ID, InputTypes.TIME}
) if self._real_scalers is None and self._cat_scalers is None:
raise ValueError("Scalers have not been set!")
# Format real inputs
output[real_inputs] = self._real_scalers.transform(df[real_inputs].values) column_definitions = self.get_column_definition()
# Format categorical inputs real_inputs = utils.extract_cols_from_data_type(
for col in categorical_inputs: DataTypes.REAL_VALUED, column_definitions, {InputTypes.ID, InputTypes.TIME}
string_df = df[col].apply(str) )
output[col] = self._cat_scalers[col].transform(string_df) categorical_inputs = utils.extract_cols_from_data_type(
DataTypes.CATEGORICAL, column_definitions, {InputTypes.ID, InputTypes.TIME}
return output )
def format_predictions(self, predictions): # Format real inputs
"""Reverts any normalisation to give predictions in original scale. output[real_inputs] = self._real_scalers.transform(df[real_inputs].values)
Args: # Format categorical inputs
predictions: Dataframe of model predictions. for col in categorical_inputs:
string_df = df[col].apply(str)
Returns: output[col] = self._cat_scalers[col].transform(string_df)
Data frame of unnormalised predictions.
""" return output
output = predictions.copy()
def format_predictions(self, predictions):
column_names = predictions.columns """Reverts any normalisation to give predictions in original scale.
for col in column_names: Args:
if col not in {"forecast_time", "identifier"}: predictions: Dataframe of model predictions.
output[col] = self._target_scaler.inverse_transform(predictions[col])
Returns:
return output Data frame of unnormalised predictions.
"""
# Default params output = predictions.copy()
def get_fixed_params(self):
"""Returns fixed model parameters for experiments.""" column_names = predictions.columns
fixed_params = { for col in column_names:
"total_time_steps": 6 + 6, if col not in {"forecast_time", "identifier"}:
"num_encoder_steps": 6, output[col] = self._target_scaler.inverse_transform(predictions[col])
"num_epochs": 100,
"early_stopping_patience": 10, return output
"multiprocessing_workers": 5,
} # Default params
def get_fixed_params(self):
return fixed_params """Returns fixed model parameters for experiments."""
def get_default_model_params(self): fixed_params = {
"""Returns default optimised model parameters.""" "total_time_steps": 6 + 6,
"num_encoder_steps": 6,
model_params = { "num_epochs": 100,
"dropout_rate": 0.4, "early_stopping_patience": 10,
"hidden_layer_size": 160, "multiprocessing_workers": 5,
"learning_rate": 0.0001, }
"minibatch_size": 128,
"max_gradient_norm": 0.0135, return fixed_params
"num_heads": 1,
"stack_size": 1, def get_default_model_params(self):
} """Returns default optimised model parameters."""
return model_params model_params = {
"dropout_rate": 0.4,
"hidden_layer_size": 160,
"learning_rate": 0.0001,
"minibatch_size": 128,
"max_gradient_norm": 0.0135,
"num_heads": 1,
"stack_size": 1,
}
return model_params