mirror of
https://github.com/microsoft/qlib.git
synced 2026-07-17 17:34:35 +08:00
Format TFT
This commit is contained in:
Wendi Li
@@ -1,15 +1,14 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
# Copyright 2020 The Google Research Authors.
|
# Copyright 2020 The Google Research Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
#
|
#
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
#
|
#
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
|||||||
@@ -1,235 +1,223 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
# Copyright 2020 The Google Research Authors.
|
# Copyright 2020 The Google Research Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
#
|
#
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
#
|
#
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Lint as: python3
|
# Lint as: python3
|
||||||
"""Default data formatting functions for experiments.
|
"""Default data formatting functions for experiments.
|
||||||
|
|
||||||
For new datasets, inherit form GenericDataFormatter and implement
|
For new datasets, inherit form GenericDataFormatter and implement
|
||||||
all abstract functions.
|
all abstract functions.
|
||||||
|
|
||||||
These dataset-specific methods:
|
These dataset-specific methods:
|
||||||
1) Define the column and input types for tabular dataframes used by model
|
1) Define the column and input types for tabular dataframes used by model
|
||||||
2) Perform the necessary input feature engineering & normalisation steps
|
2) Perform the necessary input feature engineering & normalisation steps
|
||||||
3) Reverts the normalisation for predictions
|
3) Reverts the normalisation for predictions
|
||||||
4) Are responsible for train, validation and test splits
|
4) Are responsible for train, validation and test splits
|
||||||
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import abc
|
import abc
|
||||||
import enum
|
import enum
|
||||||
|
|
||||||
|
|
||||||
# Type defintions
|
# Type defintions
|
||||||
class DataTypes(enum.IntEnum):
|
class DataTypes(enum.IntEnum):
|
||||||
"""Defines numerical types of each column."""
|
"""Defines numerical types of each column."""
|
||||||
REAL_VALUED = 0
|
|
||||||
CATEGORICAL = 1
|
REAL_VALUED = 0
|
||||||
DATE = 2
|
CATEGORICAL = 1
|
||||||
|
DATE = 2
|
||||||
|
|
||||||
class InputTypes(enum.IntEnum):
|
|
||||||
"""Defines input types of each column."""
|
class InputTypes(enum.IntEnum):
|
||||||
TARGET = 0
|
"""Defines input types of each column."""
|
||||||
OBSERVED_INPUT = 1
|
|
||||||
KNOWN_INPUT = 2
|
TARGET = 0
|
||||||
STATIC_INPUT = 3
|
OBSERVED_INPUT = 1
|
||||||
ID = 4 # Single column used as an entity identifier
|
KNOWN_INPUT = 2
|
||||||
TIME = 5 # Single column exclusively used as a time index
|
STATIC_INPUT = 3
|
||||||
|
ID = 4 # Single column used as an entity identifier
|
||||||
|
TIME = 5 # Single column exclusively used as a time index
|
||||||
class GenericDataFormatter(abc.ABC):
|
|
||||||
"""Abstract base class for all data formatters.
|
|
||||||
|
class GenericDataFormatter(abc.ABC):
|
||||||
User can implement the abstract methods below to perform dataset-specific
|
"""Abstract base class for all data formatters.
|
||||||
manipulations.
|
|
||||||
|
User can implement the abstract methods below to perform dataset-specific
|
||||||
"""
|
manipulations.
|
||||||
|
|
||||||
@abc.abstractmethod
|
"""
|
||||||
def set_scalers(self, df):
|
|
||||||
"""Calibrates scalers using the data supplied."""
|
@abc.abstractmethod
|
||||||
raise NotImplementedError()
|
def set_scalers(self, df):
|
||||||
|
"""Calibrates scalers using the data supplied."""
|
||||||
@abc.abstractmethod
|
raise NotImplementedError()
|
||||||
def transform_inputs(self, df):
|
|
||||||
"""Performs feature transformation."""
|
@abc.abstractmethod
|
||||||
raise NotImplementedError()
|
def transform_inputs(self, df):
|
||||||
|
"""Performs feature transformation."""
|
||||||
@abc.abstractmethod
|
raise NotImplementedError()
|
||||||
def format_predictions(self, df):
|
|
||||||
"""Reverts any normalisation to give predictions in original scale."""
|
@abc.abstractmethod
|
||||||
raise NotImplementedError()
|
def format_predictions(self, df):
|
||||||
|
"""Reverts any normalisation to give predictions in original scale."""
|
||||||
@abc.abstractmethod
|
raise NotImplementedError()
|
||||||
def split_data(self, df):
|
|
||||||
"""Performs the default train, validation and test splits."""
|
@abc.abstractmethod
|
||||||
raise NotImplementedError()
|
def split_data(self, df):
|
||||||
|
"""Performs the default train, validation and test splits."""
|
||||||
@property
|
raise NotImplementedError()
|
||||||
@abc.abstractmethod
|
|
||||||
def _column_definition(self):
|
@property
|
||||||
"""Defines order, input type and data type of each column."""
|
@abc.abstractmethod
|
||||||
raise NotImplementedError()
|
def _column_definition(self):
|
||||||
|
"""Defines order, input type and data type of each column."""
|
||||||
@abc.abstractmethod
|
raise NotImplementedError()
|
||||||
def get_fixed_params(self):
|
|
||||||
"""Defines the fixed parameters used by the model for training.
|
@abc.abstractmethod
|
||||||
|
def get_fixed_params(self):
|
||||||
Requires the following keys:
|
"""Defines the fixed parameters used by the model for training.
|
||||||
'total_time_steps': Defines the total number of time steps used by TFT
|
|
||||||
'num_encoder_steps': Determines length of LSTM encoder (i.e. history)
|
Requires the following keys:
|
||||||
'num_epochs': Maximum number of epochs for training
|
'total_time_steps': Defines the total number of time steps used by TFT
|
||||||
'early_stopping_patience': Early stopping param for keras
|
'num_encoder_steps': Determines length of LSTM encoder (i.e. history)
|
||||||
'multiprocessing_workers': # of cpus for data processing
|
'num_epochs': Maximum number of epochs for training
|
||||||
|
'early_stopping_patience': Early stopping param for keras
|
||||||
|
'multiprocessing_workers': # of cpus for data processing
|
||||||
Returns:
|
|
||||||
A dictionary of fixed parameters, e.g.:
|
|
||||||
|
Returns:
|
||||||
fixed_params = {
|
A dictionary of fixed parameters, e.g.:
|
||||||
'total_time_steps': 252 + 5,
|
|
||||||
'num_encoder_steps': 252,
|
fixed_params = {
|
||||||
'num_epochs': 100,
|
'total_time_steps': 252 + 5,
|
||||||
'early_stopping_patience': 5,
|
'num_encoder_steps': 252,
|
||||||
'multiprocessing_workers': 5,
|
'num_epochs': 100,
|
||||||
}
|
'early_stopping_patience': 5,
|
||||||
"""
|
'multiprocessing_workers': 5,
|
||||||
raise NotImplementedError
|
}
|
||||||
|
"""
|
||||||
# Shared functions across data-formatters
|
raise NotImplementedError
|
||||||
@property
|
|
||||||
def num_classes_per_cat_input(self):
|
# Shared functions across data-formatters
|
||||||
"""Returns number of categories per relevant input.
|
@property
|
||||||
|
def num_classes_per_cat_input(self):
|
||||||
This is seqeuently required for keras embedding layers.
|
"""Returns number of categories per relevant input.
|
||||||
"""
|
|
||||||
return self._num_classes_per_cat_input
|
This is seqeuently required for keras embedding layers.
|
||||||
|
"""
|
||||||
def get_num_samples_for_calibration(self):
|
return self._num_classes_per_cat_input
|
||||||
"""Gets the default number of training and validation samples.
|
|
||||||
|
def get_num_samples_for_calibration(self):
|
||||||
Use to sub-sample the data for network calibration and a value of -1 uses
|
"""Gets the default number of training and validation samples.
|
||||||
all available samples.
|
|
||||||
|
Use to sub-sample the data for network calibration and a value of -1 uses
|
||||||
Returns:
|
all available samples.
|
||||||
Tuple of (training samples, validation samples)
|
|
||||||
"""
|
Returns:
|
||||||
return -1, -1
|
Tuple of (training samples, validation samples)
|
||||||
|
"""
|
||||||
def get_column_definition(self):
|
return -1, -1
|
||||||
""""Returns formatted column definition in order expected by the TFT."""
|
|
||||||
|
def get_column_definition(self):
|
||||||
column_definition = self._column_definition
|
""""Returns formatted column definition in order expected by the TFT."""
|
||||||
|
|
||||||
# Sanity checks first.
|
column_definition = self._column_definition
|
||||||
# Ensure only one ID and time column exist
|
|
||||||
def _check_single_column(input_type):
|
# Sanity checks first.
|
||||||
|
# Ensure only one ID and time column exist
|
||||||
length = len([tup for tup in column_definition if tup[2] == input_type])
|
def _check_single_column(input_type):
|
||||||
|
|
||||||
if length != 1:
|
length = len([tup for tup in column_definition if tup[2] == input_type])
|
||||||
raise ValueError('Illegal number of inputs ({}) of type {}'.format(
|
|
||||||
length, input_type))
|
if length != 1:
|
||||||
|
raise ValueError("Illegal number of inputs ({}) of type {}".format(length, input_type))
|
||||||
_check_single_column(InputTypes.ID)
|
|
||||||
_check_single_column(InputTypes.TIME)
|
_check_single_column(InputTypes.ID)
|
||||||
|
_check_single_column(InputTypes.TIME)
|
||||||
identifier = [tup for tup in column_definition if tup[2] == InputTypes.ID]
|
|
||||||
time = [tup for tup in column_definition if tup[2] == InputTypes.TIME]
|
identifier = [tup for tup in column_definition if tup[2] == InputTypes.ID]
|
||||||
real_inputs = [
|
time = [tup for tup in column_definition if tup[2] == InputTypes.TIME]
|
||||||
tup for tup in column_definition if tup[1] == DataTypes.REAL_VALUED and
|
real_inputs = [
|
||||||
tup[2] not in {InputTypes.ID, InputTypes.TIME}
|
tup
|
||||||
]
|
for tup in column_definition
|
||||||
categorical_inputs = [
|
if tup[1] == DataTypes.REAL_VALUED and tup[2] not in {InputTypes.ID, InputTypes.TIME}
|
||||||
tup for tup in column_definition if tup[1] == DataTypes.CATEGORICAL and
|
]
|
||||||
tup[2] not in {InputTypes.ID, InputTypes.TIME}
|
categorical_inputs = [
|
||||||
]
|
tup
|
||||||
|
for tup in column_definition
|
||||||
return identifier + time + real_inputs + categorical_inputs
|
if tup[1] == DataTypes.CATEGORICAL and tup[2] not in {InputTypes.ID, InputTypes.TIME}
|
||||||
|
]
|
||||||
def _get_input_columns(self):
|
|
||||||
"""Returns names of all input columns."""
|
return identifier + time + real_inputs + categorical_inputs
|
||||||
return [
|
|
||||||
tup[0]
|
def _get_input_columns(self):
|
||||||
for tup in self.get_column_definition()
|
"""Returns names of all input columns."""
|
||||||
if tup[2] not in {InputTypes.ID, InputTypes.TIME}
|
return [tup[0] for tup in self.get_column_definition() if tup[2] not in {InputTypes.ID, InputTypes.TIME}]
|
||||||
]
|
|
||||||
|
def _get_tft_input_indices(self):
|
||||||
def _get_tft_input_indices(self):
|
"""Returns the relevant indexes and input sizes required by TFT."""
|
||||||
"""Returns the relevant indexes and input sizes required by TFT."""
|
|
||||||
|
# Functions
|
||||||
# Functions
|
def _extract_tuples_from_data_type(data_type, defn):
|
||||||
def _extract_tuples_from_data_type(data_type, defn):
|
return [tup for tup in defn if tup[1] == data_type and tup[2] not in {InputTypes.ID, InputTypes.TIME}]
|
||||||
return [
|
|
||||||
tup for tup in defn if tup[1] == data_type and
|
def _get_locations(input_types, defn):
|
||||||
tup[2] not in {InputTypes.ID, InputTypes.TIME}
|
return [i for i, tup in enumerate(defn) if tup[2] in input_types]
|
||||||
]
|
|
||||||
|
# Start extraction
|
||||||
def _get_locations(input_types, defn):
|
column_definition = [
|
||||||
return [i for i, tup in enumerate(defn) if tup[2] in input_types]
|
tup for tup in self.get_column_definition() if tup[2] not in {InputTypes.ID, InputTypes.TIME}
|
||||||
|
]
|
||||||
# Start extraction
|
|
||||||
column_definition = [
|
categorical_inputs = _extract_tuples_from_data_type(DataTypes.CATEGORICAL, column_definition)
|
||||||
tup for tup in self.get_column_definition()
|
real_inputs = _extract_tuples_from_data_type(DataTypes.REAL_VALUED, column_definition)
|
||||||
if tup[2] not in {InputTypes.ID, InputTypes.TIME}
|
|
||||||
]
|
locations = {
|
||||||
|
"input_size": len(self._get_input_columns()),
|
||||||
categorical_inputs = _extract_tuples_from_data_type(DataTypes.CATEGORICAL,
|
"output_size": len(_get_locations({InputTypes.TARGET}, column_definition)),
|
||||||
column_definition)
|
"category_counts": self.num_classes_per_cat_input,
|
||||||
real_inputs = _extract_tuples_from_data_type(DataTypes.REAL_VALUED,
|
"input_obs_loc": _get_locations({InputTypes.TARGET}, column_definition),
|
||||||
column_definition)
|
"static_input_loc": _get_locations({InputTypes.STATIC_INPUT}, column_definition),
|
||||||
|
"known_regular_inputs": _get_locations({InputTypes.STATIC_INPUT, InputTypes.KNOWN_INPUT}, real_inputs),
|
||||||
locations = {
|
"known_categorical_inputs": _get_locations(
|
||||||
'input_size':
|
{InputTypes.STATIC_INPUT, InputTypes.KNOWN_INPUT}, categorical_inputs
|
||||||
len(self._get_input_columns()),
|
),
|
||||||
'output_size':
|
}
|
||||||
len(_get_locations({InputTypes.TARGET}, column_definition)),
|
|
||||||
'category_counts':
|
return locations
|
||||||
self.num_classes_per_cat_input,
|
|
||||||
'input_obs_loc':
|
def get_experiment_params(self):
|
||||||
_get_locations({InputTypes.TARGET}, column_definition),
|
"""Returns fixed model parameters for experiments."""
|
||||||
'static_input_loc':
|
|
||||||
_get_locations({InputTypes.STATIC_INPUT}, column_definition),
|
required_keys = [
|
||||||
'known_regular_inputs':
|
"total_time_steps",
|
||||||
_get_locations({InputTypes.STATIC_INPUT, InputTypes.KNOWN_INPUT},
|
"num_encoder_steps",
|
||||||
real_inputs),
|
"num_epochs",
|
||||||
'known_categorical_inputs':
|
"early_stopping_patience",
|
||||||
_get_locations({InputTypes.STATIC_INPUT, InputTypes.KNOWN_INPUT},
|
"multiprocessing_workers",
|
||||||
categorical_inputs),
|
]
|
||||||
}
|
|
||||||
|
fixed_params = self.get_fixed_params()
|
||||||
return locations
|
|
||||||
|
for k in required_keys:
|
||||||
def get_experiment_params(self):
|
if k not in fixed_params:
|
||||||
"""Returns fixed model parameters for experiments."""
|
raise ValueError("Field {}".format(k) + " missing from fixed parameter definitions!")
|
||||||
|
|
||||||
required_keys = [
|
fixed_params["column_definition"] = self.get_column_definition()
|
||||||
'total_time_steps', 'num_encoder_steps', 'num_epochs',
|
|
||||||
'early_stopping_patience', 'multiprocessing_workers'
|
fixed_params.update(self._get_tft_input_indices())
|
||||||
]
|
|
||||||
|
return fixed_params
|
||||||
fixed_params = self.get_fixed_params()
|
|
||||||
|
|
||||||
for k in required_keys:
|
|
||||||
if k not in fixed_params:
|
|
||||||
raise ValueError('Field {}'.format(k) +
|
|
||||||
' missing from fixed parameter definitions!')
|
|
||||||
|
|
||||||
fixed_params['column_definition'] = self.get_column_definition()
|
|
||||||
|
|
||||||
fixed_params.update(self._get_tft_input_indices())
|
|
||||||
|
|
||||||
return fixed_params
|
|
||||||
|
|||||||
@@ -1,261 +1,254 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
# Copyright 2020 The Google Research Authors.
|
# Copyright 2020 The Google Research Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
#
|
#
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
#
|
#
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Lint as: python3
|
# Lint as: python3
|
||||||
"""Custom formatting functions for Electricity dataset.
|
"""Custom formatting functions for Electricity dataset.
|
||||||
|
|
||||||
Defines dataset specific column definitions and data transformations. Uses
|
Defines dataset specific column definitions and data transformations. Uses
|
||||||
entity specific z-score normalization.
|
entity specific z-score normalization.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import data_formatters.base
|
import data_formatters.base
|
||||||
import libs.utils as utils
|
import libs.utils as utils
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import sklearn.preprocessing
|
import sklearn.preprocessing
|
||||||
|
|
||||||
GenericDataFormatter = data_formatters.base.GenericDataFormatter
|
GenericDataFormatter = data_formatters.base.GenericDataFormatter
|
||||||
DataTypes = data_formatters.base.DataTypes
|
DataTypes = data_formatters.base.DataTypes
|
||||||
InputTypes = data_formatters.base.InputTypes
|
InputTypes = data_formatters.base.InputTypes
|
||||||
|
|
||||||
|
|
||||||
class ElectricityFormatter(GenericDataFormatter):
|
class ElectricityFormatter(GenericDataFormatter):
|
||||||
"""Defines and formats data for the electricity dataset.
|
"""Defines and formats data for the electricity dataset.
|
||||||
|
|
||||||
Note that per-entity z-score normalization is used here, and is implemented
|
Note that per-entity z-score normalization is used here, and is implemented
|
||||||
across functions.
|
across functions.
|
||||||
|
|
||||||
Attributes:
|
Attributes:
|
||||||
column_definition: Defines input and data type of column used in the
|
column_definition: Defines input and data type of column used in the
|
||||||
experiment.
|
experiment.
|
||||||
identifiers: Entity identifiers used in experiments.
|
identifiers: Entity identifiers used in experiments.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_column_definition = [
|
_column_definition = [
|
||||||
('id', DataTypes.REAL_VALUED, InputTypes.ID),
|
("id", DataTypes.REAL_VALUED, InputTypes.ID),
|
||||||
('hours_from_start', DataTypes.REAL_VALUED, InputTypes.TIME),
|
("hours_from_start", DataTypes.REAL_VALUED, InputTypes.TIME),
|
||||||
('power_usage', DataTypes.REAL_VALUED, InputTypes.TARGET),
|
("power_usage", DataTypes.REAL_VALUED, InputTypes.TARGET),
|
||||||
('hour', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
("hour", DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
||||||
('day_of_week', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
("day_of_week", DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
||||||
('hours_from_start', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
("hours_from_start", DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
||||||
('categorical_id', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
("categorical_id", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
]
|
]
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
"""Initialises formatter."""
|
"""Initialises formatter."""
|
||||||
|
|
||||||
self.identifiers = None
|
self.identifiers = None
|
||||||
self._real_scalers = None
|
self._real_scalers = None
|
||||||
self._cat_scalers = None
|
self._cat_scalers = None
|
||||||
self._target_scaler = None
|
self._target_scaler = None
|
||||||
self._num_classes_per_cat_input = None
|
self._num_classes_per_cat_input = None
|
||||||
self._time_steps = self.get_fixed_params()['total_time_steps']
|
self._time_steps = self.get_fixed_params()["total_time_steps"]
|
||||||
|
|
||||||
def split_data(self, df, valid_boundary=1315, test_boundary=1339):
|
def split_data(self, df, valid_boundary=1315, test_boundary=1339):
|
||||||
"""Splits data frame into training-validation-test data frames.
|
"""Splits data frame into training-validation-test data frames.
|
||||||
|
|
||||||
This also calibrates scaling object, and transforms data for each split.
|
This also calibrates scaling object, and transforms data for each split.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
df: Source data frame to split.
|
df: Source data frame to split.
|
||||||
valid_boundary: Starting year for validation data
|
valid_boundary: Starting year for validation data
|
||||||
test_boundary: Starting year for test data
|
test_boundary: Starting year for test data
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of transformed (train, valid, test) data.
|
Tuple of transformed (train, valid, test) data.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
print('Formatting train-valid-test splits.')
|
print("Formatting train-valid-test splits.")
|
||||||
|
|
||||||
index = df['days_from_start']
|
index = df["days_from_start"]
|
||||||
train = df.loc[index < valid_boundary]
|
train = df.loc[index < valid_boundary]
|
||||||
valid = df.loc[(index >= valid_boundary - 7) & (index < test_boundary)]
|
valid = df.loc[(index >= valid_boundary - 7) & (index < test_boundary)]
|
||||||
test = df.loc[index >= test_boundary - 7]
|
test = df.loc[index >= test_boundary - 7]
|
||||||
|
|
||||||
self.set_scalers(train)
|
self.set_scalers(train)
|
||||||
|
|
||||||
return (self.transform_inputs(data) for data in [train, valid, test])
|
return (self.transform_inputs(data) for data in [train, valid, test])
|
||||||
|
|
||||||
def set_scalers(self, df):
|
def set_scalers(self, df):
|
||||||
"""Calibrates scalers using the data supplied.
|
"""Calibrates scalers using the data supplied.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
df: Data to use to calibrate scalers.
|
df: Data to use to calibrate scalers.
|
||||||
"""
|
"""
|
||||||
print('Setting scalers with training data...')
|
print("Setting scalers with training data...")
|
||||||
|
|
||||||
column_definitions = self.get_column_definition()
|
column_definitions = self.get_column_definition()
|
||||||
id_column = utils.get_single_col_by_input_type(InputTypes.ID,
|
id_column = utils.get_single_col_by_input_type(InputTypes.ID, column_definitions)
|
||||||
column_definitions)
|
target_column = utils.get_single_col_by_input_type(InputTypes.TARGET, column_definitions)
|
||||||
target_column = utils.get_single_col_by_input_type(InputTypes.TARGET,
|
|
||||||
column_definitions)
|
# Format real scalers
|
||||||
|
real_inputs = utils.extract_cols_from_data_type(
|
||||||
# Format real scalers
|
DataTypes.REAL_VALUED, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
real_inputs = utils.extract_cols_from_data_type(
|
)
|
||||||
DataTypes.REAL_VALUED, column_definitions,
|
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
# Initialise scaler caches
|
||||||
|
self._real_scalers = {}
|
||||||
# Initialise scaler caches
|
self._target_scaler = {}
|
||||||
self._real_scalers = {}
|
identifiers = []
|
||||||
self._target_scaler = {}
|
for identifier, sliced in df.groupby(id_column):
|
||||||
identifiers = []
|
|
||||||
for identifier, sliced in df.groupby(id_column):
|
if len(sliced) >= self._time_steps:
|
||||||
|
|
||||||
if len(sliced) >= self._time_steps:
|
data = sliced[real_inputs].values
|
||||||
|
targets = sliced[[target_column]].values
|
||||||
data = sliced[real_inputs].values
|
self._real_scalers[identifier] = sklearn.preprocessing.StandardScaler().fit(data)
|
||||||
targets = sliced[[target_column]].values
|
|
||||||
self._real_scalers[identifier] \
|
self._target_scaler[identifier] = sklearn.preprocessing.StandardScaler().fit(targets)
|
||||||
= sklearn.preprocessing.StandardScaler().fit(data)
|
identifiers.append(identifier)
|
||||||
|
|
||||||
self._target_scaler[identifier] \
|
# Format categorical scalers
|
||||||
= sklearn.preprocessing.StandardScaler().fit(targets)
|
categorical_inputs = utils.extract_cols_from_data_type(
|
||||||
identifiers.append(identifier)
|
DataTypes.CATEGORICAL, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
|
)
|
||||||
# Format categorical scalers
|
|
||||||
categorical_inputs = utils.extract_cols_from_data_type(
|
categorical_scalers = {}
|
||||||
DataTypes.CATEGORICAL, column_definitions,
|
num_classes = []
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
for col in categorical_inputs:
|
||||||
|
# Set all to str so that we don't have mixed integer/string columns
|
||||||
categorical_scalers = {}
|
srs = df[col].apply(str)
|
||||||
num_classes = []
|
categorical_scalers[col] = sklearn.preprocessing.LabelEncoder().fit(srs.values)
|
||||||
for col in categorical_inputs:
|
num_classes.append(srs.nunique())
|
||||||
# Set all to str so that we don't have mixed integer/string columns
|
|
||||||
srs = df[col].apply(str)
|
# Set categorical scaler outputs
|
||||||
categorical_scalers[col] = sklearn.preprocessing.LabelEncoder().fit(
|
self._cat_scalers = categorical_scalers
|
||||||
srs.values)
|
self._num_classes_per_cat_input = num_classes
|
||||||
num_classes.append(srs.nunique())
|
|
||||||
|
# Extract identifiers in case required
|
||||||
# Set categorical scaler outputs
|
self.identifiers = identifiers
|
||||||
self._cat_scalers = categorical_scalers
|
|
||||||
self._num_classes_per_cat_input = num_classes
|
def transform_inputs(self, df):
|
||||||
|
"""Performs feature transformations.
|
||||||
# Extract identifiers in case required
|
|
||||||
self.identifiers = identifiers
|
This includes both feature engineering, preprocessing and normalisation.
|
||||||
|
|
||||||
def transform_inputs(self, df):
|
Args:
|
||||||
"""Performs feature transformations.
|
df: Data frame to transform.
|
||||||
|
|
||||||
This includes both feature engineering, preprocessing and normalisation.
|
Returns:
|
||||||
|
Transformed data frame.
|
||||||
Args:
|
|
||||||
df: Data frame to transform.
|
"""
|
||||||
|
|
||||||
Returns:
|
if self._real_scalers is None and self._cat_scalers is None:
|
||||||
Transformed data frame.
|
raise ValueError("Scalers have not been set!")
|
||||||
|
|
||||||
"""
|
# Extract relevant columns
|
||||||
|
column_definitions = self.get_column_definition()
|
||||||
if self._real_scalers is None and self._cat_scalers is None:
|
id_col = utils.get_single_col_by_input_type(InputTypes.ID, column_definitions)
|
||||||
raise ValueError('Scalers have not been set!')
|
real_inputs = utils.extract_cols_from_data_type(
|
||||||
|
DataTypes.REAL_VALUED, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
# Extract relevant columns
|
)
|
||||||
column_definitions = self.get_column_definition()
|
categorical_inputs = utils.extract_cols_from_data_type(
|
||||||
id_col = utils.get_single_col_by_input_type(InputTypes.ID,
|
DataTypes.CATEGORICAL, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
column_definitions)
|
)
|
||||||
real_inputs = utils.extract_cols_from_data_type(
|
|
||||||
DataTypes.REAL_VALUED, column_definitions,
|
# Transform real inputs per entity
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
df_list = []
|
||||||
categorical_inputs = utils.extract_cols_from_data_type(
|
for identifier, sliced in df.groupby(id_col):
|
||||||
DataTypes.CATEGORICAL, column_definitions,
|
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
# Filter out any trajectories that are too short
|
||||||
|
if len(sliced) >= self._time_steps:
|
||||||
# Transform real inputs per entity
|
sliced_copy = sliced.copy()
|
||||||
df_list = []
|
sliced_copy[real_inputs] = self._real_scalers[identifier].transform(sliced_copy[real_inputs].values)
|
||||||
for identifier, sliced in df.groupby(id_col):
|
df_list.append(sliced_copy)
|
||||||
|
|
||||||
# Filter out any trajectories that are too short
|
output = pd.concat(df_list, axis=0)
|
||||||
if len(sliced) >= self._time_steps:
|
|
||||||
sliced_copy = sliced.copy()
|
# Format categorical inputs
|
||||||
sliced_copy[real_inputs] = self._real_scalers[identifier].transform(
|
for col in categorical_inputs:
|
||||||
sliced_copy[real_inputs].values)
|
string_df = df[col].apply(str)
|
||||||
df_list.append(sliced_copy)
|
output[col] = self._cat_scalers[col].transform(string_df)
|
||||||
|
|
||||||
output = pd.concat(df_list, axis=0)
|
return output
|
||||||
|
|
||||||
# Format categorical inputs
|
def format_predictions(self, predictions):
|
||||||
for col in categorical_inputs:
|
"""Reverts any normalisation to give predictions in original scale.
|
||||||
string_df = df[col].apply(str)
|
|
||||||
output[col] = self._cat_scalers[col].transform(string_df)
|
Args:
|
||||||
|
predictions: Dataframe of model predictions.
|
||||||
return output
|
|
||||||
|
Returns:
|
||||||
def format_predictions(self, predictions):
|
Data frame of unnormalised predictions.
|
||||||
"""Reverts any normalisation to give predictions in original scale.
|
"""
|
||||||
|
|
||||||
Args:
|
if self._target_scaler is None:
|
||||||
predictions: Dataframe of model predictions.
|
raise ValueError("Scalers have not been set!")
|
||||||
|
|
||||||
Returns:
|
column_names = predictions.columns
|
||||||
Data frame of unnormalised predictions.
|
|
||||||
"""
|
df_list = []
|
||||||
|
for identifier, sliced in predictions.groupby("identifier"):
|
||||||
if self._target_scaler is None:
|
sliced_copy = sliced.copy()
|
||||||
raise ValueError('Scalers have not been set!')
|
target_scaler = self._target_scaler[identifier]
|
||||||
|
|
||||||
column_names = predictions.columns
|
for col in column_names:
|
||||||
|
if col not in {"forecast_time", "identifier"}:
|
||||||
df_list = []
|
sliced_copy[col] = target_scaler.inverse_transform(sliced_copy[col])
|
||||||
for identifier, sliced in predictions.groupby('identifier'):
|
df_list.append(sliced_copy)
|
||||||
sliced_copy = sliced.copy()
|
|
||||||
target_scaler = self._target_scaler[identifier]
|
output = pd.concat(df_list, axis=0)
|
||||||
|
|
||||||
for col in column_names:
|
return output
|
||||||
if col not in {'forecast_time', 'identifier'}:
|
|
||||||
sliced_copy[col] = target_scaler.inverse_transform(sliced_copy[col])
|
# Default params
|
||||||
df_list.append(sliced_copy)
|
def get_fixed_params(self):
|
||||||
|
"""Returns fixed model parameters for experiments."""
|
||||||
output = pd.concat(df_list, axis=0)
|
|
||||||
|
fixed_params = {
|
||||||
return output
|
"total_time_steps": 8 * 24,
|
||||||
|
"num_encoder_steps": 7 * 24,
|
||||||
# Default params
|
"num_epochs": 100,
|
||||||
def get_fixed_params(self):
|
"early_stopping_patience": 5,
|
||||||
"""Returns fixed model parameters for experiments."""
|
"multiprocessing_workers": 5,
|
||||||
|
}
|
||||||
fixed_params = {
|
|
||||||
'total_time_steps': 8 * 24,
|
return fixed_params
|
||||||
'num_encoder_steps': 7 * 24,
|
|
||||||
'num_epochs': 100,
|
def get_default_model_params(self):
|
||||||
'early_stopping_patience': 5,
|
"""Returns default optimised model parameters."""
|
||||||
'multiprocessing_workers': 5
|
|
||||||
}
|
model_params = {
|
||||||
|
"dropout_rate": 0.1,
|
||||||
return fixed_params
|
"hidden_layer_size": 160,
|
||||||
|
"learning_rate": 0.001,
|
||||||
def get_default_model_params(self):
|
"minibatch_size": 64,
|
||||||
"""Returns default optimised model parameters."""
|
"max_gradient_norm": 0.01,
|
||||||
|
"num_heads": 4,
|
||||||
model_params = {
|
"stack_size": 1,
|
||||||
'dropout_rate': 0.1,
|
}
|
||||||
'hidden_layer_size': 160,
|
|
||||||
'learning_rate': 0.001,
|
return model_params
|
||||||
'minibatch_size': 64,
|
|
||||||
'max_gradient_norm': 0.01,
|
def get_num_samples_for_calibration(self):
|
||||||
'num_heads': 4,
|
"""Gets the default number of training and validation samples.
|
||||||
'stack_size': 1
|
|
||||||
}
|
Use to sub-sample the data for network calibration and a value of -1 uses
|
||||||
|
all available samples.
|
||||||
return model_params
|
|
||||||
|
Returns:
|
||||||
def get_num_samples_for_calibration(self):
|
Tuple of (training samples, validation samples)
|
||||||
"""Gets the default number of training and validation samples.
|
"""
|
||||||
|
return 450000, 50000
|
||||||
Use to sub-sample the data for network calibration and a value of -1 uses
|
|
||||||
all available samples.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Tuple of (training samples, validation samples)
|
|
||||||
"""
|
|
||||||
return 450000, 50000
|
|
||||||
|
|||||||
@@ -1,327 +1,333 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
# Copyright 2020 The Google Research Authors.
|
# Copyright 2020 The Google Research Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
#
|
#
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
#
|
#
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Lint as: python3
|
# Lint as: python3
|
||||||
"""Custom formatting functions for Favorita dataset.
|
"""Custom formatting functions for Favorita dataset.
|
||||||
|
|
||||||
Defines dataset specific column definitions and data transformations.
|
Defines dataset specific column definitions and data transformations.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import data_formatters.base
|
import data_formatters.base
|
||||||
import libs.utils as utils
|
import libs.utils as utils
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import sklearn.preprocessing
|
import sklearn.preprocessing
|
||||||
|
|
||||||
DataTypes = data_formatters.base.DataTypes
|
DataTypes = data_formatters.base.DataTypes
|
||||||
InputTypes = data_formatters.base.InputTypes
|
InputTypes = data_formatters.base.InputTypes
|
||||||
|
|
||||||
|
|
||||||
class FavoritaFormatter(data_formatters.base.GenericDataFormatter):
|
class FavoritaFormatter(data_formatters.base.GenericDataFormatter):
|
||||||
"""Defines and formats data for the Favorita dataset.
|
"""Defines and formats data for the Favorita dataset.
|
||||||
|
|
||||||
Attributes:
|
Attributes:
|
||||||
column_definition: Defines input and data type of column used in the
|
column_definition: Defines input and data type of column used in the
|
||||||
experiment.
|
experiment.
|
||||||
identifiers: Entity identifiers used in experiments.
|
identifiers: Entity identifiers used in experiments.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_column_definition = [
|
_column_definition = [
|
||||||
('traj_id', DataTypes.REAL_VALUED, InputTypes.ID),
|
("traj_id", DataTypes.REAL_VALUED, InputTypes.ID),
|
||||||
('date', DataTypes.DATE, InputTypes.TIME),
|
("date", DataTypes.DATE, InputTypes.TIME),
|
||||||
('log_sales', DataTypes.REAL_VALUED, InputTypes.TARGET),
|
("log_sales", DataTypes.REAL_VALUED, InputTypes.TARGET),
|
||||||
('onpromotion', DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
("onpromotion", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
||||||
('transactions', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
("transactions", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
('oil', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
("oil", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
('day_of_week', DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
("day_of_week", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
||||||
('day_of_month', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
("day_of_month", DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
||||||
('month', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
("month", DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
||||||
('national_hol', DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
("national_hol", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
||||||
('regional_hol', DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
("regional_hol", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
||||||
('local_hol', DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
("local_hol", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
||||||
('open', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
("open", DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
||||||
('item_nbr', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
("item_nbr", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
('store_nbr', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
("store_nbr", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
('city', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
("city", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
('state', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
("state", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
('type', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
("type", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
('cluster', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
("cluster", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
('family', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
("family", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
('class', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
("class", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
('perishable', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT)
|
("perishable", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
]
|
]
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
"""Initialises formatter."""
|
"""Initialises formatter."""
|
||||||
|
|
||||||
self.identifiers = None
|
self.identifiers = None
|
||||||
self._real_scalers = None
|
self._real_scalers = None
|
||||||
self._cat_scalers = None
|
self._cat_scalers = None
|
||||||
self._target_scaler = None
|
self._target_scaler = None
|
||||||
self._num_classes_per_cat_input = None
|
self._num_classes_per_cat_input = None
|
||||||
|
|
||||||
def split_data(self, df, valid_boundary=None, test_boundary=None):
|
def split_data(self, df, valid_boundary=None, test_boundary=None):
|
||||||
"""Splits data frame into training-validation-test data frames.
|
"""Splits data frame into training-validation-test data frames.
|
||||||
|
|
||||||
This also calibrates scaling object, and transforms data for each split.
|
This also calibrates scaling object, and transforms data for each split.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
df: Source data frame to split.
|
df: Source data frame to split.
|
||||||
valid_boundary: Starting year for validation data
|
valid_boundary: Starting year for validation data
|
||||||
test_boundary: Starting year for test data
|
test_boundary: Starting year for test data
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of transformed (train, valid, test) data.
|
Tuple of transformed (train, valid, test) data.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
print('Formatting train-valid-test splits.')
|
print("Formatting train-valid-test splits.")
|
||||||
|
|
||||||
if valid_boundary is None:
|
if valid_boundary is None:
|
||||||
valid_boundary = pd.datetime(2015, 12, 1)
|
valid_boundary = pd.datetime(2015, 12, 1)
|
||||||
|
|
||||||
fixed_params = self.get_fixed_params()
|
fixed_params = self.get_fixed_params()
|
||||||
time_steps = fixed_params['total_time_steps']
|
time_steps = fixed_params["total_time_steps"]
|
||||||
lookback = fixed_params['num_encoder_steps']
|
lookback = fixed_params["num_encoder_steps"]
|
||||||
forecast_horizon = time_steps - lookback
|
forecast_horizon = time_steps - lookback
|
||||||
|
|
||||||
df['date'] = pd.to_datetime(df['date'])
|
df["date"] = pd.to_datetime(df["date"])
|
||||||
df_lists = {'train': [], 'valid': [], 'test': []}
|
df_lists = {"train": [], "valid": [], "test": []}
|
||||||
for _, sliced in df.groupby('traj_id'):
|
for _, sliced in df.groupby("traj_id"):
|
||||||
index = sliced['date']
|
index = sliced["date"]
|
||||||
train = sliced.loc[index < valid_boundary]
|
train = sliced.loc[index < valid_boundary]
|
||||||
train_len = len(train)
|
train_len = len(train)
|
||||||
valid_len = train_len + forecast_horizon
|
valid_len = train_len + forecast_horizon
|
||||||
valid = sliced.iloc[train_len - lookback:valid_len, :]
|
valid = sliced.iloc[train_len - lookback : valid_len, :]
|
||||||
test = sliced.iloc[valid_len - lookback:valid_len + forecast_horizon, :]
|
test = sliced.iloc[valid_len - lookback : valid_len + forecast_horizon, :]
|
||||||
|
|
||||||
sliced_map = {'train': train, 'valid': valid, 'test': test}
|
sliced_map = {"train": train, "valid": valid, "test": test}
|
||||||
|
|
||||||
for k in sliced_map:
|
for k in sliced_map:
|
||||||
item = sliced_map[k]
|
item = sliced_map[k]
|
||||||
|
|
||||||
if len(item) >= time_steps:
|
if len(item) >= time_steps:
|
||||||
df_lists[k].append(item)
|
df_lists[k].append(item)
|
||||||
|
|
||||||
dfs = {k: pd.concat(df_lists[k], axis=0) for k in df_lists}
|
dfs = {k: pd.concat(df_lists[k], axis=0) for k in df_lists}
|
||||||
|
|
||||||
train = dfs['train']
|
train = dfs["train"]
|
||||||
self.set_scalers(train, set_real=True)
|
self.set_scalers(train, set_real=True)
|
||||||
|
|
||||||
# Use all data for label encoding to handle labels not present in training.
|
# Use all data for label encoding to handle labels not present in training.
|
||||||
self.set_scalers(df, set_real=False)
|
self.set_scalers(df, set_real=False)
|
||||||
|
|
||||||
# Filter out identifiers not present in training (i.e. cold-started items).
|
# Filter out identifiers not present in training (i.e. cold-started items).
|
||||||
def filter_ids(frame):
|
def filter_ids(frame):
|
||||||
identifiers = set(self.identifiers)
|
identifiers = set(self.identifiers)
|
||||||
index = frame['traj_id']
|
index = frame["traj_id"]
|
||||||
return frame.loc[index.apply(lambda x: x in identifiers)]
|
return frame.loc[index.apply(lambda x: x in identifiers)]
|
||||||
|
|
||||||
valid = filter_ids(dfs['valid'])
|
valid = filter_ids(dfs["valid"])
|
||||||
test = filter_ids(dfs['test'])
|
test = filter_ids(dfs["test"])
|
||||||
|
|
||||||
return (self.transform_inputs(data) for data in [train, valid, test])
|
return (self.transform_inputs(data) for data in [train, valid, test])
|
||||||
|
|
||||||
def set_scalers(self, df, set_real=True):
|
def set_scalers(self, df, set_real=True):
|
||||||
"""Calibrates scalers using the data supplied.
|
"""Calibrates scalers using the data supplied.
|
||||||
|
|
||||||
Label encoding is applied to the entire dataset (i.e. including test),
|
Label encoding is applied to the entire dataset (i.e. including test),
|
||||||
so that unseen labels can be handled at run-time.
|
so that unseen labels can be handled at run-time.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
df: Data to use to calibrate scalers.
|
df: Data to use to calibrate scalers.
|
||||||
set_real: Whether to fit set real-valued or categorical scalers
|
set_real: Whether to fit set real-valued or categorical scalers
|
||||||
"""
|
"""
|
||||||
print('Setting scalers with training data...')
|
print("Setting scalers with training data...")
|
||||||
|
|
||||||
column_definitions = self.get_column_definition()
|
column_definitions = self.get_column_definition()
|
||||||
id_column = utils.get_single_col_by_input_type(InputTypes.ID,
|
id_column = utils.get_single_col_by_input_type(InputTypes.ID, column_definitions)
|
||||||
column_definitions)
|
target_column = utils.get_single_col_by_input_type(InputTypes.TARGET, column_definitions)
|
||||||
target_column = utils.get_single_col_by_input_type(InputTypes.TARGET,
|
|
||||||
column_definitions)
|
if set_real:
|
||||||
|
|
||||||
if set_real:
|
# Extract identifiers in case required
|
||||||
|
self.identifiers = list(df[id_column].unique())
|
||||||
# Extract identifiers in case required
|
|
||||||
self.identifiers = list(df[id_column].unique())
|
# Format real scalers
|
||||||
|
self._real_scalers = {}
|
||||||
# Format real scalers
|
for col in ["oil", "transactions", "log_sales"]:
|
||||||
self._real_scalers = {}
|
self._real_scalers[col] = (df[col].mean(), df[col].std())
|
||||||
for col in ['oil', 'transactions', 'log_sales']:
|
|
||||||
self._real_scalers[col] = (df[col].mean(), df[col].std())
|
self._target_scaler = (df[target_column].mean(), df[target_column].std())
|
||||||
|
|
||||||
self._target_scaler = (df[target_column].mean(), df[target_column].std())
|
else:
|
||||||
|
# Format categorical scalers
|
||||||
else:
|
categorical_inputs = utils.extract_cols_from_data_type(
|
||||||
# Format categorical scalers
|
DataTypes.CATEGORICAL, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
categorical_inputs = utils.extract_cols_from_data_type(
|
)
|
||||||
DataTypes.CATEGORICAL, column_definitions,
|
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
categorical_scalers = {}
|
||||||
|
num_classes = []
|
||||||
categorical_scalers = {}
|
if self.identifiers is None:
|
||||||
num_classes = []
|
raise ValueError("Scale real-valued inputs first!")
|
||||||
if self.identifiers is None:
|
id_set = set(self.identifiers)
|
||||||
raise ValueError('Scale real-valued inputs first!')
|
valid_idx = df["traj_id"].apply(lambda x: x in id_set)
|
||||||
id_set = set(self.identifiers)
|
for col in categorical_inputs:
|
||||||
valid_idx = df['traj_id'].apply(lambda x: x in id_set)
|
# Set all to str so that we don't have mixed integer/string columns
|
||||||
for col in categorical_inputs:
|
srs = df[col].apply(str).loc[valid_idx]
|
||||||
# Set all to str so that we don't have mixed integer/string columns
|
categorical_scalers[col] = sklearn.preprocessing.LabelEncoder().fit(srs.values)
|
||||||
srs = df[col].apply(str).loc[valid_idx]
|
|
||||||
categorical_scalers[col] = sklearn.preprocessing.LabelEncoder().fit(
|
num_classes.append(srs.nunique())
|
||||||
srs.values)
|
|
||||||
|
# Set categorical scaler outputs
|
||||||
num_classes.append(srs.nunique())
|
self._cat_scalers = categorical_scalers
|
||||||
|
self._num_classes_per_cat_input = num_classes
|
||||||
# Set categorical scaler outputs
|
|
||||||
self._cat_scalers = categorical_scalers
|
def transform_inputs(self, df):
|
||||||
self._num_classes_per_cat_input = num_classes
|
"""Performs feature transformations.
|
||||||
|
|
||||||
def transform_inputs(self, df):
|
This includes both feature engineering, preprocessing and normalisation.
|
||||||
"""Performs feature transformations.
|
|
||||||
|
Args:
|
||||||
This includes both feature engineering, preprocessing and normalisation.
|
df: Data frame to transform.
|
||||||
|
|
||||||
Args:
|
Returns:
|
||||||
df: Data frame to transform.
|
Transformed data frame.
|
||||||
|
|
||||||
Returns:
|
"""
|
||||||
Transformed data frame.
|
output = df.copy()
|
||||||
|
|
||||||
"""
|
if self._real_scalers is None and self._cat_scalers is None:
|
||||||
output = df.copy()
|
raise ValueError("Scalers have not been set!")
|
||||||
|
|
||||||
if self._real_scalers is None and self._cat_scalers is None:
|
column_definitions = self.get_column_definition()
|
||||||
raise ValueError('Scalers have not been set!')
|
|
||||||
|
categorical_inputs = utils.extract_cols_from_data_type(
|
||||||
column_definitions = self.get_column_definition()
|
DataTypes.CATEGORICAL, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
|
)
|
||||||
categorical_inputs = utils.extract_cols_from_data_type(
|
|
||||||
DataTypes.CATEGORICAL, column_definitions,
|
# Format real inputs
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
for col in ["log_sales", "oil", "transactions"]:
|
||||||
|
mean, std = self._real_scalers[col]
|
||||||
# Format real inputs
|
output[col] = (df[col] - mean) / std
|
||||||
for col in ['log_sales', 'oil', 'transactions']:
|
|
||||||
mean, std = self._real_scalers[col]
|
if col == "log_sales":
|
||||||
output[col] = (df[col] - mean) / std
|
output[col] = output[col].fillna(0.0) # mean imputation
|
||||||
|
|
||||||
if col == 'log_sales':
|
# Format categorical inputs
|
||||||
output[col] = output[col].fillna(0.) # mean imputation
|
for col in categorical_inputs:
|
||||||
|
string_df = df[col].apply(str)
|
||||||
# Format categorical inputs
|
output[col] = self._cat_scalers[col].transform(string_df)
|
||||||
for col in categorical_inputs:
|
|
||||||
string_df = df[col].apply(str)
|
return output
|
||||||
output[col] = self._cat_scalers[col].transform(string_df)
|
|
||||||
|
def format_predictions(self, predictions):
|
||||||
return output
|
"""Reverts any normalisation to give predictions in original scale.
|
||||||
|
|
||||||
def format_predictions(self, predictions):
|
Args:
|
||||||
"""Reverts any normalisation to give predictions in original scale.
|
predictions: Dataframe of model predictions.
|
||||||
|
|
||||||
Args:
|
Returns:
|
||||||
predictions: Dataframe of model predictions.
|
Data frame of unnormalised predictions.
|
||||||
|
"""
|
||||||
Returns:
|
output = predictions.copy()
|
||||||
Data frame of unnormalised predictions.
|
|
||||||
"""
|
column_names = predictions.columns
|
||||||
output = predictions.copy()
|
mean, std = self._target_scaler
|
||||||
|
for col in column_names:
|
||||||
column_names = predictions.columns
|
if col not in {"forecast_time", "identifier"}:
|
||||||
mean, std = self._target_scaler
|
output[col] = (predictions[col] * std) + mean
|
||||||
for col in column_names:
|
|
||||||
if col not in {'forecast_time', 'identifier'}:
|
return output
|
||||||
output[col] = (predictions[col] * std) + mean
|
|
||||||
|
# Default params
|
||||||
return output
|
def get_fixed_params(self):
|
||||||
|
"""Returns fixed model parameters for experiments."""
|
||||||
# Default params
|
|
||||||
def get_fixed_params(self):
|
fixed_params = {
|
||||||
"""Returns fixed model parameters for experiments."""
|
"total_time_steps": 120,
|
||||||
|
"num_encoder_steps": 90,
|
||||||
fixed_params = {
|
"num_epochs": 100,
|
||||||
'total_time_steps': 120,
|
"early_stopping_patience": 5,
|
||||||
'num_encoder_steps': 90,
|
"multiprocessing_workers": 5,
|
||||||
'num_epochs': 100,
|
}
|
||||||
'early_stopping_patience': 5,
|
|
||||||
'multiprocessing_workers': 5
|
return fixed_params
|
||||||
}
|
|
||||||
|
def get_default_model_params(self):
|
||||||
return fixed_params
|
"""Returns default optimised model parameters."""
|
||||||
|
|
||||||
def get_default_model_params(self):
|
model_params = {
|
||||||
"""Returns default optimised model parameters."""
|
"dropout_rate": 0.1,
|
||||||
|
"hidden_layer_size": 240,
|
||||||
model_params = {
|
"learning_rate": 0.001,
|
||||||
'dropout_rate': 0.1,
|
"minibatch_size": 128,
|
||||||
'hidden_layer_size': 240,
|
"max_gradient_norm": 100.0,
|
||||||
'learning_rate': 0.001,
|
"num_heads": 4,
|
||||||
'minibatch_size': 128,
|
"stack_size": 1,
|
||||||
'max_gradient_norm': 100.,
|
}
|
||||||
'num_heads': 4,
|
|
||||||
'stack_size': 1
|
return model_params
|
||||||
}
|
|
||||||
|
def get_num_samples_for_calibration(self):
|
||||||
return model_params
|
"""Gets the default number of training and validation samples.
|
||||||
|
|
||||||
def get_num_samples_for_calibration(self):
|
Use to sub-sample the data for network calibration and a value of -1 uses
|
||||||
"""Gets the default number of training and validation samples.
|
all available samples.
|
||||||
|
|
||||||
Use to sub-sample the data for network calibration and a value of -1 uses
|
Returns:
|
||||||
all available samples.
|
Tuple of (training samples, validation samples)
|
||||||
|
"""
|
||||||
Returns:
|
return 450000, 50000
|
||||||
Tuple of (training samples, validation samples)
|
|
||||||
"""
|
def get_column_definition(self):
|
||||||
return 450000, 50000
|
""" "Formats column definition in order expected by the TFT.
|
||||||
|
|
||||||
def get_column_definition(self):
|
Modified for Favorita to match column order of original experiment.
|
||||||
""""Formats column definition in order expected by the TFT.
|
|
||||||
|
Returns:
|
||||||
Modified for Favorita to match column order of original experiment.
|
Favorita-specific column definition
|
||||||
|
"""
|
||||||
Returns:
|
|
||||||
Favorita-specific column definition
|
column_definition = self._column_definition
|
||||||
"""
|
|
||||||
|
# Sanity checks first.
|
||||||
column_definition = self._column_definition
|
# Ensure only one ID and time column exist
|
||||||
|
def _check_single_column(input_type):
|
||||||
# Sanity checks first.
|
|
||||||
# Ensure only one ID and time column exist
|
length = len([tup for tup in column_definition if tup[2] == input_type])
|
||||||
def _check_single_column(input_type):
|
|
||||||
|
if length != 1:
|
||||||
length = len([tup for tup in column_definition if tup[2] == input_type])
|
raise ValueError("Illegal number of inputs ({}) of type {}".format(length, input_type))
|
||||||
|
|
||||||
if length != 1:
|
_check_single_column(InputTypes.ID)
|
||||||
raise ValueError('Illegal number of inputs ({}) of type {}'.format(
|
_check_single_column(InputTypes.TIME)
|
||||||
length, input_type))
|
|
||||||
|
identifier = [tup for tup in column_definition if tup[2] == InputTypes.ID]
|
||||||
_check_single_column(InputTypes.ID)
|
time = [tup for tup in column_definition if tup[2] == InputTypes.TIME]
|
||||||
_check_single_column(InputTypes.TIME)
|
real_inputs = [
|
||||||
|
tup
|
||||||
identifier = [tup for tup in column_definition if tup[2] == InputTypes.ID]
|
for tup in column_definition
|
||||||
time = [tup for tup in column_definition if tup[2] == InputTypes.TIME]
|
if tup[1] == DataTypes.REAL_VALUED and tup[2] not in {InputTypes.ID, InputTypes.TIME}
|
||||||
real_inputs = [
|
]
|
||||||
tup for tup in column_definition if tup[1] == DataTypes.REAL_VALUED and
|
|
||||||
tup[2] not in {InputTypes.ID, InputTypes.TIME}
|
col_definition_map = {tup[0]: tup for tup in column_definition}
|
||||||
]
|
col_order = [
|
||||||
|
"item_nbr",
|
||||||
col_definition_map = {tup[0]: tup for tup in column_definition}
|
"store_nbr",
|
||||||
col_order = [
|
"city",
|
||||||
'item_nbr', 'store_nbr', 'city', 'state', 'type', 'cluster', 'family',
|
"state",
|
||||||
'class', 'perishable', 'onpromotion', 'day_of_week', 'national_hol',
|
"type",
|
||||||
'regional_hol', 'local_hol'
|
"cluster",
|
||||||
]
|
"family",
|
||||||
categorical_inputs = [
|
"class",
|
||||||
col_definition_map[k] for k in col_order if k in col_definition_map
|
"perishable",
|
||||||
]
|
"onpromotion",
|
||||||
|
"day_of_week",
|
||||||
return identifier + time + real_inputs + categorical_inputs
|
"national_hol",
|
||||||
|
"regional_hol",
|
||||||
|
"local_hol",
|
||||||
|
]
|
||||||
|
categorical_inputs = [col_definition_map[k] for k in col_order if k in col_definition_map]
|
||||||
|
|
||||||
|
return identifier + time + real_inputs + categorical_inputs
|
||||||
|
|||||||
@@ -1,220 +1,219 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
# Copyright 2020 The Google Research Authors.
|
# Copyright 2020 The Google Research Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
#
|
#
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
#
|
#
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Lint as: python3
|
# Lint as: python3
|
||||||
"""Custom formatting functions for Alpha158 dataset.
|
"""Custom formatting functions for Alpha158 dataset.
|
||||||
|
|
||||||
Defines dataset specific column definitions and data transformations.
|
Defines dataset specific column definitions and data transformations.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import data_formatters.base
|
import data_formatters.base
|
||||||
import libs.utils as utils
|
import libs.utils as utils
|
||||||
import sklearn.preprocessing
|
import sklearn.preprocessing
|
||||||
|
|
||||||
GenericDataFormatter = data_formatters.base.GenericDataFormatter
|
GenericDataFormatter = data_formatters.base.GenericDataFormatter
|
||||||
DataTypes = data_formatters.base.DataTypes
|
DataTypes = data_formatters.base.DataTypes
|
||||||
InputTypes = data_formatters.base.InputTypes
|
InputTypes = data_formatters.base.InputTypes
|
||||||
|
|
||||||
class Alpha158Formatter(GenericDataFormatter):
|
|
||||||
"""Defines and formats data for the Alpha158 dataset.
|
class Alpha158Formatter(GenericDataFormatter):
|
||||||
|
"""Defines and formats data for the Alpha158 dataset.
|
||||||
Attributes:
|
|
||||||
column_definition: Defines input and data type of column used in the
|
Attributes:
|
||||||
experiment.
|
column_definition: Defines input and data type of column used in the
|
||||||
identifiers: Entity identifiers used in experiments.
|
experiment.
|
||||||
"""
|
identifiers: Entity identifiers used in experiments.
|
||||||
|
"""
|
||||||
_column_definition = [
|
|
||||||
('instrument', DataTypes.CATEGORICAL, InputTypes.ID),
|
_column_definition = [
|
||||||
('LABEL0', DataTypes.REAL_VALUED, InputTypes.TARGET),
|
("instrument", DataTypes.CATEGORICAL, InputTypes.ID),
|
||||||
('date', DataTypes.DATE, InputTypes.TIME),
|
("LABEL0", DataTypes.REAL_VALUED, InputTypes.TARGET),
|
||||||
('month', DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
("date", DataTypes.DATE, InputTypes.TIME),
|
||||||
('day_of_week', DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
("month", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
||||||
# Selected 10 features
|
("day_of_week", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
||||||
('RESI5', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
# Selected 10 features
|
||||||
('WVMA5', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
("RESI5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
('RSQR5', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
("WVMA5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
('KLEN', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
("RSQR5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
('RSQR10', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
("KLEN", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
('CORR5', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
("RSQR10", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
('CORD5', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
("CORR5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
('CORR10', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
("CORD5", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
('ROC60', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
("CORR10", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
('RESI10', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
("ROC60", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
('const', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
("RESI10", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
]
|
("const", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
|
]
|
||||||
def __init__(self):
|
|
||||||
"""Initialises formatter."""
|
def __init__(self):
|
||||||
|
"""Initialises formatter."""
|
||||||
self.identifiers = None
|
|
||||||
self._real_scalers = None
|
self.identifiers = None
|
||||||
self._cat_scalers = None
|
self._real_scalers = None
|
||||||
self._target_scaler = None
|
self._cat_scalers = None
|
||||||
self._num_classes_per_cat_input = None
|
self._target_scaler = None
|
||||||
|
self._num_classes_per_cat_input = None
|
||||||
def split_data(self, df, valid_boundary=2016, test_boundary=2018):
|
|
||||||
"""Splits data frame into training-validation-test data frames.
|
def split_data(self, df, valid_boundary=2016, test_boundary=2018):
|
||||||
|
"""Splits data frame into training-validation-test data frames.
|
||||||
This also calibrates scaling object, and transforms data for each split.
|
|
||||||
|
This also calibrates scaling object, and transforms data for each split.
|
||||||
Args:
|
|
||||||
df: Source data frame to split.
|
Args:
|
||||||
valid_boundary: Starting year for validation data
|
df: Source data frame to split.
|
||||||
test_boundary: Starting year for test data
|
valid_boundary: Starting year for validation data
|
||||||
|
test_boundary: Starting year for test data
|
||||||
Returns:
|
|
||||||
Tuple of transformed (train, valid, test) data.
|
Returns:
|
||||||
"""
|
Tuple of transformed (train, valid, test) data.
|
||||||
|
"""
|
||||||
print('Formatting train-valid-test splits.')
|
|
||||||
|
print("Formatting train-valid-test splits.")
|
||||||
index = df['year']
|
|
||||||
train = df.loc[index < valid_boundary]
|
index = df["year"]
|
||||||
valid = df.loc[(index >= valid_boundary) & (index < test_boundary)]
|
train = df.loc[index < valid_boundary]
|
||||||
test = df.loc[index >= test_boundary]
|
valid = df.loc[(index >= valid_boundary) & (index < test_boundary)]
|
||||||
|
test = df.loc[index >= test_boundary]
|
||||||
self.set_scalers(train)
|
|
||||||
|
self.set_scalers(train)
|
||||||
return (self.transform_inputs(data) for data in [train, valid, test])
|
|
||||||
|
return (self.transform_inputs(data) for data in [train, valid, test])
|
||||||
def set_scalers(self, df):
|
|
||||||
"""Calibrates scalers using the data supplied.
|
def set_scalers(self, df):
|
||||||
|
"""Calibrates scalers using the data supplied.
|
||||||
Args:
|
|
||||||
df: Data to use to calibrate scalers.
|
Args:
|
||||||
"""
|
df: Data to use to calibrate scalers.
|
||||||
print('Setting scalers with training data...')
|
"""
|
||||||
|
print("Setting scalers with training data...")
|
||||||
column_definitions = self.get_column_definition()
|
|
||||||
id_column = utils.get_single_col_by_input_type(InputTypes.ID,
|
column_definitions = self.get_column_definition()
|
||||||
column_definitions)
|
id_column = utils.get_single_col_by_input_type(InputTypes.ID, column_definitions)
|
||||||
target_column = utils.get_single_col_by_input_type(InputTypes.TARGET,
|
target_column = utils.get_single_col_by_input_type(InputTypes.TARGET, column_definitions)
|
||||||
column_definitions)
|
|
||||||
|
# Extract identifiers in case required
|
||||||
# Extract identifiers in case required
|
self.identifiers = list(df[id_column].unique())
|
||||||
self.identifiers = list(df[id_column].unique())
|
|
||||||
|
# Format real scalers
|
||||||
# Format real scalers
|
real_inputs = utils.extract_cols_from_data_type(
|
||||||
real_inputs = utils.extract_cols_from_data_type(
|
DataTypes.REAL_VALUED, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
DataTypes.REAL_VALUED, column_definitions,
|
)
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
|
||||||
|
data = df[real_inputs].values
|
||||||
data = df[real_inputs].values
|
self._real_scalers = sklearn.preprocessing.StandardScaler().fit(data)
|
||||||
self._real_scalers = sklearn.preprocessing.StandardScaler().fit(data)
|
self._target_scaler = sklearn.preprocessing.StandardScaler().fit(
|
||||||
self._target_scaler = sklearn.preprocessing.StandardScaler().fit(
|
df[[target_column]].values
|
||||||
df[[target_column]].values) # used for predictions
|
) # used for predictions
|
||||||
|
|
||||||
# Format categorical scalers
|
# Format categorical scalers
|
||||||
categorical_inputs = utils.extract_cols_from_data_type(
|
categorical_inputs = utils.extract_cols_from_data_type(
|
||||||
DataTypes.CATEGORICAL, column_definitions,
|
DataTypes.CATEGORICAL, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
)
|
||||||
|
|
||||||
categorical_scalers = {}
|
categorical_scalers = {}
|
||||||
num_classes = []
|
num_classes = []
|
||||||
for col in categorical_inputs:
|
for col in categorical_inputs:
|
||||||
# Set all to str so that we don't have mixed integer/string columns
|
# Set all to str so that we don't have mixed integer/string columns
|
||||||
srs = df[col].apply(str)
|
srs = df[col].apply(str)
|
||||||
categorical_scalers[col] = sklearn.preprocessing.LabelEncoder().fit(
|
categorical_scalers[col] = sklearn.preprocessing.LabelEncoder().fit(srs.values)
|
||||||
srs.values)
|
num_classes.append(srs.nunique())
|
||||||
num_classes.append(srs.nunique())
|
|
||||||
|
# Set categorical scaler outputs
|
||||||
# Set categorical scaler outputs
|
self._cat_scalers = categorical_scalers
|
||||||
self._cat_scalers = categorical_scalers
|
self._num_classes_per_cat_input = num_classes
|
||||||
self._num_classes_per_cat_input = num_classes
|
|
||||||
|
def transform_inputs(self, df):
|
||||||
def transform_inputs(self, df):
|
"""Performs feature transformations.
|
||||||
"""Performs feature transformations.
|
|
||||||
|
This includes both feature engineering, preprocessing and normalisation.
|
||||||
This includes both feature engineering, preprocessing and normalisation.
|
|
||||||
|
Args:
|
||||||
Args:
|
df: Data frame to transform.
|
||||||
df: Data frame to transform.
|
|
||||||
|
Returns:
|
||||||
Returns:
|
Transformed data frame.
|
||||||
Transformed data frame.
|
|
||||||
|
"""
|
||||||
"""
|
output = df.copy()
|
||||||
output = df.copy()
|
|
||||||
|
if self._real_scalers is None and self._cat_scalers is None:
|
||||||
if self._real_scalers is None and self._cat_scalers is None:
|
raise ValueError("Scalers have not been set!")
|
||||||
raise ValueError('Scalers have not been set!')
|
|
||||||
|
column_definitions = self.get_column_definition()
|
||||||
column_definitions = self.get_column_definition()
|
|
||||||
|
real_inputs = utils.extract_cols_from_data_type(
|
||||||
real_inputs = utils.extract_cols_from_data_type(
|
DataTypes.REAL_VALUED, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
DataTypes.REAL_VALUED, column_definitions,
|
)
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
categorical_inputs = utils.extract_cols_from_data_type(
|
||||||
categorical_inputs = utils.extract_cols_from_data_type(
|
DataTypes.CATEGORICAL, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
DataTypes.CATEGORICAL, column_definitions,
|
)
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
|
||||||
|
# Format real inputs
|
||||||
# Format real inputs
|
output[real_inputs] = self._real_scalers.transform(df[real_inputs].values)
|
||||||
output[real_inputs] = self._real_scalers.transform(df[real_inputs].values)
|
|
||||||
|
# Format categorical inputs
|
||||||
# Format categorical inputs
|
for col in categorical_inputs:
|
||||||
for col in categorical_inputs:
|
string_df = df[col].apply(str)
|
||||||
string_df = df[col].apply(str)
|
output[col] = self._cat_scalers[col].transform(string_df)
|
||||||
output[col] = self._cat_scalers[col].transform(string_df)
|
|
||||||
|
return output
|
||||||
return output
|
|
||||||
|
def format_predictions(self, predictions):
|
||||||
def format_predictions(self, predictions):
|
"""Reverts any normalisation to give predictions in original scale.
|
||||||
"""Reverts any normalisation to give predictions in original scale.
|
|
||||||
|
Args:
|
||||||
Args:
|
predictions: Dataframe of model predictions.
|
||||||
predictions: Dataframe of model predictions.
|
|
||||||
|
Returns:
|
||||||
Returns:
|
Data frame of unnormalised predictions.
|
||||||
Data frame of unnormalised predictions.
|
"""
|
||||||
"""
|
output = predictions.copy()
|
||||||
output = predictions.copy()
|
|
||||||
|
column_names = predictions.columns
|
||||||
column_names = predictions.columns
|
|
||||||
|
for col in column_names:
|
||||||
for col in column_names:
|
if col not in {"forecast_time", "identifier"}:
|
||||||
if col not in {'forecast_time', 'identifier'}:
|
output[col] = self._target_scaler.inverse_transform(predictions[col])
|
||||||
output[col] = self._target_scaler.inverse_transform(predictions[col])
|
|
||||||
|
return output
|
||||||
return output
|
|
||||||
|
# Default params
|
||||||
# Default params
|
def get_fixed_params(self):
|
||||||
def get_fixed_params(self):
|
"""Returns fixed model parameters for experiments."""
|
||||||
"""Returns fixed model parameters for experiments."""
|
|
||||||
|
fixed_params = {
|
||||||
fixed_params = {
|
"total_time_steps": 16 + 6,
|
||||||
'total_time_steps': 16 + 6,
|
"num_encoder_steps": 16,
|
||||||
'num_encoder_steps': 16,
|
"num_epochs": 100,
|
||||||
'num_epochs': 100,
|
"early_stopping_patience": 5,
|
||||||
'early_stopping_patience': 5,
|
"multiprocessing_workers": 5,
|
||||||
'multiprocessing_workers': 5,
|
}
|
||||||
}
|
|
||||||
|
return fixed_params
|
||||||
return fixed_params
|
|
||||||
|
def get_default_model_params(self):
|
||||||
def get_default_model_params(self):
|
"""Returns default optimised model parameters."""
|
||||||
"""Returns default optimised model parameters."""
|
|
||||||
|
model_params = {
|
||||||
model_params = {
|
"dropout_rate": 0.3,
|
||||||
'dropout_rate': 0.3,
|
"hidden_layer_size": 160,
|
||||||
'hidden_layer_size': 160,
|
"learning_rate": 0.01,
|
||||||
'learning_rate': 0.01,
|
"minibatch_size": 64,
|
||||||
'minibatch_size': 64,
|
"max_gradient_norm": 0.01,
|
||||||
'max_gradient_norm': 0.01,
|
"num_heads": 1,
|
||||||
'num_heads': 1,
|
"stack_size": 1,
|
||||||
'stack_size': 1
|
}
|
||||||
}
|
|
||||||
|
return model_params
|
||||||
return model_params
|
|
||||||
|
|||||||
@@ -1,117 +1,117 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
# Copyright 2020 The Google Research Authors.
|
# Copyright 2020 The Google Research Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
#
|
#
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
#
|
#
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Lint as: python3
|
# Lint as: python3
|
||||||
"""Custom formatting functions for Traffic dataset.
|
"""Custom formatting functions for Traffic dataset.
|
||||||
|
|
||||||
Defines dataset specific column definitions and data transformations. This also
|
Defines dataset specific column definitions and data transformations. This also
|
||||||
performs z-score normalization across the entire dataset, hence re-uses most of
|
performs z-score normalization across the entire dataset, hence re-uses most of
|
||||||
the same functions as volatility.
|
the same functions as volatility.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import data_formatters.base
|
import data_formatters.base
|
||||||
import data_formatters.volatility
|
import data_formatters.volatility
|
||||||
|
|
||||||
VolatilityFormatter = data_formatters.volatility.VolatilityFormatter
|
VolatilityFormatter = data_formatters.volatility.VolatilityFormatter
|
||||||
DataTypes = data_formatters.base.DataTypes
|
DataTypes = data_formatters.base.DataTypes
|
||||||
InputTypes = data_formatters.base.InputTypes
|
InputTypes = data_formatters.base.InputTypes
|
||||||
|
|
||||||
|
|
||||||
class TrafficFormatter(VolatilityFormatter):
|
class TrafficFormatter(VolatilityFormatter):
|
||||||
"""Defines and formats data for the traffic dataset.
|
"""Defines and formats data for the traffic dataset.
|
||||||
|
|
||||||
This also performs z-score normalization across the entire dataset, hence
|
This also performs z-score normalization across the entire dataset, hence
|
||||||
re-uses most of the same functions as volatility.
|
re-uses most of the same functions as volatility.
|
||||||
|
|
||||||
Attributes:
|
Attributes:
|
||||||
column_definition: Defines input and data type of column used in the
|
column_definition: Defines input and data type of column used in the
|
||||||
experiment.
|
experiment.
|
||||||
identifiers: Entity identifiers used in experiments.
|
identifiers: Entity identifiers used in experiments.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_column_definition = [
|
_column_definition = [
|
||||||
('id', DataTypes.REAL_VALUED, InputTypes.ID),
|
("id", DataTypes.REAL_VALUED, InputTypes.ID),
|
||||||
('hours_from_start', DataTypes.REAL_VALUED, InputTypes.TIME),
|
("hours_from_start", DataTypes.REAL_VALUED, InputTypes.TIME),
|
||||||
('values', DataTypes.REAL_VALUED, InputTypes.TARGET),
|
("values", DataTypes.REAL_VALUED, InputTypes.TARGET),
|
||||||
('time_on_day', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
("time_on_day", DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
||||||
('day_of_week', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
("day_of_week", DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
||||||
('hours_from_start', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
("hours_from_start", DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
||||||
('categorical_id', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
("categorical_id", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
]
|
]
|
||||||
|
|
||||||
def split_data(self, df, valid_boundary=151, test_boundary=166):
|
def split_data(self, df, valid_boundary=151, test_boundary=166):
|
||||||
"""Splits data frame into training-validation-test data frames.
|
"""Splits data frame into training-validation-test data frames.
|
||||||
|
|
||||||
This also calibrates scaling object, and transforms data for each split.
|
This also calibrates scaling object, and transforms data for each split.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
df: Source data frame to split.
|
df: Source data frame to split.
|
||||||
valid_boundary: Starting year for validation data
|
valid_boundary: Starting year for validation data
|
||||||
test_boundary: Starting year for test data
|
test_boundary: Starting year for test data
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of transformed (train, valid, test) data.
|
Tuple of transformed (train, valid, test) data.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
print('Formatting train-valid-test splits.')
|
print("Formatting train-valid-test splits.")
|
||||||
|
|
||||||
index = df['sensor_day']
|
index = df["sensor_day"]
|
||||||
train = df.loc[index < valid_boundary]
|
train = df.loc[index < valid_boundary]
|
||||||
valid = df.loc[(index >= valid_boundary - 7) & (index < test_boundary)]
|
valid = df.loc[(index >= valid_boundary - 7) & (index < test_boundary)]
|
||||||
test = df.loc[index >= test_boundary - 7]
|
test = df.loc[index >= test_boundary - 7]
|
||||||
|
|
||||||
self.set_scalers(train)
|
self.set_scalers(train)
|
||||||
|
|
||||||
return (self.transform_inputs(data) for data in [train, valid, test])
|
return (self.transform_inputs(data) for data in [train, valid, test])
|
||||||
|
|
||||||
# Default params
|
# Default params
|
||||||
def get_fixed_params(self):
|
def get_fixed_params(self):
|
||||||
"""Returns fixed model parameters for experiments."""
|
"""Returns fixed model parameters for experiments."""
|
||||||
|
|
||||||
fixed_params = {
|
fixed_params = {
|
||||||
'total_time_steps': 8 * 24,
|
"total_time_steps": 8 * 24,
|
||||||
'num_encoder_steps': 7 * 24,
|
"num_encoder_steps": 7 * 24,
|
||||||
'num_epochs': 100,
|
"num_epochs": 100,
|
||||||
'early_stopping_patience': 5,
|
"early_stopping_patience": 5,
|
||||||
'multiprocessing_workers': 5
|
"multiprocessing_workers": 5,
|
||||||
}
|
}
|
||||||
|
|
||||||
return fixed_params
|
return fixed_params
|
||||||
|
|
||||||
def get_default_model_params(self):
|
def get_default_model_params(self):
|
||||||
"""Returns default optimised model parameters."""
|
"""Returns default optimised model parameters."""
|
||||||
|
|
||||||
model_params = {
|
model_params = {
|
||||||
'dropout_rate': 0.3,
|
"dropout_rate": 0.3,
|
||||||
'hidden_layer_size': 320,
|
"hidden_layer_size": 320,
|
||||||
'learning_rate': 0.001,
|
"learning_rate": 0.001,
|
||||||
'minibatch_size': 128,
|
"minibatch_size": 128,
|
||||||
'max_gradient_norm': 100.,
|
"max_gradient_norm": 100.0,
|
||||||
'num_heads': 4,
|
"num_heads": 4,
|
||||||
'stack_size': 1
|
"stack_size": 1,
|
||||||
}
|
}
|
||||||
|
|
||||||
return model_params
|
return model_params
|
||||||
|
|
||||||
def get_num_samples_for_calibration(self):
|
def get_num_samples_for_calibration(self):
|
||||||
"""Gets the default number of training and validation samples.
|
"""Gets the default number of training and validation samples.
|
||||||
|
|
||||||
Use to sub-sample the data for network calibration and a value of -1 uses
|
Use to sub-sample the data for network calibration and a value of -1 uses
|
||||||
all available samples.
|
all available samples.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (training samples, validation samples)
|
Tuple of (training samples, validation samples)
|
||||||
"""
|
"""
|
||||||
return 450000, 50000
|
return 450000, 50000
|
||||||
|
|||||||
@@ -1,214 +1,212 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
# Copyright 2020 The Google Research Authors.
|
# Copyright 2020 The Google Research Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
#
|
#
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
#
|
#
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Lint as: python3
|
# Lint as: python3
|
||||||
"""Custom formatting functions for Volatility dataset.
|
"""Custom formatting functions for Volatility dataset.
|
||||||
|
|
||||||
Defines dataset specific column definitions and data transformations.
|
Defines dataset specific column definitions and data transformations.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import data_formatters.base
|
import data_formatters.base
|
||||||
import libs.utils as utils
|
import libs.utils as utils
|
||||||
import sklearn.preprocessing
|
import sklearn.preprocessing
|
||||||
|
|
||||||
GenericDataFormatter = data_formatters.base.GenericDataFormatter
|
GenericDataFormatter = data_formatters.base.GenericDataFormatter
|
||||||
DataTypes = data_formatters.base.DataTypes
|
DataTypes = data_formatters.base.DataTypes
|
||||||
InputTypes = data_formatters.base.InputTypes
|
InputTypes = data_formatters.base.InputTypes
|
||||||
|
|
||||||
|
|
||||||
class VolatilityFormatter(GenericDataFormatter):
|
class VolatilityFormatter(GenericDataFormatter):
|
||||||
"""Defines and formats data for the volatility dataset.
|
"""Defines and formats data for the volatility dataset.
|
||||||
|
|
||||||
Attributes:
|
Attributes:
|
||||||
column_definition: Defines input and data type of column used in the
|
column_definition: Defines input and data type of column used in the
|
||||||
experiment.
|
experiment.
|
||||||
identifiers: Entity identifiers used in experiments.
|
identifiers: Entity identifiers used in experiments.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_column_definition = [
|
_column_definition = [
|
||||||
('Symbol', DataTypes.CATEGORICAL, InputTypes.ID),
|
("Symbol", DataTypes.CATEGORICAL, InputTypes.ID),
|
||||||
('date', DataTypes.DATE, InputTypes.TIME),
|
("date", DataTypes.DATE, InputTypes.TIME),
|
||||||
('log_vol', DataTypes.REAL_VALUED, InputTypes.TARGET),
|
("log_vol", DataTypes.REAL_VALUED, InputTypes.TARGET),
|
||||||
('open_to_close', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
("open_to_close", DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),
|
||||||
('days_from_start', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
("days_from_start", DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),
|
||||||
('day_of_week', DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
("day_of_week", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
||||||
('day_of_month', DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
("day_of_month", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
||||||
('week_of_year', DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
("week_of_year", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
||||||
('month', DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
("month", DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),
|
||||||
('Region', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
("Region", DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),
|
||||||
]
|
]
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
"""Initialises formatter."""
|
"""Initialises formatter."""
|
||||||
|
|
||||||
self.identifiers = None
|
self.identifiers = None
|
||||||
self._real_scalers = None
|
self._real_scalers = None
|
||||||
self._cat_scalers = None
|
self._cat_scalers = None
|
||||||
self._target_scaler = None
|
self._target_scaler = None
|
||||||
self._num_classes_per_cat_input = None
|
self._num_classes_per_cat_input = None
|
||||||
|
|
||||||
def split_data(self, df, valid_boundary=2016, test_boundary=2018):
|
def split_data(self, df, valid_boundary=2016, test_boundary=2018):
|
||||||
"""Splits data frame into training-validation-test data frames.
|
"""Splits data frame into training-validation-test data frames.
|
||||||
|
|
||||||
This also calibrates scaling object, and transforms data for each split.
|
This also calibrates scaling object, and transforms data for each split.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
df: Source data frame to split.
|
df: Source data frame to split.
|
||||||
valid_boundary: Starting year for validation data
|
valid_boundary: Starting year for validation data
|
||||||
test_boundary: Starting year for test data
|
test_boundary: Starting year for test data
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of transformed (train, valid, test) data.
|
Tuple of transformed (train, valid, test) data.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
print('Formatting train-valid-test splits.')
|
print("Formatting train-valid-test splits.")
|
||||||
|
|
||||||
index = df['year']
|
index = df["year"]
|
||||||
train = df.loc[index < valid_boundary]
|
train = df.loc[index < valid_boundary]
|
||||||
valid = df.loc[(index >= valid_boundary) & (index < test_boundary)]
|
valid = df.loc[(index >= valid_boundary) & (index < test_boundary)]
|
||||||
test = df.loc[index >= test_boundary]
|
test = df.loc[index >= test_boundary]
|
||||||
|
|
||||||
self.set_scalers(train)
|
self.set_scalers(train)
|
||||||
|
|
||||||
return (self.transform_inputs(data) for data in [train, valid, test])
|
return (self.transform_inputs(data) for data in [train, valid, test])
|
||||||
|
|
||||||
def set_scalers(self, df):
|
def set_scalers(self, df):
|
||||||
"""Calibrates scalers using the data supplied.
|
"""Calibrates scalers using the data supplied.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
df: Data to use to calibrate scalers.
|
df: Data to use to calibrate scalers.
|
||||||
"""
|
"""
|
||||||
print('Setting scalers with training data...')
|
print("Setting scalers with training data...")
|
||||||
|
|
||||||
column_definitions = self.get_column_definition()
|
column_definitions = self.get_column_definition()
|
||||||
id_column = utils.get_single_col_by_input_type(InputTypes.ID,
|
id_column = utils.get_single_col_by_input_type(InputTypes.ID, column_definitions)
|
||||||
column_definitions)
|
target_column = utils.get_single_col_by_input_type(InputTypes.TARGET, column_definitions)
|
||||||
target_column = utils.get_single_col_by_input_type(InputTypes.TARGET,
|
|
||||||
column_definitions)
|
# Extract identifiers in case required
|
||||||
|
self.identifiers = list(df[id_column].unique())
|
||||||
# Extract identifiers in case required
|
|
||||||
self.identifiers = list(df[id_column].unique())
|
# Format real scalers
|
||||||
|
real_inputs = utils.extract_cols_from_data_type(
|
||||||
# Format real scalers
|
DataTypes.REAL_VALUED, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
real_inputs = utils.extract_cols_from_data_type(
|
)
|
||||||
DataTypes.REAL_VALUED, column_definitions,
|
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
data = df[real_inputs].values
|
||||||
|
self._real_scalers = sklearn.preprocessing.StandardScaler().fit(data)
|
||||||
data = df[real_inputs].values
|
self._target_scaler = sklearn.preprocessing.StandardScaler().fit(
|
||||||
self._real_scalers = sklearn.preprocessing.StandardScaler().fit(data)
|
df[[target_column]].values
|
||||||
self._target_scaler = sklearn.preprocessing.StandardScaler().fit(
|
) # used for predictions
|
||||||
df[[target_column]].values) # used for predictions
|
|
||||||
|
# Format categorical scalers
|
||||||
# Format categorical scalers
|
categorical_inputs = utils.extract_cols_from_data_type(
|
||||||
categorical_inputs = utils.extract_cols_from_data_type(
|
DataTypes.CATEGORICAL, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
DataTypes.CATEGORICAL, column_definitions,
|
)
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
|
||||||
|
categorical_scalers = {}
|
||||||
categorical_scalers = {}
|
num_classes = []
|
||||||
num_classes = []
|
for col in categorical_inputs:
|
||||||
for col in categorical_inputs:
|
# Set all to str so that we don't have mixed integer/string columns
|
||||||
# Set all to str so that we don't have mixed integer/string columns
|
srs = df[col].apply(str)
|
||||||
srs = df[col].apply(str)
|
categorical_scalers[col] = sklearn.preprocessing.LabelEncoder().fit(srs.values)
|
||||||
categorical_scalers[col] = sklearn.preprocessing.LabelEncoder().fit(
|
num_classes.append(srs.nunique())
|
||||||
srs.values)
|
|
||||||
num_classes.append(srs.nunique())
|
# Set categorical scaler outputs
|
||||||
|
self._cat_scalers = categorical_scalers
|
||||||
# Set categorical scaler outputs
|
self._num_classes_per_cat_input = num_classes
|
||||||
self._cat_scalers = categorical_scalers
|
|
||||||
self._num_classes_per_cat_input = num_classes
|
def transform_inputs(self, df):
|
||||||
|
"""Performs feature transformations.
|
||||||
def transform_inputs(self, df):
|
|
||||||
"""Performs feature transformations.
|
This includes both feature engineering, preprocessing and normalisation.
|
||||||
|
|
||||||
This includes both feature engineering, preprocessing and normalisation.
|
Args:
|
||||||
|
df: Data frame to transform.
|
||||||
Args:
|
|
||||||
df: Data frame to transform.
|
Returns:
|
||||||
|
Transformed data frame.
|
||||||
Returns:
|
|
||||||
Transformed data frame.
|
"""
|
||||||
|
output = df.copy()
|
||||||
"""
|
|
||||||
output = df.copy()
|
if self._real_scalers is None and self._cat_scalers is None:
|
||||||
|
raise ValueError("Scalers have not been set!")
|
||||||
if self._real_scalers is None and self._cat_scalers is None:
|
|
||||||
raise ValueError('Scalers have not been set!')
|
column_definitions = self.get_column_definition()
|
||||||
|
|
||||||
column_definitions = self.get_column_definition()
|
real_inputs = utils.extract_cols_from_data_type(
|
||||||
|
DataTypes.REAL_VALUED, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
real_inputs = utils.extract_cols_from_data_type(
|
)
|
||||||
DataTypes.REAL_VALUED, column_definitions,
|
categorical_inputs = utils.extract_cols_from_data_type(
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
DataTypes.CATEGORICAL, column_definitions, {InputTypes.ID, InputTypes.TIME}
|
||||||
categorical_inputs = utils.extract_cols_from_data_type(
|
)
|
||||||
DataTypes.CATEGORICAL, column_definitions,
|
|
||||||
{InputTypes.ID, InputTypes.TIME})
|
# Format real inputs
|
||||||
|
output[real_inputs] = self._real_scalers.transform(df[real_inputs].values)
|
||||||
# Format real inputs
|
|
||||||
output[real_inputs] = self._real_scalers.transform(df[real_inputs].values)
|
# Format categorical inputs
|
||||||
|
for col in categorical_inputs:
|
||||||
# Format categorical inputs
|
string_df = df[col].apply(str)
|
||||||
for col in categorical_inputs:
|
output[col] = self._cat_scalers[col].transform(string_df)
|
||||||
string_df = df[col].apply(str)
|
|
||||||
output[col] = self._cat_scalers[col].transform(string_df)
|
return output
|
||||||
|
|
||||||
return output
|
def format_predictions(self, predictions):
|
||||||
|
"""Reverts any normalisation to give predictions in original scale.
|
||||||
def format_predictions(self, predictions):
|
|
||||||
"""Reverts any normalisation to give predictions in original scale.
|
Args:
|
||||||
|
predictions: Dataframe of model predictions.
|
||||||
Args:
|
|
||||||
predictions: Dataframe of model predictions.
|
Returns:
|
||||||
|
Data frame of unnormalised predictions.
|
||||||
Returns:
|
"""
|
||||||
Data frame of unnormalised predictions.
|
output = predictions.copy()
|
||||||
"""
|
|
||||||
output = predictions.copy()
|
column_names = predictions.columns
|
||||||
|
|
||||||
column_names = predictions.columns
|
for col in column_names:
|
||||||
|
if col not in {"forecast_time", "identifier"}:
|
||||||
for col in column_names:
|
output[col] = self._target_scaler.inverse_transform(predictions[col])
|
||||||
if col not in {'forecast_time', 'identifier'}:
|
|
||||||
output[col] = self._target_scaler.inverse_transform(predictions[col])
|
return output
|
||||||
|
|
||||||
return output
|
# Default params
|
||||||
|
def get_fixed_params(self):
|
||||||
# Default params
|
"""Returns fixed model parameters for experiments."""
|
||||||
def get_fixed_params(self):
|
|
||||||
"""Returns fixed model parameters for experiments."""
|
fixed_params = {
|
||||||
|
"total_time_steps": 252 + 5,
|
||||||
fixed_params = {
|
"num_encoder_steps": 252,
|
||||||
'total_time_steps': 252 + 5,
|
"num_epochs": 100,
|
||||||
'num_encoder_steps': 252,
|
"early_stopping_patience": 5,
|
||||||
'num_epochs': 100,
|
"multiprocessing_workers": 5,
|
||||||
'early_stopping_patience': 5,
|
}
|
||||||
'multiprocessing_workers': 5,
|
|
||||||
}
|
return fixed_params
|
||||||
|
|
||||||
return fixed_params
|
def get_default_model_params(self):
|
||||||
|
"""Returns default optimised model parameters."""
|
||||||
def get_default_model_params(self):
|
|
||||||
"""Returns default optimised model parameters."""
|
model_params = {
|
||||||
|
"dropout_rate": 0.3,
|
||||||
model_params = {
|
"hidden_layer_size": 160,
|
||||||
'dropout_rate': 0.3,
|
"learning_rate": 0.01,
|
||||||
'hidden_layer_size': 160,
|
"minibatch_size": 64,
|
||||||
'learning_rate': 0.01,
|
"max_gradient_norm": 0.01,
|
||||||
'minibatch_size': 64,
|
"num_heads": 1,
|
||||||
'max_gradient_norm': 0.01,
|
"stack_size": 1,
|
||||||
'num_heads': 1,
|
}
|
||||||
'stack_size': 1
|
|
||||||
}
|
return model_params
|
||||||
|
|
||||||
return model_params
|
|
||||||
|
|||||||
@@ -1,15 +1,14 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
# Copyright 2020 The Google Research Authors.
|
# Copyright 2020 The Google Research Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
#
|
#
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
#
|
#
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
|||||||
@@ -1,111 +1,107 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
# Copyright 2020 The Google Research Authors.
|
# Copyright 2020 The Google Research Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
#
|
#
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
#
|
#
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Lint as: python3
|
# Lint as: python3
|
||||||
"""Default configs for TFT experiments.
|
"""Default configs for TFT experiments.
|
||||||
|
|
||||||
Contains the default output paths for data, serialised models and predictions
|
Contains the default output paths for data, serialised models and predictions
|
||||||
for the main experiments used in the publication.
|
for the main experiments used in the publication.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import data_formatters.electricity
|
import data_formatters.electricity
|
||||||
import data_formatters.favorita
|
import data_formatters.favorita
|
||||||
import data_formatters.traffic
|
import data_formatters.traffic
|
||||||
import data_formatters.volatility
|
import data_formatters.volatility
|
||||||
import data_formatters.qlib_Alpha158
|
import data_formatters.qlib_Alpha158
|
||||||
|
|
||||||
|
|
||||||
class ExperimentConfig(object):
|
class ExperimentConfig(object):
|
||||||
"""Defines experiment configs and paths to outputs.
|
"""Defines experiment configs and paths to outputs.
|
||||||
|
|
||||||
Attributes:
|
Attributes:
|
||||||
root_folder: Root folder to contain all experimental outputs.
|
root_folder: Root folder to contain all experimental outputs.
|
||||||
experiment: Name of experiment to run.
|
experiment: Name of experiment to run.
|
||||||
data_folder: Folder to store data for experiment.
|
data_folder: Folder to store data for experiment.
|
||||||
model_folder: Folder to store serialised models.
|
model_folder: Folder to store serialised models.
|
||||||
results_folder: Folder to store results.
|
results_folder: Folder to store results.
|
||||||
data_csv_path: Path to primary data csv file used in experiment.
|
data_csv_path: Path to primary data csv file used in experiment.
|
||||||
hyperparam_iterations: Default number of random search iterations for
|
hyperparam_iterations: Default number of random search iterations for
|
||||||
experiment.
|
experiment.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
default_experiments = ['volatility', 'electricity', 'traffic', 'favorita', 'Alpha158']
|
default_experiments = ["volatility", "electricity", "traffic", "favorita", "Alpha158"]
|
||||||
|
|
||||||
def __init__(self, experiment='volatility', root_folder=None):
|
def __init__(self, experiment="volatility", root_folder=None):
|
||||||
"""Creates configs based on default experiment chosen.
|
"""Creates configs based on default experiment chosen.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
experiment: Name of experiment.
|
experiment: Name of experiment.
|
||||||
root_folder: Root folder to save all outputs of training.
|
root_folder: Root folder to save all outputs of training.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
if experiment not in self.default_experiments:
|
if experiment not in self.default_experiments:
|
||||||
raise ValueError('Unrecognised experiment={}'.format(experiment))
|
raise ValueError("Unrecognised experiment={}".format(experiment))
|
||||||
|
|
||||||
# Defines all relevant paths
|
# Defines all relevant paths
|
||||||
if root_folder is None:
|
if root_folder is None:
|
||||||
root_folder = os.path.join(
|
root_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "outputs")
|
||||||
os.path.dirname(os.path.realpath(__file__)), '..', 'outputs')
|
print("Using root folder {}".format(root_folder))
|
||||||
print('Using root folder {}'.format(root_folder))
|
|
||||||
|
self.root_folder = root_folder
|
||||||
self.root_folder = root_folder
|
self.experiment = experiment
|
||||||
self.experiment = experiment
|
self.data_folder = os.path.join(root_folder, "data", experiment)
|
||||||
self.data_folder = os.path.join(root_folder, 'data', experiment)
|
self.model_folder = os.path.join(root_folder, "saved_models", experiment)
|
||||||
self.model_folder = os.path.join(root_folder, 'saved_models', experiment)
|
self.results_folder = os.path.join(root_folder, "results", experiment)
|
||||||
self.results_folder = os.path.join(root_folder, 'results', experiment)
|
|
||||||
|
# Creates folders if they don't exist
|
||||||
# Creates folders if they don't exist
|
for relevant_directory in [self.root_folder, self.data_folder, self.model_folder, self.results_folder]:
|
||||||
for relevant_directory in [
|
if not os.path.exists(relevant_directory):
|
||||||
self.root_folder, self.data_folder, self.model_folder,
|
os.makedirs(relevant_directory)
|
||||||
self.results_folder
|
|
||||||
]:
|
@property
|
||||||
if not os.path.exists(relevant_directory):
|
def data_csv_path(self):
|
||||||
os.makedirs(relevant_directory)
|
csv_map = {
|
||||||
|
"volatility": "formatted_omi_vol.csv",
|
||||||
@property
|
"electricity": "hourly_electricity.csv",
|
||||||
def data_csv_path(self):
|
"traffic": "hourly_data.csv",
|
||||||
csv_map = {
|
"favorita": "favorita_consolidated.csv",
|
||||||
'volatility': 'formatted_omi_vol.csv',
|
"Alpha158": "Alpha158.csv",
|
||||||
'electricity': 'hourly_electricity.csv',
|
}
|
||||||
'traffic': 'hourly_data.csv',
|
|
||||||
'favorita': 'favorita_consolidated.csv',
|
return os.path.join(self.data_folder, csv_map[self.experiment])
|
||||||
'Alpha158': 'Alpha158.csv',
|
|
||||||
}
|
@property
|
||||||
|
def hyperparam_iterations(self):
|
||||||
return os.path.join(self.data_folder, csv_map[self.experiment])
|
|
||||||
|
return 240 if self.experiment == "volatility" else 60
|
||||||
@property
|
|
||||||
def hyperparam_iterations(self):
|
def make_data_formatter(self):
|
||||||
|
"""Gets a data formatter object for experiment.
|
||||||
return 240 if self.experiment == 'volatility' else 60
|
|
||||||
|
Returns:
|
||||||
def make_data_formatter(self):
|
Default DataFormatter per experiment.
|
||||||
"""Gets a data formatter object for experiment.
|
"""
|
||||||
|
|
||||||
Returns:
|
data_formatter_class = {
|
||||||
Default DataFormatter per experiment.
|
"volatility": data_formatters.volatility.VolatilityFormatter,
|
||||||
"""
|
"electricity": data_formatters.electricity.ElectricityFormatter,
|
||||||
|
"traffic": data_formatters.traffic.TrafficFormatter,
|
||||||
data_formatter_class = {
|
"favorita": data_formatters.favorita.FavoritaFormatter,
|
||||||
'volatility': data_formatters.volatility.VolatilityFormatter,
|
"Alpha158": data_formatters.qlib_Alpha158.Alpha158Formatter,
|
||||||
'electricity': data_formatters.electricity.ElectricityFormatter,
|
}
|
||||||
'traffic': data_formatters.traffic.TrafficFormatter,
|
|
||||||
'favorita': data_formatters.favorita.FavoritaFormatter,
|
return data_formatter_class[self.experiment]()
|
||||||
'Alpha158': data_formatters.qlib_Alpha158.Alpha158Formatter,
|
|
||||||
}
|
|
||||||
|
|
||||||
return data_formatter_class[self.experiment]()
|
|
||||||
|
|||||||
@@ -1,15 +1,14 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
# Copyright 2020 The Google Research Authors.
|
# Copyright 2020 The Google Research Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
#
|
#
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
#
|
#
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
|||||||
@@ -1,438 +1,430 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
# Copyright 2020 The Google Research Authors.
|
# Copyright 2020 The Google Research Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
#
|
#
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
#
|
#
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Lint as: python3
|
# Lint as: python3
|
||||||
"""Classes used for hyperparameter optimisation.
|
"""Classes used for hyperparameter optimisation.
|
||||||
|
|
||||||
Two main classes exist:
|
Two main classes exist:
|
||||||
1) HyperparamOptManager used for optimisation on a single machine/GPU.
|
1) HyperparamOptManager used for optimisation on a single machine/GPU.
|
||||||
2) DistributedHyperparamOptManager for multiple GPUs on different machines.
|
2) DistributedHyperparamOptManager for multiple GPUs on different machines.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import absolute_import
|
from __future__ import absolute_import
|
||||||
from __future__ import division
|
from __future__ import division
|
||||||
from __future__ import print_function
|
from __future__ import print_function
|
||||||
|
|
||||||
import collections
|
import collections
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import libs.utils as utils
|
import libs.utils as utils
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
|
|
||||||
Deque = collections.deque
|
Deque = collections.deque
|
||||||
|
|
||||||
|
|
||||||
class HyperparamOptManager:
|
class HyperparamOptManager:
|
||||||
"""Manages hyperparameter optimisation using random search for a single GPU.
|
"""Manages hyperparameter optimisation using random search for a single GPU.
|
||||||
|
|
||||||
Attributes:
|
Attributes:
|
||||||
param_ranges: Discrete hyperparameter range for random search.
|
param_ranges: Discrete hyperparameter range for random search.
|
||||||
results: Dataframe of validation results.
|
results: Dataframe of validation results.
|
||||||
fixed_params: Fixed model parameters per experiment.
|
fixed_params: Fixed model parameters per experiment.
|
||||||
saved_params: Dataframe of parameters trained.
|
saved_params: Dataframe of parameters trained.
|
||||||
best_score: Minimum validation loss observed thus far.
|
best_score: Minimum validation loss observed thus far.
|
||||||
optimal_name: Key to best configuration.
|
optimal_name: Key to best configuration.
|
||||||
hyperparam_folder: Where to save optimisation outputs.
|
hyperparam_folder: Where to save optimisation outputs.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self,
|
def __init__(self, param_ranges, fixed_params, model_folder, override_w_fixed_params=True):
|
||||||
param_ranges,
|
"""Instantiates model.
|
||||||
fixed_params,
|
|
||||||
model_folder,
|
Args:
|
||||||
override_w_fixed_params=True):
|
param_ranges: Discrete hyperparameter range for random search.
|
||||||
"""Instantiates model.
|
fixed_params: Fixed model parameters per experiment.
|
||||||
|
model_folder: Folder to store optimisation artifacts.
|
||||||
Args:
|
override_w_fixed_params: Whether to override serialsed fixed model
|
||||||
param_ranges: Discrete hyperparameter range for random search.
|
parameters with new supplied values.
|
||||||
fixed_params: Fixed model parameters per experiment.
|
"""
|
||||||
model_folder: Folder to store optimisation artifacts.
|
|
||||||
override_w_fixed_params: Whether to override serialsed fixed model
|
self.param_ranges = param_ranges
|
||||||
parameters with new supplied values.
|
|
||||||
"""
|
self._max_tries = 1000
|
||||||
|
self.results = pd.DataFrame()
|
||||||
self.param_ranges = param_ranges
|
self.fixed_params = fixed_params
|
||||||
|
self.saved_params = pd.DataFrame()
|
||||||
self._max_tries = 1000
|
|
||||||
self.results = pd.DataFrame()
|
self.best_score = np.Inf
|
||||||
self.fixed_params = fixed_params
|
self.optimal_name = ""
|
||||||
self.saved_params = pd.DataFrame()
|
|
||||||
|
# Setup
|
||||||
self.best_score = np.Inf
|
# Create folder for saving if its not there
|
||||||
self.optimal_name = ""
|
self.hyperparam_folder = model_folder
|
||||||
|
utils.create_folder_if_not_exist(self.hyperparam_folder)
|
||||||
# Setup
|
|
||||||
# Create folder for saving if its not there
|
self._override_w_fixed_params = override_w_fixed_params
|
||||||
self.hyperparam_folder = model_folder
|
|
||||||
utils.create_folder_if_not_exist(self.hyperparam_folder)
|
def load_results(self):
|
||||||
|
"""Loads results from previous hyperparameter optimisation.
|
||||||
self._override_w_fixed_params = override_w_fixed_params
|
|
||||||
|
Returns:
|
||||||
def load_results(self):
|
A boolean indicating if previous results can be loaded.
|
||||||
"""Loads results from previous hyperparameter optimisation.
|
"""
|
||||||
|
print("Loading results from", self.hyperparam_folder)
|
||||||
Returns:
|
|
||||||
A boolean indicating if previous results can be loaded.
|
results_file = os.path.join(self.hyperparam_folder, "results.csv")
|
||||||
"""
|
params_file = os.path.join(self.hyperparam_folder, "params.csv")
|
||||||
print("Loading results from", self.hyperparam_folder)
|
|
||||||
|
if os.path.exists(results_file) and os.path.exists(params_file):
|
||||||
results_file = os.path.join(self.hyperparam_folder, "results.csv")
|
|
||||||
params_file = os.path.join(self.hyperparam_folder, "params.csv")
|
self.results = pd.read_csv(results_file, index_col=0)
|
||||||
|
self.saved_params = pd.read_csv(params_file, index_col=0)
|
||||||
if os.path.exists(results_file) and os.path.exists(params_file):
|
|
||||||
|
if not self.results.empty:
|
||||||
self.results = pd.read_csv(results_file, index_col=0)
|
self.results.at["loss"] = self.results.loc["loss"].apply(float)
|
||||||
self.saved_params = pd.read_csv(params_file, index_col=0)
|
self.best_score = self.results.loc["loss"].min()
|
||||||
|
|
||||||
if not self.results.empty:
|
is_optimal = self.results.loc["loss"] == self.best_score
|
||||||
self.results.at["loss"] = self.results.loc["loss"].apply(float)
|
self.optimal_name = self.results.T[is_optimal].index[0]
|
||||||
self.best_score = self.results.loc["loss"].min()
|
|
||||||
|
return True
|
||||||
is_optimal = self.results.loc["loss"] == self.best_score
|
|
||||||
self.optimal_name = self.results.T[is_optimal].index[0]
|
return False
|
||||||
|
|
||||||
return True
|
def _get_params_from_name(self, name):
|
||||||
|
"""Returns previously saved parameters given a key."""
|
||||||
return False
|
params = self.saved_params
|
||||||
|
|
||||||
def _get_params_from_name(self, name):
|
selected_params = dict(params[name])
|
||||||
"""Returns previously saved parameters given a key."""
|
|
||||||
params = self.saved_params
|
if self._override_w_fixed_params:
|
||||||
|
for k in self.fixed_params:
|
||||||
selected_params = dict(params[name])
|
selected_params[k] = self.fixed_params[k]
|
||||||
|
|
||||||
if self._override_w_fixed_params:
|
return selected_params
|
||||||
for k in self.fixed_params:
|
|
||||||
selected_params[k] = self.fixed_params[k]
|
def get_best_params(self):
|
||||||
|
"""Returns the optimal hyperparameters thus far."""
|
||||||
return selected_params
|
|
||||||
|
optimal_name = self.optimal_name
|
||||||
def get_best_params(self):
|
|
||||||
"""Returns the optimal hyperparameters thus far."""
|
return self._get_params_from_name(optimal_name)
|
||||||
|
|
||||||
optimal_name = self.optimal_name
|
def clear(self):
|
||||||
|
"""Clears all previous results and saved parameters."""
|
||||||
return self._get_params_from_name(optimal_name)
|
shutil.rmtree(self.hyperparam_folder)
|
||||||
|
os.makedirs(self.hyperparam_folder)
|
||||||
def clear(self):
|
self.results = pd.DataFrame()
|
||||||
"""Clears all previous results and saved parameters."""
|
self.saved_params = pd.DataFrame()
|
||||||
shutil.rmtree(self.hyperparam_folder)
|
|
||||||
os.makedirs(self.hyperparam_folder)
|
def _check_params(self, params):
|
||||||
self.results = pd.DataFrame()
|
"""Checks that parameter map is properly defined."""
|
||||||
self.saved_params = pd.DataFrame()
|
|
||||||
|
valid_fields = list(self.param_ranges.keys()) + list(self.fixed_params.keys())
|
||||||
def _check_params(self, params):
|
invalid_fields = [k for k in params if k not in valid_fields]
|
||||||
"""Checks that parameter map is properly defined."""
|
missing_fields = [k for k in valid_fields if k not in params]
|
||||||
|
|
||||||
valid_fields = list(self.param_ranges.keys()) + list(
|
if invalid_fields:
|
||||||
self.fixed_params.keys())
|
raise ValueError("Invalid Fields Found {} - Valid ones are {}".format(invalid_fields, valid_fields))
|
||||||
invalid_fields = [k for k in params if k not in valid_fields]
|
if missing_fields:
|
||||||
missing_fields = [k for k in valid_fields if k not in params]
|
raise ValueError("Missing Fields Found {} - Valid ones are {}".format(missing_fields, valid_fields))
|
||||||
|
|
||||||
if invalid_fields:
|
def _get_name(self, params):
|
||||||
raise ValueError("Invalid Fields Found {} - Valid ones are {}".format(
|
"""Returns a unique key for the supplied set of params."""
|
||||||
invalid_fields, valid_fields))
|
|
||||||
if missing_fields:
|
self._check_params(params)
|
||||||
raise ValueError("Missing Fields Found {} - Valid ones are {}".format(
|
|
||||||
missing_fields, valid_fields))
|
fields = list(params.keys())
|
||||||
|
fields.sort()
|
||||||
def _get_name(self, params):
|
|
||||||
"""Returns a unique key for the supplied set of params."""
|
return "_".join([str(params[k]) for k in fields])
|
||||||
|
|
||||||
self._check_params(params)
|
def get_next_parameters(self, ranges_to_skip=None):
|
||||||
|
"""Returns the next set of parameters to optimise.
|
||||||
fields = list(params.keys())
|
|
||||||
fields.sort()
|
Args:
|
||||||
|
ranges_to_skip: Explicitly defines a set of keys to skip.
|
||||||
return "_".join([str(params[k]) for k in fields])
|
"""
|
||||||
|
if ranges_to_skip is None:
|
||||||
def get_next_parameters(self, ranges_to_skip=None):
|
ranges_to_skip = set(self.results.index)
|
||||||
"""Returns the next set of parameters to optimise.
|
|
||||||
|
if not isinstance(self.param_ranges, dict):
|
||||||
Args:
|
raise ValueError("Only works for random search!")
|
||||||
ranges_to_skip: Explicitly defines a set of keys to skip.
|
|
||||||
"""
|
param_range_keys = list(self.param_ranges.keys())
|
||||||
if ranges_to_skip is None:
|
param_range_keys.sort()
|
||||||
ranges_to_skip = set(self.results.index)
|
|
||||||
|
def _get_next():
|
||||||
if not isinstance(self.param_ranges, dict):
|
"""Returns next hyperparameter set per try."""
|
||||||
raise ValueError("Only works for random search!")
|
|
||||||
|
parameters = {k: np.random.choice(self.param_ranges[k]) for k in param_range_keys}
|
||||||
param_range_keys = list(self.param_ranges.keys())
|
|
||||||
param_range_keys.sort()
|
# Adds fixed params
|
||||||
|
for k in self.fixed_params:
|
||||||
def _get_next():
|
parameters[k] = self.fixed_params[k]
|
||||||
"""Returns next hyperparameter set per try."""
|
|
||||||
|
return parameters
|
||||||
parameters = {
|
|
||||||
k: np.random.choice(self.param_ranges[k]) for k in param_range_keys
|
for _ in range(self._max_tries):
|
||||||
}
|
|
||||||
|
parameters = _get_next()
|
||||||
# Adds fixed params
|
name = self._get_name(parameters)
|
||||||
for k in self.fixed_params:
|
|
||||||
parameters[k] = self.fixed_params[k]
|
if name not in ranges_to_skip:
|
||||||
|
return parameters
|
||||||
return parameters
|
|
||||||
|
raise ValueError("Exceeded max number of hyperparameter searches!!")
|
||||||
for _ in range(self._max_tries):
|
|
||||||
|
def update_score(self, parameters, loss, model, info=""):
|
||||||
parameters = _get_next()
|
"""Updates the results from last optimisation run.
|
||||||
name = self._get_name(parameters)
|
|
||||||
|
Args:
|
||||||
if name not in ranges_to_skip:
|
parameters: Hyperparameters used in optimisation.
|
||||||
return parameters
|
loss: Validation loss obtained.
|
||||||
|
model: Model to serialised if required.
|
||||||
raise ValueError("Exceeded max number of hyperparameter searches!!")
|
info: Any ancillary information to tag on to results.
|
||||||
|
|
||||||
def update_score(self, parameters, loss, model, info=""):
|
Returns:
|
||||||
"""Updates the results from last optimisation run.
|
Boolean flag indicating if the model is the best seen so far.
|
||||||
|
"""
|
||||||
Args:
|
|
||||||
parameters: Hyperparameters used in optimisation.
|
if np.isnan(loss):
|
||||||
loss: Validation loss obtained.
|
loss = np.Inf
|
||||||
model: Model to serialised if required.
|
|
||||||
info: Any ancillary information to tag on to results.
|
if not os.path.isdir(self.hyperparam_folder):
|
||||||
|
os.makedirs(self.hyperparam_folder)
|
||||||
Returns:
|
|
||||||
Boolean flag indicating if the model is the best seen so far.
|
name = self._get_name(parameters)
|
||||||
"""
|
|
||||||
|
is_optimal = self.results.empty or loss < self.best_score
|
||||||
if np.isnan(loss):
|
|
||||||
loss = np.Inf
|
# save the first model
|
||||||
|
if is_optimal:
|
||||||
if not os.path.isdir(self.hyperparam_folder):
|
# Try saving first, before updating info
|
||||||
os.makedirs(self.hyperparam_folder)
|
if model is not None:
|
||||||
|
print("Optimal model found, updating")
|
||||||
name = self._get_name(parameters)
|
model.save(self.hyperparam_folder)
|
||||||
|
self.best_score = loss
|
||||||
is_optimal = self.results.empty or loss < self.best_score
|
self.optimal_name = name
|
||||||
|
|
||||||
# save the first model
|
self.results[name] = pd.Series({"loss": loss, "info": info})
|
||||||
if is_optimal:
|
self.saved_params[name] = pd.Series(parameters)
|
||||||
# Try saving first, before updating info
|
|
||||||
if model is not None:
|
self.results.to_csv(os.path.join(self.hyperparam_folder, "results.csv"))
|
||||||
print("Optimal model found, updating")
|
self.saved_params.to_csv(os.path.join(self.hyperparam_folder, "params.csv"))
|
||||||
model.save(self.hyperparam_folder)
|
|
||||||
self.best_score = loss
|
return is_optimal
|
||||||
self.optimal_name = name
|
|
||||||
|
|
||||||
self.results[name] = pd.Series({"loss": loss, "info": info})
|
class DistributedHyperparamOptManager(HyperparamOptManager):
|
||||||
self.saved_params[name] = pd.Series(parameters)
|
"""Manages distributed hyperparameter optimisation across many gpus."""
|
||||||
|
|
||||||
self.results.to_csv(os.path.join(self.hyperparam_folder, "results.csv"))
|
def __init__(
|
||||||
self.saved_params.to_csv(os.path.join(self.hyperparam_folder, "params.csv"))
|
self,
|
||||||
|
param_ranges,
|
||||||
return is_optimal
|
fixed_params,
|
||||||
|
root_model_folder,
|
||||||
|
worker_number,
|
||||||
class DistributedHyperparamOptManager(HyperparamOptManager):
|
search_iterations=1000,
|
||||||
"""Manages distributed hyperparameter optimisation across many gpus."""
|
num_iterations_per_worker=5,
|
||||||
|
clear_serialised_params=False,
|
||||||
def __init__(self,
|
):
|
||||||
param_ranges,
|
"""Instantiates optimisation manager.
|
||||||
fixed_params,
|
|
||||||
root_model_folder,
|
This hyperparameter optimisation pre-generates #search_iterations
|
||||||
worker_number,
|
hyperparameter combinations and serialises them
|
||||||
search_iterations=1000,
|
at the start. At runtime, each worker goes through their own set of
|
||||||
num_iterations_per_worker=5,
|
parameter ranges. The pregeneration
|
||||||
clear_serialised_params=False):
|
allows for multiple workers to run in parallel on different machines without
|
||||||
"""Instantiates optimisation manager.
|
resulting in parameter overlaps.
|
||||||
|
|
||||||
This hyperparameter optimisation pre-generates #search_iterations
|
Args:
|
||||||
hyperparameter combinations and serialises them
|
param_ranges: Discrete hyperparameter range for random search.
|
||||||
at the start. At runtime, each worker goes through their own set of
|
fixed_params: Fixed model parameters per experiment.
|
||||||
parameter ranges. The pregeneration
|
root_model_folder: Folder to store optimisation artifacts.
|
||||||
allows for multiple workers to run in parallel on different machines without
|
worker_number: Worker index definining which set of hyperparameters to
|
||||||
resulting in parameter overlaps.
|
test.
|
||||||
|
search_iterations: Maximum numer of random search iterations.
|
||||||
Args:
|
num_iterations_per_worker: How many iterations are handled per worker.
|
||||||
param_ranges: Discrete hyperparameter range for random search.
|
clear_serialised_params: Whether to regenerate hyperparameter
|
||||||
fixed_params: Fixed model parameters per experiment.
|
combinations.
|
||||||
root_model_folder: Folder to store optimisation artifacts.
|
"""
|
||||||
worker_number: Worker index definining which set of hyperparameters to
|
|
||||||
test.
|
max_workers = int(np.ceil(search_iterations / num_iterations_per_worker))
|
||||||
search_iterations: Maximum numer of random search iterations.
|
|
||||||
num_iterations_per_worker: How many iterations are handled per worker.
|
# Sanity checks
|
||||||
clear_serialised_params: Whether to regenerate hyperparameter
|
if worker_number > max_workers:
|
||||||
combinations.
|
raise ValueError(
|
||||||
"""
|
"Worker number ({}) cannot be larger than the total number of workers!".format(max_workers)
|
||||||
|
)
|
||||||
max_workers = int(np.ceil(search_iterations / num_iterations_per_worker))
|
if worker_number > search_iterations:
|
||||||
|
raise ValueError(
|
||||||
# Sanity checks
|
"Worker number ({}) cannot be larger than the max search iterations ({})!".format(
|
||||||
if worker_number > max_workers:
|
worker_number, search_iterations
|
||||||
raise ValueError(
|
)
|
||||||
"Worker number ({}) cannot be larger than the total number of workers!"
|
)
|
||||||
.format(max_workers))
|
|
||||||
if worker_number > search_iterations:
|
print("*** Creating hyperparameter manager for worker {} ***".format(worker_number))
|
||||||
raise ValueError(
|
|
||||||
"Worker number ({}) cannot be larger than the max search iterations ({})!"
|
hyperparam_folder = os.path.join(root_model_folder, str(worker_number))
|
||||||
.format(worker_number, search_iterations))
|
super().__init__(param_ranges, fixed_params, hyperparam_folder, override_w_fixed_params=True)
|
||||||
|
|
||||||
print("*** Creating hyperparameter manager for worker {} ***".format(
|
serialised_ranges_folder = os.path.join(root_model_folder, "hyperparams")
|
||||||
worker_number))
|
if clear_serialised_params:
|
||||||
|
print("Regenerating hyperparameter list")
|
||||||
hyperparam_folder = os.path.join(root_model_folder, str(worker_number))
|
if os.path.exists(serialised_ranges_folder):
|
||||||
super().__init__(
|
shutil.rmtree(serialised_ranges_folder)
|
||||||
param_ranges,
|
|
||||||
fixed_params,
|
utils.create_folder_if_not_exist(serialised_ranges_folder)
|
||||||
hyperparam_folder,
|
|
||||||
override_w_fixed_params=True)
|
self.serialised_ranges_path = os.path.join(serialised_ranges_folder, "ranges_{}.csv".format(search_iterations))
|
||||||
|
self.hyperparam_folder = hyperparam_folder # override
|
||||||
serialised_ranges_folder = os.path.join(root_model_folder, "hyperparams")
|
self.worker_num = worker_number
|
||||||
if clear_serialised_params:
|
self.total_search_iterations = search_iterations
|
||||||
print("Regenerating hyperparameter list")
|
self.num_iterations_per_worker = num_iterations_per_worker
|
||||||
if os.path.exists(serialised_ranges_folder):
|
self.global_hyperparam_df = self.load_serialised_hyperparam_df()
|
||||||
shutil.rmtree(serialised_ranges_folder)
|
self.worker_search_queue = self._get_worker_search_queue()
|
||||||
|
|
||||||
utils.create_folder_if_not_exist(serialised_ranges_folder)
|
@property
|
||||||
|
def optimisation_completed(self):
|
||||||
self.serialised_ranges_path = os.path.join(
|
return False if self.worker_search_queue else True
|
||||||
serialised_ranges_folder, "ranges_{}.csv".format(search_iterations))
|
|
||||||
self.hyperparam_folder = hyperparam_folder # override
|
def get_next_parameters(self):
|
||||||
self.worker_num = worker_number
|
"""Returns next dictionary of hyperparameters to optimise."""
|
||||||
self.total_search_iterations = search_iterations
|
param_name = self.worker_search_queue.pop()
|
||||||
self.num_iterations_per_worker = num_iterations_per_worker
|
|
||||||
self.global_hyperparam_df = self.load_serialised_hyperparam_df()
|
params = self.global_hyperparam_df.loc[param_name, :].to_dict()
|
||||||
self.worker_search_queue = self._get_worker_search_queue()
|
|
||||||
|
# Always override!
|
||||||
@property
|
for k in self.fixed_params:
|
||||||
def optimisation_completed(self):
|
print("Overriding saved {}: {}".format(k, self.fixed_params[k]))
|
||||||
return False if self.worker_search_queue else True
|
|
||||||
|
params[k] = self.fixed_params[k]
|
||||||
def get_next_parameters(self):
|
|
||||||
"""Returns next dictionary of hyperparameters to optimise."""
|
return params
|
||||||
param_name = self.worker_search_queue.pop()
|
|
||||||
|
def load_serialised_hyperparam_df(self):
|
||||||
params = self.global_hyperparam_df.loc[param_name, :].to_dict()
|
"""Loads serialsed hyperparameter ranges from file.
|
||||||
|
|
||||||
# Always override!
|
Returns:
|
||||||
for k in self.fixed_params:
|
DataFrame containing hyperparameter combinations.
|
||||||
print("Overriding saved {}: {}".format(k, self.fixed_params[k]))
|
"""
|
||||||
|
print(
|
||||||
params[k] = self.fixed_params[k]
|
"Loading params for {} search iterations form {}".format(
|
||||||
|
self.total_search_iterations, self.serialised_ranges_path
|
||||||
return params
|
)
|
||||||
|
)
|
||||||
def load_serialised_hyperparam_df(self):
|
|
||||||
"""Loads serialsed hyperparameter ranges from file.
|
if os.path.exists(self.serialised_ranges_folder):
|
||||||
|
df = pd.read_csv(self.serialised_ranges_path, index_col=0)
|
||||||
Returns:
|
else:
|
||||||
DataFrame containing hyperparameter combinations.
|
print("Unable to load - regenerating serach ranges instead")
|
||||||
"""
|
df = self.update_serialised_hyperparam_df()
|
||||||
print("Loading params for {} search iterations form {}".format(
|
|
||||||
self.total_search_iterations, self.serialised_ranges_path))
|
return df
|
||||||
|
|
||||||
if os.path.exists(self.serialised_ranges_folder):
|
def update_serialised_hyperparam_df(self):
|
||||||
df = pd.read_csv(self.serialised_ranges_path, index_col=0)
|
"""Regenerates hyperparameter combinations and saves to file.
|
||||||
else:
|
|
||||||
print("Unable to load - regenerating serach ranges instead")
|
Returns:
|
||||||
df = self.update_serialised_hyperparam_df()
|
DataFrame containing hyperparameter combinations.
|
||||||
|
"""
|
||||||
return df
|
search_df = self._generate_full_hyperparam_df()
|
||||||
|
|
||||||
def update_serialised_hyperparam_df(self):
|
print(
|
||||||
"""Regenerates hyperparameter combinations and saves to file.
|
"Serialising params for {} search iterations to {}".format(
|
||||||
|
self.total_search_iterations, self.serialised_ranges_path
|
||||||
Returns:
|
)
|
||||||
DataFrame containing hyperparameter combinations.
|
)
|
||||||
"""
|
|
||||||
search_df = self._generate_full_hyperparam_df()
|
search_df.to_csv(self.serialised_ranges_path)
|
||||||
|
|
||||||
print("Serialising params for {} search iterations to {}".format(
|
return search_df
|
||||||
self.total_search_iterations, self.serialised_ranges_path))
|
|
||||||
|
def _generate_full_hyperparam_df(self):
|
||||||
search_df.to_csv(self.serialised_ranges_path)
|
"""Generates actual hyperparameter combinations.
|
||||||
|
|
||||||
return search_df
|
Returns:
|
||||||
|
DataFrame containing hyperparameter combinations.
|
||||||
def _generate_full_hyperparam_df(self):
|
"""
|
||||||
"""Generates actual hyperparameter combinations.
|
|
||||||
|
np.random.seed(131) # for reproducibility of hyperparam list
|
||||||
Returns:
|
|
||||||
DataFrame containing hyperparameter combinations.
|
name_list = []
|
||||||
"""
|
param_list = []
|
||||||
|
for _ in range(self.total_search_iterations):
|
||||||
np.random.seed(131) # for reproducibility of hyperparam list
|
params = super().get_next_parameters(name_list)
|
||||||
|
|
||||||
name_list = []
|
name = self._get_name(params)
|
||||||
param_list = []
|
|
||||||
for _ in range(self.total_search_iterations):
|
name_list.append(name)
|
||||||
params = super().get_next_parameters(name_list)
|
param_list.append(params)
|
||||||
|
|
||||||
name = self._get_name(params)
|
full_search_df = pd.DataFrame(param_list, index=name_list)
|
||||||
|
|
||||||
name_list.append(name)
|
return full_search_df
|
||||||
param_list.append(params)
|
|
||||||
|
def clear(self): # reset when cleared
|
||||||
full_search_df = pd.DataFrame(param_list, index=name_list)
|
"""Clears results for hyperparameter manager and resets."""
|
||||||
|
super().clear()
|
||||||
return full_search_df
|
self.worker_search_queue = self._get_worker_search_queue()
|
||||||
|
|
||||||
def clear(self): # reset when cleared
|
def load_results(self):
|
||||||
"""Clears results for hyperparameter manager and resets."""
|
"""Load results from file and queue parameter combinations to try.
|
||||||
super().clear()
|
|
||||||
self.worker_search_queue = self._get_worker_search_queue()
|
Returns:
|
||||||
|
Boolean indicating if results were successfully loaded.
|
||||||
def load_results(self):
|
"""
|
||||||
"""Load results from file and queue parameter combinations to try.
|
success = super().load_results()
|
||||||
|
|
||||||
Returns:
|
if success:
|
||||||
Boolean indicating if results were successfully loaded.
|
self.worker_search_queue = self._get_worker_search_queue()
|
||||||
"""
|
|
||||||
success = super().load_results()
|
return success
|
||||||
|
|
||||||
if success:
|
def _get_worker_search_queue(self):
|
||||||
self.worker_search_queue = self._get_worker_search_queue()
|
"""Generates the queue of param combinations for current worker.
|
||||||
|
|
||||||
return success
|
Returns:
|
||||||
|
Queue of hyperparameter combinations outstanding.
|
||||||
def _get_worker_search_queue(self):
|
"""
|
||||||
"""Generates the queue of param combinations for current worker.
|
global_df = self.assign_worker_numbers(self.global_hyperparam_df)
|
||||||
|
worker_df = global_df[global_df["worker"] == self.worker_num]
|
||||||
Returns:
|
|
||||||
Queue of hyperparameter combinations outstanding.
|
left_overs = [s for s in worker_df.index if s not in self.results.columns]
|
||||||
"""
|
|
||||||
global_df = self.assign_worker_numbers(self.global_hyperparam_df)
|
return Deque(left_overs)
|
||||||
worker_df = global_df[global_df["worker"] == self.worker_num]
|
|
||||||
|
def assign_worker_numbers(self, df):
|
||||||
left_overs = [s for s in worker_df.index if s not in self.results.columns]
|
"""Updates parameter combinations with the index of the worker used.
|
||||||
|
|
||||||
return Deque(left_overs)
|
Args:
|
||||||
|
df: DataFrame of parameter combinations.
|
||||||
def assign_worker_numbers(self, df):
|
|
||||||
"""Updates parameter combinations with the index of the worker used.
|
Returns:
|
||||||
|
Updated DataFrame with worker number.
|
||||||
Args:
|
"""
|
||||||
df: DataFrame of parameter combinations.
|
output = df.copy()
|
||||||
|
|
||||||
Returns:
|
n = self.total_search_iterations
|
||||||
Updated DataFrame with worker number.
|
batch_size = self.num_iterations_per_worker
|
||||||
"""
|
|
||||||
output = df.copy()
|
max_worker_num = int(np.ceil(n / batch_size))
|
||||||
|
|
||||||
n = self.total_search_iterations
|
worker_idx = np.concatenate([np.tile(i + 1, self.num_iterations_per_worker) for i in range(max_worker_num)])
|
||||||
batch_size = self.num_iterations_per_worker
|
|
||||||
|
output["worker"] = worker_idx[: len(output)]
|
||||||
max_worker_num = int(np.ceil(n / batch_size))
|
|
||||||
|
return output
|
||||||
worker_idx = np.concatenate([
|
|
||||||
np.tile(i + 1, self.num_iterations_per_worker)
|
|
||||||
for i in range(max_worker_num)
|
|
||||||
])
|
|
||||||
|
|
||||||
output["worker"] = worker_idx[:len(output)]
|
|
||||||
|
|
||||||
return output
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,236 +1,224 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
# Copyright 2020 The Google Research Authors.
|
# Copyright 2020 The Google Research Authors.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
#
|
#
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
#
|
#
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Lint as: python3
|
# Lint as: python3
|
||||||
"""Generic helper functions used across codebase."""
|
"""Generic helper functions used across codebase."""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import tensorflow as tf
|
import tensorflow as tf
|
||||||
from tensorflow.python.tools.inspect_checkpoint import print_tensors_in_checkpoint_file
|
from tensorflow.python.tools.inspect_checkpoint import print_tensors_in_checkpoint_file
|
||||||
|
|
||||||
|
|
||||||
# Generic.
|
# Generic.
|
||||||
def get_single_col_by_input_type(input_type, column_definition):
|
def get_single_col_by_input_type(input_type, column_definition):
|
||||||
"""Returns name of single column.
|
"""Returns name of single column.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
input_type: Input type of column to extract
|
input_type: Input type of column to extract
|
||||||
column_definition: Column definition list for experiment
|
column_definition: Column definition list for experiment
|
||||||
"""
|
"""
|
||||||
|
|
||||||
l = [tup[0] for tup in column_definition if tup[2] == input_type]
|
l = [tup[0] for tup in column_definition if tup[2] == input_type]
|
||||||
|
|
||||||
if len(l) != 1:
|
if len(l) != 1:
|
||||||
raise ValueError('Invalid number of columns for {}'.format(input_type))
|
raise ValueError("Invalid number of columns for {}".format(input_type))
|
||||||
|
|
||||||
return l[0]
|
return l[0]
|
||||||
|
|
||||||
|
|
||||||
def extract_cols_from_data_type(data_type, column_definition,
|
def extract_cols_from_data_type(data_type, column_definition, excluded_input_types):
|
||||||
excluded_input_types):
|
"""Extracts the names of columns that correspond to a define data_type.
|
||||||
"""Extracts the names of columns that correspond to a define data_type.
|
|
||||||
|
Args:
|
||||||
Args:
|
data_type: DataType of columns to extract.
|
||||||
data_type: DataType of columns to extract.
|
column_definition: Column definition to use.
|
||||||
column_definition: Column definition to use.
|
excluded_input_types: Set of input types to exclude
|
||||||
excluded_input_types: Set of input types to exclude
|
|
||||||
|
Returns:
|
||||||
Returns:
|
List of names for columns with data type specified.
|
||||||
List of names for columns with data type specified.
|
"""
|
||||||
"""
|
return [tup[0] for tup in column_definition if tup[1] == data_type and tup[2] not in excluded_input_types]
|
||||||
return [
|
|
||||||
tup[0]
|
|
||||||
for tup in column_definition
|
# Loss functions.
|
||||||
if tup[1] == data_type and tup[2] not in excluded_input_types
|
def tensorflow_quantile_loss(y, y_pred, quantile):
|
||||||
]
|
"""Computes quantile loss for tensorflow.
|
||||||
|
|
||||||
|
Standard quantile loss as defined in the "Training Procedure" section of
|
||||||
# Loss functions.
|
the main TFT paper
|
||||||
def tensorflow_quantile_loss(y, y_pred, quantile):
|
|
||||||
"""Computes quantile loss for tensorflow.
|
Args:
|
||||||
|
y: Targets
|
||||||
Standard quantile loss as defined in the "Training Procedure" section of
|
y_pred: Predictions
|
||||||
the main TFT paper
|
quantile: Quantile to use for loss calculations (between 0 & 1)
|
||||||
|
|
||||||
Args:
|
Returns:
|
||||||
y: Targets
|
Tensor for quantile loss.
|
||||||
y_pred: Predictions
|
"""
|
||||||
quantile: Quantile to use for loss calculations (between 0 & 1)
|
|
||||||
|
# Checks quantile
|
||||||
Returns:
|
if quantile < 0 or quantile > 1:
|
||||||
Tensor for quantile loss.
|
raise ValueError("Illegal quantile value={}! Values should be between 0 and 1.".format(quantile))
|
||||||
"""
|
|
||||||
|
prediction_underflow = y - y_pred
|
||||||
# Checks quantile
|
q_loss = quantile * tf.maximum(prediction_underflow, 0.0) + (1.0 - quantile) * tf.maximum(
|
||||||
if quantile < 0 or quantile > 1:
|
-prediction_underflow, 0.0
|
||||||
raise ValueError(
|
)
|
||||||
'Illegal quantile value={}! Values should be between 0 and 1.'.format(
|
|
||||||
quantile))
|
return tf.reduce_sum(q_loss, axis=-1)
|
||||||
|
|
||||||
prediction_underflow = y - y_pred
|
|
||||||
q_loss = quantile * tf.maximum(prediction_underflow, 0.) + (
|
def numpy_normalised_quantile_loss(y, y_pred, quantile):
|
||||||
1. - quantile) * tf.maximum(-prediction_underflow, 0.)
|
"""Computes normalised quantile loss for numpy arrays.
|
||||||
|
|
||||||
return tf.reduce_sum(q_loss, axis=-1)
|
Uses the q-Risk metric as defined in the "Training Procedure" section of the
|
||||||
|
main TFT paper.
|
||||||
|
|
||||||
def numpy_normalised_quantile_loss(y, y_pred, quantile):
|
Args:
|
||||||
"""Computes normalised quantile loss for numpy arrays.
|
y: Targets
|
||||||
|
y_pred: Predictions
|
||||||
Uses the q-Risk metric as defined in the "Training Procedure" section of the
|
quantile: Quantile to use for loss calculations (between 0 & 1)
|
||||||
main TFT paper.
|
|
||||||
|
Returns:
|
||||||
Args:
|
Float for normalised quantile loss.
|
||||||
y: Targets
|
"""
|
||||||
y_pred: Predictions
|
prediction_underflow = y - y_pred
|
||||||
quantile: Quantile to use for loss calculations (between 0 & 1)
|
weighted_errors = quantile * np.maximum(prediction_underflow, 0.0) + (1.0 - quantile) * np.maximum(
|
||||||
|
-prediction_underflow, 0.0
|
||||||
Returns:
|
)
|
||||||
Float for normalised quantile loss.
|
|
||||||
"""
|
quantile_loss = weighted_errors.mean()
|
||||||
prediction_underflow = y - y_pred
|
normaliser = y.abs().mean()
|
||||||
weighted_errors = quantile * np.maximum(prediction_underflow, 0.) \
|
|
||||||
+ (1. - quantile) * np.maximum(-prediction_underflow, 0.)
|
return 2 * quantile_loss / normaliser
|
||||||
|
|
||||||
quantile_loss = weighted_errors.mean()
|
|
||||||
normaliser = y.abs().mean()
|
# OS related functions.
|
||||||
|
def create_folder_if_not_exist(directory):
|
||||||
return 2 * quantile_loss / normaliser
|
"""Creates folder if it doesn't exist.
|
||||||
|
|
||||||
|
Args:
|
||||||
# OS related functions.
|
directory: Folder path to create.
|
||||||
def create_folder_if_not_exist(directory):
|
"""
|
||||||
"""Creates folder if it doesn't exist.
|
# Also creates directories recursively
|
||||||
|
pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
|
||||||
Args:
|
|
||||||
directory: Folder path to create.
|
|
||||||
"""
|
# Tensorflow related functions.
|
||||||
# Also creates directories recursively
|
def get_default_tensorflow_config(tf_device="gpu", gpu_id=0):
|
||||||
pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
|
"""Creates tensorflow config for graphs to run on CPU or GPU.
|
||||||
|
|
||||||
|
Specifies whether to run graph on gpu or cpu and which GPU ID to use for multi
|
||||||
# Tensorflow related functions.
|
GPU machines.
|
||||||
def get_default_tensorflow_config(tf_device='gpu', gpu_id=0):
|
|
||||||
"""Creates tensorflow config for graphs to run on CPU or GPU.
|
Args:
|
||||||
|
tf_device: 'cpu' or 'gpu'
|
||||||
Specifies whether to run graph on gpu or cpu and which GPU ID to use for multi
|
gpu_id: GPU ID to use if relevant
|
||||||
GPU machines.
|
|
||||||
|
Returns:
|
||||||
Args:
|
Tensorflow config.
|
||||||
tf_device: 'cpu' or 'gpu'
|
"""
|
||||||
gpu_id: GPU ID to use if relevant
|
|
||||||
|
if tf_device == "cpu":
|
||||||
Returns:
|
os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # for training on cpu
|
||||||
Tensorflow config.
|
tf_config = tf.ConfigProto(log_device_placement=False, device_count={"GPU": 0})
|
||||||
"""
|
|
||||||
|
else:
|
||||||
if tf_device == 'cpu':
|
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
|
||||||
os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # for training on cpu
|
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id)
|
||||||
tf_config = tf.ConfigProto(
|
|
||||||
log_device_placement=False, device_count={'GPU': 0})
|
print("Selecting GPU ID={}".format(gpu_id))
|
||||||
|
|
||||||
else:
|
tf_config = tf.ConfigProto(log_device_placement=False)
|
||||||
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
|
tf_config.gpu_options.allow_growth = True
|
||||||
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
|
|
||||||
|
return tf_config
|
||||||
print('Selecting GPU ID={}'.format(gpu_id))
|
|
||||||
|
|
||||||
tf_config = tf.ConfigProto(log_device_placement=False)
|
def save(tf_session, model_folder, cp_name, scope=None):
|
||||||
tf_config.gpu_options.allow_growth = True
|
"""Saves Tensorflow graph to checkpoint.
|
||||||
|
|
||||||
return tf_config
|
Saves all trainiable variables under a given variable scope to checkpoint.
|
||||||
|
|
||||||
|
Args:
|
||||||
def save(tf_session, model_folder, cp_name, scope=None):
|
tf_session: Session containing graph
|
||||||
"""Saves Tensorflow graph to checkpoint.
|
model_folder: Folder to save models
|
||||||
|
cp_name: Name of Tensorflow checkpoint
|
||||||
Saves all trainiable variables under a given variable scope to checkpoint.
|
scope: Variable scope containing variables to save
|
||||||
|
"""
|
||||||
Args:
|
# Save model
|
||||||
tf_session: Session containing graph
|
if scope is None:
|
||||||
model_folder: Folder to save models
|
saver = tf.train.Saver()
|
||||||
cp_name: Name of Tensorflow checkpoint
|
else:
|
||||||
scope: Variable scope containing variables to save
|
var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope)
|
||||||
"""
|
saver = tf.train.Saver(var_list=var_list, max_to_keep=100000)
|
||||||
# Save model
|
|
||||||
if scope is None:
|
save_path = saver.save(tf_session, os.path.join(model_folder, "{0}.ckpt".format(cp_name)))
|
||||||
saver = tf.train.Saver()
|
print("Model saved to: {0}".format(save_path))
|
||||||
else:
|
|
||||||
var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope)
|
|
||||||
saver = tf.train.Saver(var_list=var_list, max_to_keep=100000)
|
def load(tf_session, model_folder, cp_name, scope=None, verbose=False):
|
||||||
|
"""Loads Tensorflow graph from checkpoint.
|
||||||
save_path = saver.save(tf_session,
|
|
||||||
os.path.join(model_folder, '{0}.ckpt'.format(cp_name)))
|
Args:
|
||||||
print('Model saved to: {0}'.format(save_path))
|
tf_session: Session to load graph into
|
||||||
|
model_folder: Folder containing serialised model
|
||||||
|
cp_name: Name of Tensorflow checkpoint
|
||||||
def load(tf_session, model_folder, cp_name, scope=None, verbose=False):
|
scope: Variable scope to use.
|
||||||
"""Loads Tensorflow graph from checkpoint.
|
verbose: Whether to print additional debugging information.
|
||||||
|
"""
|
||||||
Args:
|
# Load model proper
|
||||||
tf_session: Session to load graph into
|
load_path = os.path.join(model_folder, "{0}.ckpt".format(cp_name))
|
||||||
model_folder: Folder containing serialised model
|
|
||||||
cp_name: Name of Tensorflow checkpoint
|
print("Loading model from {0}".format(load_path))
|
||||||
scope: Variable scope to use.
|
|
||||||
verbose: Whether to print additional debugging information.
|
print_weights_in_checkpoint(model_folder, cp_name)
|
||||||
"""
|
|
||||||
# Load model proper
|
initial_vars = set([v.name for v in tf.get_default_graph().as_graph_def().node])
|
||||||
load_path = os.path.join(model_folder, '{0}.ckpt'.format(cp_name))
|
|
||||||
|
# Saver
|
||||||
print('Loading model from {0}'.format(load_path))
|
if scope is None:
|
||||||
|
saver = tf.train.Saver()
|
||||||
print_weights_in_checkpoint(model_folder, cp_name)
|
else:
|
||||||
|
var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope)
|
||||||
initial_vars = set(
|
saver = tf.train.Saver(var_list=var_list, max_to_keep=100000)
|
||||||
[v.name for v in tf.get_default_graph().as_graph_def().node])
|
# Load
|
||||||
|
saver.restore(tf_session, load_path)
|
||||||
# Saver
|
all_vars = set([v.name for v in tf.get_default_graph().as_graph_def().node])
|
||||||
if scope is None:
|
|
||||||
saver = tf.train.Saver()
|
if verbose:
|
||||||
else:
|
print("Restored {0}".format(",".join(initial_vars.difference(all_vars))))
|
||||||
var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope)
|
print("Existing {0}".format(",".join(all_vars.difference(initial_vars))))
|
||||||
saver = tf.train.Saver(var_list=var_list, max_to_keep=100000)
|
print("All {0}".format(",".join(all_vars)))
|
||||||
# Load
|
|
||||||
saver.restore(tf_session, load_path)
|
print("Done.")
|
||||||
all_vars = set([v.name for v in tf.get_default_graph().as_graph_def().node])
|
|
||||||
|
|
||||||
if verbose:
|
def print_weights_in_checkpoint(model_folder, cp_name):
|
||||||
print('Restored {0}'.format(','.join(initial_vars.difference(all_vars))))
|
"""Prints all weights in Tensorflow checkpoint.
|
||||||
print('Existing {0}'.format(','.join(all_vars.difference(initial_vars))))
|
|
||||||
print('All {0}'.format(','.join(all_vars)))
|
Args:
|
||||||
|
model_folder: Folder containing checkpoint
|
||||||
print('Done.')
|
cp_name: Name of checkpoint
|
||||||
|
|
||||||
|
Returns:
|
||||||
def print_weights_in_checkpoint(model_folder, cp_name):
|
|
||||||
"""Prints all weights in Tensorflow checkpoint.
|
"""
|
||||||
|
load_path = os.path.join(model_folder, "{0}.ckpt".format(cp_name))
|
||||||
Args:
|
|
||||||
model_folder: Folder containing checkpoint
|
print_tensors_in_checkpoint_file(file_name=load_path, tensor_name="", all_tensors=True, all_tensor_names=True)
|
||||||
cp_name: Name of checkpoint
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
|
|
||||||
"""
|
|
||||||
load_path = os.path.join(model_folder, '{0}.ckpt'.format(cp_name))
|
|
||||||
|
|
||||||
print_tensors_in_checkpoint_file(
|
|
||||||
file_name=load_path,
|
|
||||||
tensor_name='',
|
|
||||||
all_tensors=True,
|
|
||||||
all_tensor_names=True)
|
|
||||||
|
|||||||
@@ -1,246 +1,248 @@
|
|||||||
# Copyright (c) Microsoft Corporation.
|
# Copyright (c) Microsoft Corporation.
|
||||||
# Licensed under the MIT License.
|
# Licensed under the MIT License.
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import tensorflow.compat.v1 as tf
|
import tensorflow.compat.v1 as tf
|
||||||
import data_formatters.base
|
import data_formatters.base
|
||||||
import expt_settings.configs
|
import expt_settings.configs
|
||||||
import libs.hyperparam_opt
|
import libs.hyperparam_opt
|
||||||
import libs.tft_model
|
import libs.tft_model
|
||||||
import libs.utils as utils
|
import libs.utils as utils
|
||||||
import os
|
import os
|
||||||
import datetime as dte
|
import datetime as dte
|
||||||
|
|
||||||
|
|
||||||
from qlib.model.base import ModelFT
|
from qlib.model.base import ModelFT
|
||||||
from qlib.data.dataset import DatasetH
|
from qlib.data.dataset import DatasetH
|
||||||
from qlib.data.dataset.handler import DataHandlerLP
|
from qlib.data.dataset.handler import DataHandlerLP
|
||||||
|
|
||||||
|
|
||||||
|
# To register new datasets, please add them here.
|
||||||
# To register new datasets, please add them here.
|
ALLOW_DATASET = ["Alpha158"]
|
||||||
ALLOW_DATASET = ['Alpha158']
|
DATASET_SETTING = {
|
||||||
DATASET_SETTING = {
|
"Alpha158": {
|
||||||
'Alpha158': {
|
"feature_col": ["RESI5", "WVMA5", "RSQR5", "KLEN", "RSQR10", "CORR5", "CORD5", "CORR10", "ROC60", "RESI10"],
|
||||||
'feature_col': ['RESI5', 'WVMA5', 'RSQR5', 'KLEN', 'RSQR10', 'CORR5', 'CORD5', 'CORR10', 'ROC60', 'RESI10'],
|
"label_col": ["LABEL0"],
|
||||||
'label_col': ['LABEL0'],
|
},
|
||||||
},
|
}
|
||||||
}
|
# To register new datasets, please add their configurations here.
|
||||||
# To register new datasets, please add their configurations here.
|
|
||||||
|
|
||||||
def get_shifted_label(data_df, shifts=5, col_shift='LABEL0'):
|
def get_shifted_label(data_df, shifts=5, col_shift="LABEL0"):
|
||||||
return data_df[[col_shift]].groupby('instrument').apply(lambda df: df.shift(shifts))
|
return data_df[[col_shift]].groupby("instrument").apply(lambda df: df.shift(shifts))
|
||||||
|
|
||||||
def fill_test_na(test_df):
|
|
||||||
test_df_res = test_df.copy()
|
def fill_test_na(test_df):
|
||||||
feature_cols = ~test_df_res.columns.str.contains('label', case=False)
|
test_df_res = test_df.copy()
|
||||||
test_feature_fna = test_df_res.loc[:, feature_cols].groupby('datetime').apply(lambda df: df.fillna(df.mean()))
|
feature_cols = ~test_df_res.columns.str.contains("label", case=False)
|
||||||
test_df_res.loc[:, feature_cols] = test_feature_fna
|
test_feature_fna = test_df_res.loc[:, feature_cols].groupby("datetime").apply(lambda df: df.fillna(df.mean()))
|
||||||
return test_df_res
|
test_df_res.loc[:, feature_cols] = test_feature_fna
|
||||||
|
return test_df_res
|
||||||
def process_qlib_data(df, dataset, fillna=False):
|
|
||||||
"""Prepare data to fit the TFT model.
|
|
||||||
|
def process_qlib_data(df, dataset, fillna=False):
|
||||||
Args:
|
"""Prepare data to fit the TFT model.
|
||||||
df: Original DataFrame.
|
|
||||||
fillna: Whether to fill the data with the mean values.
|
Args:
|
||||||
|
df: Original DataFrame.
|
||||||
Returns:
|
fillna: Whether to fill the data with the mean values.
|
||||||
Transformed DataFrame.
|
|
||||||
|
Returns:
|
||||||
"""
|
Transformed DataFrame.
|
||||||
# Several features selected manually
|
|
||||||
feature_col = DATASET_SETTING[dataset]['feature_col']
|
"""
|
||||||
label_col = DATASET_SETTING[dataset]['label_col']
|
# Several features selected manually
|
||||||
temp_df = df.loc[:, feature_col+label_col]
|
feature_col = DATASET_SETTING[dataset]["feature_col"]
|
||||||
if fillna:
|
label_col = DATASET_SETTING[dataset]["label_col"]
|
||||||
temp_df = fill_test_na(temp_df)
|
temp_df = df.loc[:, feature_col + label_col]
|
||||||
temp_df = temp_df.swaplevel()
|
if fillna:
|
||||||
temp_df = temp_df.sort_index()
|
temp_df = fill_test_na(temp_df)
|
||||||
temp_df = temp_df.reset_index(level=0)
|
temp_df = temp_df.swaplevel()
|
||||||
dates = pd.to_datetime(temp_df.index)
|
temp_df = temp_df.sort_index()
|
||||||
temp_df['date'] = dates
|
temp_df = temp_df.reset_index(level=0)
|
||||||
temp_df['day_of_week'] = dates.dayofweek
|
dates = pd.to_datetime(temp_df.index)
|
||||||
temp_df['month'] = dates.month
|
temp_df["date"] = dates
|
||||||
temp_df['year'] = dates.year
|
temp_df["day_of_week"] = dates.dayofweek
|
||||||
temp_df['const'] = 1.0
|
temp_df["month"] = dates.month
|
||||||
return temp_df
|
temp_df["year"] = dates.year
|
||||||
|
temp_df["const"] = 1.0
|
||||||
def process_predicted(df, col_name):
|
return temp_df
|
||||||
"""Transform the TFT predicted data into Qlib format.
|
|
||||||
|
|
||||||
Args:
|
def process_predicted(df, col_name):
|
||||||
df: Original DataFrame.
|
"""Transform the TFT predicted data into Qlib format.
|
||||||
fillna: New column name.
|
|
||||||
|
Args:
|
||||||
Returns:
|
df: Original DataFrame.
|
||||||
Transformed DataFrame.
|
fillna: New column name.
|
||||||
|
|
||||||
"""
|
Returns:
|
||||||
df_res = df.copy()
|
Transformed DataFrame.
|
||||||
df_res = df_res.rename(columns={"forecast_time": "datetime", "identifier": "instrument", "t+0": col_name})
|
|
||||||
df_res = df_res.set_index(['datetime','instrument']).sort_index()
|
"""
|
||||||
df_res = df_res[[col_name]]
|
df_res = df.copy()
|
||||||
return df_res
|
df_res = df_res.rename(columns={"forecast_time": "datetime", "identifier": "instrument", "t+0": col_name})
|
||||||
|
df_res = df_res.set_index(["datetime", "instrument"]).sort_index()
|
||||||
def format_score(forecast_df, col_name='pred', label_shift=5):
|
df_res = df_res[[col_name]]
|
||||||
pred = process_predicted(forecast_df, col_name=col_name)
|
return df_res
|
||||||
pred = get_shifted_label(pred, shifts=-label_shift, col_shift=col_name)
|
|
||||||
pred = pred.dropna()[col_name]
|
|
||||||
return pred
|
def format_score(forecast_df, col_name="pred", label_shift=5):
|
||||||
|
pred = process_predicted(forecast_df, col_name=col_name)
|
||||||
def transform_df(df, col_name='LABEL0'):
|
pred = get_shifted_label(pred, shifts=-label_shift, col_shift=col_name)
|
||||||
df_res = df['feature']
|
pred = pred.dropna()[col_name]
|
||||||
df_res[col_name] = df['label']
|
return pred
|
||||||
return df_res
|
|
||||||
|
|
||||||
class TFTModel(ModelFT):
|
def transform_df(df, col_name="LABEL0"):
|
||||||
"""TFT Model"""
|
df_res = df["feature"]
|
||||||
|
df_res[col_name] = df["label"]
|
||||||
def __init__(self, **kwargs):
|
return df_res
|
||||||
self.model = None
|
|
||||||
|
|
||||||
def _prepare_data(self, dataset: DatasetH):
|
class TFTModel(ModelFT):
|
||||||
df_train, df_valid = dataset.prepare(
|
"""TFT Model"""
|
||||||
["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
|
|
||||||
)
|
def __init__(self, **kwargs):
|
||||||
return transform_df(df_train), transform_df(df_valid)
|
self.model = None
|
||||||
|
|
||||||
def fit(
|
def _prepare_data(self, dataset: DatasetH):
|
||||||
self,
|
df_train, df_valid = dataset.prepare(
|
||||||
dataset: DatasetH,
|
["train", "valid"], col_set=["feature", "label"], data_key=DataHandlerLP.DK_L
|
||||||
DATASET = 'Alpha158',
|
)
|
||||||
MODEL_FOLDER = 'qlib_alpha158_model',
|
return transform_df(df_train), transform_df(df_valid)
|
||||||
LABEL_COL = 'LABEL0',
|
|
||||||
LABEL_SHIFT = 5,
|
def fit(
|
||||||
USE_GPU_ID = 0,
|
self,
|
||||||
**kwargs
|
dataset: DatasetH,
|
||||||
):
|
DATASET="Alpha158",
|
||||||
|
MODEL_FOLDER="qlib_alpha158_model",
|
||||||
if DATASET not in ALLOW_DATASET:
|
LABEL_COL="LABEL0",
|
||||||
raise AssertionError("The dataset is not supported, please make a new formatter to fit this dataset")
|
LABEL_SHIFT=5,
|
||||||
|
USE_GPU_ID=0,
|
||||||
dtrain, dvalid = self._prepare_data(dataset)
|
**kwargs
|
||||||
dtrain.loc[:, LABEL_COL] = get_shifted_label(dtrain, shifts=LABEL_SHIFT, col_shift=LABEL_COL)
|
):
|
||||||
dvalid.loc[:, LABEL_COL] = get_shifted_label(dvalid, shifts=LABEL_SHIFT, col_shift=LABEL_COL)
|
|
||||||
|
if DATASET not in ALLOW_DATASET:
|
||||||
|
raise AssertionError("The dataset is not supported, please make a new formatter to fit this dataset")
|
||||||
train = process_qlib_data(dtrain, DATASET, fillna=True).dropna()
|
|
||||||
valid = process_qlib_data(dvalid, DATASET, fillna=True).dropna()
|
dtrain, dvalid = self._prepare_data(dataset)
|
||||||
|
dtrain.loc[:, LABEL_COL] = get_shifted_label(dtrain, shifts=LABEL_SHIFT, col_shift=LABEL_COL)
|
||||||
ExperimentConfig = expt_settings.configs.ExperimentConfig
|
dvalid.loc[:, LABEL_COL] = get_shifted_label(dvalid, shifts=LABEL_SHIFT, col_shift=LABEL_COL)
|
||||||
config = ExperimentConfig(DATASET)
|
|
||||||
self.data_formatter = config.make_data_formatter()
|
train = process_qlib_data(dtrain, DATASET, fillna=True).dropna()
|
||||||
self.model_folder = MODEL_FOLDER
|
valid = process_qlib_data(dvalid, DATASET, fillna=True).dropna()
|
||||||
self.gpu_id = USE_GPU_ID
|
|
||||||
self.label_shift = LABEL_SHIFT
|
ExperimentConfig = expt_settings.configs.ExperimentConfig
|
||||||
self.expt_name = DATASET
|
config = ExperimentConfig(DATASET)
|
||||||
self.label_col = LABEL_COL
|
self.data_formatter = config.make_data_formatter()
|
||||||
|
self.model_folder = MODEL_FOLDER
|
||||||
use_gpu = (True, self.gpu_id)
|
self.gpu_id = USE_GPU_ID
|
||||||
#===========================Training Process===========================
|
self.label_shift = LABEL_SHIFT
|
||||||
ModelClass = libs.tft_model.TemporalFusionTransformer
|
self.expt_name = DATASET
|
||||||
if not isinstance(self.data_formatter, data_formatters.base.GenericDataFormatter):
|
self.label_col = LABEL_COL
|
||||||
raise ValueError(
|
|
||||||
"Data formatters should inherit from" +
|
use_gpu = (True, self.gpu_id)
|
||||||
"AbstractDataFormatter! Type={}".format(type(self.data_formatter)))
|
# ===========================Training Process===========================
|
||||||
|
ModelClass = libs.tft_model.TemporalFusionTransformer
|
||||||
default_keras_session = tf.keras.backend.get_session()
|
if not isinstance(self.data_formatter, data_formatters.base.GenericDataFormatter):
|
||||||
|
raise ValueError(
|
||||||
if use_gpu[0]:
|
"Data formatters should inherit from"
|
||||||
self.tf_config = utils.get_default_tensorflow_config(tf_device="gpu", gpu_id=use_gpu[1])
|
+ "AbstractDataFormatter! Type={}".format(type(self.data_formatter))
|
||||||
else:
|
)
|
||||||
self.tf_config = utils.get_default_tensorflow_config(tf_device="cpu")
|
|
||||||
|
default_keras_session = tf.keras.backend.get_session()
|
||||||
self.data_formatter.set_scalers(train)
|
|
||||||
|
if use_gpu[0]:
|
||||||
# Sets up default params
|
self.tf_config = utils.get_default_tensorflow_config(tf_device="gpu", gpu_id=use_gpu[1])
|
||||||
fixed_params = self.data_formatter.get_experiment_params()
|
else:
|
||||||
params = self.data_formatter.get_default_model_params()
|
self.tf_config = utils.get_default_tensorflow_config(tf_device="cpu")
|
||||||
|
|
||||||
# Wendi: 合并调优的参数和非调优的参数
|
self.data_formatter.set_scalers(train)
|
||||||
params = {**params, **fixed_params}
|
|
||||||
|
# Sets up default params
|
||||||
if not os.path.exists(self.model_folder):
|
fixed_params = self.data_formatter.get_experiment_params()
|
||||||
os.makedirs(self.model_folder)
|
params = self.data_formatter.get_default_model_params()
|
||||||
params['model_folder'] = self.model_folder
|
|
||||||
|
# Wendi: 合并调优的参数和非调优的参数
|
||||||
print("*** Begin training ***")
|
params = {**params, **fixed_params}
|
||||||
best_loss = np.Inf
|
|
||||||
|
if not os.path.exists(self.model_folder):
|
||||||
tf.reset_default_graph()
|
os.makedirs(self.model_folder)
|
||||||
|
params["model_folder"] = self.model_folder
|
||||||
self.tf_graph = tf.Graph()
|
|
||||||
with self.tf_graph.as_default():
|
print("*** Begin training ***")
|
||||||
self.sess = tf.Session(config=self.tf_config)
|
best_loss = np.Inf
|
||||||
tf.keras.backend.set_session(self.sess)
|
|
||||||
self.model = ModelClass(params, use_cudnn=use_gpu[0])
|
tf.reset_default_graph()
|
||||||
self.sess.run(tf.global_variables_initializer())
|
|
||||||
self.model.fit(train_df=train, valid_df=valid)
|
self.tf_graph = tf.Graph()
|
||||||
print("*** Finished training ***")
|
with self.tf_graph.as_default():
|
||||||
saved_model_dir = self.model_folder+'/'+'saved_model'
|
self.sess = tf.Session(config=self.tf_config)
|
||||||
if not os.path.exists(saved_model_dir):
|
tf.keras.backend.set_session(self.sess)
|
||||||
os.makedirs(saved_model_dir)
|
self.model = ModelClass(params, use_cudnn=use_gpu[0])
|
||||||
self.model.save(saved_model_dir)
|
self.sess.run(tf.global_variables_initializer())
|
||||||
|
self.model.fit(train_df=train, valid_df=valid)
|
||||||
def extract_numerical_data(data):
|
print("*** Finished training ***")
|
||||||
"""Strips out forecast time and identifier columns."""
|
saved_model_dir = self.model_folder + "/" + "saved_model"
|
||||||
return data[[
|
if not os.path.exists(saved_model_dir):
|
||||||
col for col in data.columns
|
os.makedirs(saved_model_dir)
|
||||||
if col not in {"forecast_time", "identifier"}
|
self.model.save(saved_model_dir)
|
||||||
]]
|
|
||||||
|
def extract_numerical_data(data):
|
||||||
#p50_loss = utils.numpy_normalised_quantile_loss(
|
"""Strips out forecast time and identifier columns."""
|
||||||
# extract_numerical_data(targets), extract_numerical_data(p50_forecast),
|
return data[[col for col in data.columns if col not in {"forecast_time", "identifier"}]]
|
||||||
# 0.5)
|
|
||||||
#p90_loss = utils.numpy_normalised_quantile_loss(
|
# p50_loss = utils.numpy_normalised_quantile_loss(
|
||||||
# extract_numerical_data(targets), extract_numerical_data(p90_forecast),
|
# extract_numerical_data(targets), extract_numerical_data(p50_forecast),
|
||||||
# 0.9)
|
# 0.5)
|
||||||
tf.keras.backend.set_session(default_keras_session)
|
# p90_loss = utils.numpy_normalised_quantile_loss(
|
||||||
print("Training completed.".format(dte.datetime.now()))
|
# extract_numerical_data(targets), extract_numerical_data(p90_forecast),
|
||||||
#===========================Training Process===========================
|
# 0.9)
|
||||||
|
tf.keras.backend.set_session(default_keras_session)
|
||||||
def predict(self, dataset):
|
print("Training completed.".format(dte.datetime.now()))
|
||||||
if self.model is None:
|
# ===========================Training Process===========================
|
||||||
raise ValueError("model is not fitted yet!")
|
|
||||||
d_test = dataset.prepare("test", col_set=["feature", "label"])
|
def predict(self, dataset):
|
||||||
d_test = transform_df(d_test)
|
if self.model is None:
|
||||||
d_test.loc[:, self.label_col] = get_shifted_label(d_test, shifts=self.label_shift, col_shift=self.label_col)
|
raise ValueError("model is not fitted yet!")
|
||||||
test = process_qlib_data(d_test, self.expt_name, fillna=True).dropna()
|
d_test = dataset.prepare("test", col_set=["feature", "label"])
|
||||||
|
d_test = transform_df(d_test)
|
||||||
use_gpu = (True, self.gpu_id)
|
d_test.loc[:, self.label_col] = get_shifted_label(d_test, shifts=self.label_shift, col_shift=self.label_col)
|
||||||
#===========================Predicting Process===========================
|
test = process_qlib_data(d_test, self.expt_name, fillna=True).dropna()
|
||||||
default_keras_session = tf.keras.backend.get_session()
|
|
||||||
|
use_gpu = (True, self.gpu_id)
|
||||||
# Sets up default params
|
# ===========================Predicting Process===========================
|
||||||
fixed_params = self.data_formatter.get_experiment_params()
|
default_keras_session = tf.keras.backend.get_session()
|
||||||
params = self.data_formatter.get_default_model_params()
|
|
||||||
params = {**params, **fixed_params}
|
# Sets up default params
|
||||||
|
fixed_params = self.data_formatter.get_experiment_params()
|
||||||
|
params = self.data_formatter.get_default_model_params()
|
||||||
print("*** Begin predicting ***")
|
params = {**params, **fixed_params}
|
||||||
tf.reset_default_graph()
|
|
||||||
|
print("*** Begin predicting ***")
|
||||||
with self.tf_graph.as_default():
|
tf.reset_default_graph()
|
||||||
tf.keras.backend.set_session(self.sess)
|
|
||||||
output_map = self.model.predict(test, return_targets=True)
|
with self.tf_graph.as_default():
|
||||||
targets = self.data_formatter.format_predictions(output_map["targets"])
|
tf.keras.backend.set_session(self.sess)
|
||||||
p50_forecast = self.data_formatter.format_predictions(output_map["p50"])
|
output_map = self.model.predict(test, return_targets=True)
|
||||||
p90_forecast = self.data_formatter.format_predictions(output_map["p90"])
|
targets = self.data_formatter.format_predictions(output_map["targets"])
|
||||||
tf.keras.backend.set_session(default_keras_session)
|
p50_forecast = self.data_formatter.format_predictions(output_map["p50"])
|
||||||
|
p90_forecast = self.data_formatter.format_predictions(output_map["p90"])
|
||||||
predict = format_score(p90_forecast, 'pred', self.label_shift)
|
tf.keras.backend.set_session(default_keras_session)
|
||||||
label = format_score(targets, 'label', self.label_shift)
|
|
||||||
#===========================Predicting Process===========================
|
predict = format_score(p90_forecast, "pred", self.label_shift)
|
||||||
return predict, label
|
label = format_score(targets, "label", self.label_shift)
|
||||||
|
# ===========================Predicting Process===========================
|
||||||
def finetune(self, dataset: DatasetH):
|
return predict, label
|
||||||
"""
|
|
||||||
finetune model
|
def finetune(self, dataset: DatasetH):
|
||||||
Parameters
|
"""
|
||||||
----------
|
finetune model
|
||||||
dataset : DatasetH
|
Parameters
|
||||||
dataset for finetuning
|
----------
|
||||||
"""
|
dataset : DatasetH
|
||||||
pass
|
dataset for finetuning
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|||||||
@@ -1,130 +1,132 @@
|
|||||||
#Copyright (c) Microsoft Corporation.
|
# Copyright (c) Microsoft Corporation.
|
||||||
#Licensed under the MIT License.
|
# Licensed under the MIT License.
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import qlib
|
import qlib
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from qlib.config import REG_CN
|
from qlib.config import REG_CN
|
||||||
from qlib.contrib.model.pytorch_lstm import LSTM
|
from qlib.contrib.model.pytorch_lstm import LSTM
|
||||||
from qlib.contrib.data.handler import ALPHA360_Denoise
|
from qlib.contrib.data.handler import ALPHA360_Denoise
|
||||||
from qlib.contrib.strategy.strategy import TopkDropoutStrategy
|
from qlib.contrib.strategy.strategy import TopkDropoutStrategy
|
||||||
from qlib.contrib.evaluate import (
|
from qlib.contrib.evaluate import (
|
||||||
backtest as normal_backtest,
|
backtest as normal_backtest,
|
||||||
risk_analysis,
|
risk_analysis,
|
||||||
)
|
)
|
||||||
from qlib.utils import exists_qlib_data
|
from qlib.utils import exists_qlib_data
|
||||||
|
|
||||||
# from qlib.model.learner import train_model
|
# from qlib.model.learner import train_model
|
||||||
from qlib.utils import init_instance_by_config
|
from qlib.utils import init_instance_by_config
|
||||||
|
|
||||||
import pickle
|
import pickle
|
||||||
from tft import TFTModel
|
from tft import TFTModel
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
# use default data
|
# use default data
|
||||||
provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
|
provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
|
||||||
if not exists_qlib_data(provider_uri):
|
if not exists_qlib_data(provider_uri):
|
||||||
print(f"Qlib data is not found in {provider_uri}")
|
print(f"Qlib data is not found in {provider_uri}")
|
||||||
sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts")))
|
sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts")))
|
||||||
from get_data import GetData
|
from get_data import GetData
|
||||||
|
|
||||||
GetData().qlib_data_cn(target_dir=provider_uri)
|
GetData().qlib_data_cn(target_dir=provider_uri)
|
||||||
|
|
||||||
qlib.init(provider_uri=provider_uri, region=REG_CN)
|
qlib.init(provider_uri=provider_uri, region=REG_CN)
|
||||||
|
|
||||||
MARKET = "csi300"
|
MARKET = "csi300"
|
||||||
BENCHMARK = "SH000300"
|
BENCHMARK = "SH000300"
|
||||||
|
|
||||||
###################################
|
###################################
|
||||||
# train model
|
# train model
|
||||||
###################################
|
###################################
|
||||||
DATA_HANDLER_CONFIG = {
|
DATA_HANDLER_CONFIG = {
|
||||||
"start_time": "2008-01-01",
|
"start_time": "2008-01-01",
|
||||||
"end_time": "2020-08-01",
|
"end_time": "2020-08-01",
|
||||||
"fit_start_time": "2008-01-01",
|
"fit_start_time": "2008-01-01",
|
||||||
"fit_end_time": "2014-12-31",
|
"fit_end_time": "2014-12-31",
|
||||||
"instruments": MARKET,
|
"instruments": MARKET,
|
||||||
}
|
}
|
||||||
|
|
||||||
TRAINER_CONFIG = {
|
TRAINER_CONFIG = {
|
||||||
"train_start_time": "2008-01-01",
|
"train_start_time": "2008-01-01",
|
||||||
"train_end_time": "2014-12-31",
|
"train_end_time": "2014-12-31",
|
||||||
"validate_start_time": "2015-01-01",
|
"validate_start_time": "2015-01-01",
|
||||||
"validate_end_time": "2016-12-31",
|
"validate_end_time": "2016-12-31",
|
||||||
"test_start_time": "2017-01-01",
|
"test_start_time": "2017-01-01",
|
||||||
"test_end_time": "2020-08-01",
|
"test_end_time": "2020-08-01",
|
||||||
}
|
}
|
||||||
|
|
||||||
task = {
|
task = {
|
||||||
"dataset": {
|
"dataset": {
|
||||||
"class": "DatasetH",
|
"class": "DatasetH",
|
||||||
"module_path": "qlib.data.dataset",
|
"module_path": "qlib.data.dataset",
|
||||||
"kwargs": {
|
"kwargs": {
|
||||||
'handler': {
|
"handler": {
|
||||||
"class": "Alpha158",
|
"class": "Alpha158",
|
||||||
"module_path": "qlib.contrib.data.handler",
|
"module_path": "qlib.contrib.data.handler",
|
||||||
"kwargs": DATA_HANDLER_CONFIG
|
"kwargs": DATA_HANDLER_CONFIG,
|
||||||
},
|
},
|
||||||
'segments': {
|
"segments": {
|
||||||
'train': ("2008-01-01", "2014-12-31"),
|
"train": ("2008-01-01", "2014-12-31"),
|
||||||
'valid': ("2015-01-01", "2016-12-31",),
|
"valid": (
|
||||||
'test': ("2017-01-01", "2020-08-01",),
|
"2015-01-01",
|
||||||
}
|
"2016-12-31",
|
||||||
}
|
),
|
||||||
}
|
"test": (
|
||||||
# You shoud record the data in specific sequence
|
"2017-01-01",
|
||||||
# "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
|
"2020-08-01",
|
||||||
}
|
),
|
||||||
|
},
|
||||||
|
},
|
||||||
model = TFTModel()
|
}
|
||||||
dataset = init_instance_by_config(task["dataset"])
|
# You shoud record the data in specific sequence
|
||||||
model.fit(dataset)
|
# "record": ['SignalRecord', 'SigAnaRecord', 'PortAnaRecord'],
|
||||||
|
}
|
||||||
pred_score, label_score = model.predict(dataset)
|
|
||||||
|
model = TFTModel()
|
||||||
# save pred_score to file
|
dataset = init_instance_by_config(task["dataset"])
|
||||||
pred_score_path = Path("~/tmp/qlib/pred_score.pkl").expanduser()
|
model.fit(dataset)
|
||||||
pred_score_path.parent.mkdir(exist_ok=True, parents=True)
|
|
||||||
pred_score.to_pickle(pred_score_path)
|
pred_score, label_score = model.predict(dataset)
|
||||||
|
|
||||||
|
# save pred_score to file
|
||||||
###################################
|
pred_score_path = Path("~/tmp/qlib/pred_score.pkl").expanduser()
|
||||||
# backtest
|
pred_score_path.parent.mkdir(exist_ok=True, parents=True)
|
||||||
###################################
|
pred_score.to_pickle(pred_score_path)
|
||||||
STRATEGY_CONFIG = {
|
|
||||||
"topk": 50,
|
###################################
|
||||||
"n_drop": 5,
|
# backtest
|
||||||
}
|
###################################
|
||||||
BACKTEST_CONFIG = {
|
STRATEGY_CONFIG = {
|
||||||
"verbose": False,
|
"topk": 50,
|
||||||
"limit_threshold": 0.095,
|
"n_drop": 5,
|
||||||
"account": 100000000,
|
}
|
||||||
"benchmark": BENCHMARK,
|
BACKTEST_CONFIG = {
|
||||||
"deal_price": "close",
|
"verbose": False,
|
||||||
"open_cost": 0.0005,
|
"limit_threshold": 0.095,
|
||||||
"close_cost": 0.0015,
|
"account": 100000000,
|
||||||
"min_cost": 5,
|
"benchmark": BENCHMARK,
|
||||||
}
|
"deal_price": "close",
|
||||||
|
"open_cost": 0.0005,
|
||||||
# use default strategy
|
"close_cost": 0.0015,
|
||||||
# custom Strategy, refer to: TODO: Strategy API url
|
"min_cost": 5,
|
||||||
strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)
|
}
|
||||||
report_normal, positions_normal = normal_backtest(pred_score, strategy=strategy, **BACKTEST_CONFIG)
|
|
||||||
|
# use default strategy
|
||||||
###################################
|
# custom Strategy, refer to: TODO: Strategy API url
|
||||||
# analyze
|
strategy = TopkDropoutStrategy(**STRATEGY_CONFIG)
|
||||||
# If need a more detailed analysis, refer to: examples/train_and_bakctest.ipynb
|
report_normal, positions_normal = normal_backtest(pred_score, strategy=strategy, **BACKTEST_CONFIG)
|
||||||
###################################
|
|
||||||
analysis = dict()
|
###################################
|
||||||
analysis["excess_return_without_cost"] = risk_analysis(report_normal["return"] - report_normal["bench"])
|
# analyze
|
||||||
analysis["excess_return_with_cost"] = risk_analysis(
|
# If need a more detailed analysis, refer to: examples/train_and_bakctest.ipynb
|
||||||
report_normal["return"] - report_normal["bench"] - report_normal["cost"]
|
###################################
|
||||||
)
|
analysis = dict()
|
||||||
analysis_df = pd.concat(analysis) # type: pd.DataFrame
|
analysis["excess_return_without_cost"] = risk_analysis(report_normal["return"] - report_normal["bench"])
|
||||||
print(analysis_df)
|
analysis["excess_return_with_cost"] = risk_analysis(
|
||||||
|
report_normal["return"] - report_normal["bench"] - report_normal["cost"]
|
||||||
|
)
|
||||||
|
analysis_df = pd.concat(analysis) # type: pd.DataFrame
|
||||||
|
print(analysis_df)
|
||||||
|
|||||||
Reference in New Issue
Block a user