From 45f73361e3bbcf6625600c52dccff9b3cdca723c Mon Sep 17 00:00:00 2001 From: lwwang1995 Date: Thu, 18 Mar 2021 11:17:42 +0800 Subject: [PATCH 01/26] add tcts baseline --- .../TCTS/workflow_config_tcts_Alpha360.yaml | 93 +++++ qlib/contrib/model/pytorch_tcts.py | 393 ++++++++++++++++++ 2 files changed, 486 insertions(+) create mode 100644 examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml create mode 100644 qlib/contrib/model/pytorch_tcts.py diff --git a/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml b/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml new file mode 100644 index 000000000..589f4b43e --- /dev/null +++ b/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml @@ -0,0 +1,93 @@ +qlib_init: + provider_uri: "~/.qlib/qlib_data/cn_data" + region: cn +market: &market csi300 +benchmark: &benchmark SH000300 +data_handler_config: &data_handler_config + start_time: 2008-01-01 + end_time: 2020-08-01 + fit_start_time: 2008-01-01 + fit_end_time: 2014-12-31 + instruments: *market + infer_processors: + - class: RobustZScoreNorm + kwargs: + fields_group: feature + clip_outlier: true + - class: Fillna + kwargs: + fields_group: feature + learn_processors: + - class: DropnaLabel + - class: CSRankNorm + kwargs: + fields_group: label + label: ["Ref($close, -2) / Ref($close, -1) - 1", + "Ref($close, -3) / Ref($close, -1) - 1", + "Ref($close, -4) / Ref($close, -1) - 1", + "Ref($close, -5) / Ref($close, -1) - 1", + "Ref($close, -6) / Ref($close, -1) - 1"] +port_analysis_config: &port_analysis_config + strategy: + class: TopkDropoutStrategy + module_path: qlib.contrib.strategy.strategy + kwargs: + topk: 50 + n_drop: 5 + backtest: + verbose: False + limit_threshold: 0.095 + account: 100000000 + benchmark: *benchmark + deal_price: close + open_cost: 0.0005 + close_cost: 0.0015 + min_cost: 5 +task: + model: + class: TCTS + module_path: qlib.contrib.model.pytorch_tcts + kwargs: + d_feat: 6 + hidden_size: 64 + num_layers: 2 + dropout: 0.0 + n_epochs: 200 + lr: 1e-3 + early_stop: 20 + batch_size: 800 + metric: loss + loss: mse + GPU: 0 + fore_optimizer: adam + weight_optimizer: adam + output_dim: 5 + fore_lr: 5e-7 + weight_lr: 5e-7 + steps: 3 + target_label: 0 + dataset: + class: DatasetH + module_path: qlib.data.dataset + kwargs: + handler: + class: Alpha360 + module_path: qlib.contrib.data.handler + kwargs: *data_handler_config + segments: + train: [2008-01-01, 2014-12-31] + valid: [2015-01-01, 2016-12-31] + test: [2017-01-01, 2020-08-01] + record: + - class: SignalRecord + module_path: qlib.workflow.record_temp + kwargs: {} + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 + - class: PortAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + config: *port_analysis_config \ No newline at end of file diff --git a/qlib/contrib/model/pytorch_tcts.py b/qlib/contrib/model/pytorch_tcts.py new file mode 100644 index 000000000..9f44ba31c --- /dev/null +++ b/qlib/contrib/model/pytorch_tcts.py @@ -0,0 +1,393 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + + +from __future__ import division +from __future__ import print_function + +import os +import numpy as np +import pandas as pd +import copy +from sklearn.metrics import roc_auc_score, mean_squared_error +import logging +from ...utils import ( + unpack_archive_with_buffer, + save_multiple_parts_file, + create_save_path, + drop_nan_by_y_index, +) +from ...log import get_module_logger, TimeInspector + +import torch +import torch.nn as nn +import torch.optim as optim + +from ...model.base import Model +from ...data.dataset import DatasetH +from ...data.dataset.handler import DataHandlerLP + + +class TCTS(Model): + """TCTS Model + + Parameters + ---------- + d_feat : int + input dimension for each time step + metric: str + the evaluate metric used in early stop + optimizer : str + optimizer name + GPU : str + the GPU ID(s) used for training + """ + + def __init__( + self, + d_feat=6, + hidden_size=64, + num_layers=2, + dropout=0.0, + n_epochs=200, + batch_size=2000, + early_stop=20, + loss="mse", + fore_optimizer="adam", + weight_optimizer="adam", + output_dim=5, + fore_lr=5e-7, + weight_lr=5e-7, + steps=3, + GPU=0, + seed=None, + target_label=0, + **kwargs + ): + # Set logger. + self.logger = get_module_logger("TCTS") + self.logger.info("TCTS pytorch version...") + + # set hyper-parameters. + self.d_feat = d_feat + self.hidden_size = hidden_size + self.num_layers = num_layers + self.dropout = dropout + self.n_epochs = n_epochs + self.batch_size = batch_size + self.early_stop = early_stop + self.loss = loss + self.device = torch.device("cuda:%d" % (GPU) if torch.cuda.is_available() else "cpu") + self.use_gpu = torch.cuda.is_available() + self.seed = seed + self.output_dim = output_dim + self.fore_lr = fore_lr + self.weight_lr = weight_lr + self.steps = steps + self.target_label = target_label + + self.logger.info( + "TCTS parameters setting:" + "\nd_feat : {}" + "\nhidden_size : {}" + "\nnum_layers : {}" + "\ndropout : {}" + "\nn_epochs : {}" + "\nbatch_size : {}" + "\nearly_stop : {}" + "\nloss_type : {}" + "\nvisible_GPU : {}" + "\nuse_GPU : {}" + "\nseed : {}".format( + d_feat, + hidden_size, + num_layers, + dropout, + n_epochs, + batch_size, + early_stop, + loss, + GPU, + self.use_gpu, + seed, + ) + ) + + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) + + self.fore_model = GRUModel( + d_feat=self.d_feat, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + dropout=self.dropout, + ) + self.weight_model = MLPModel( + d_feat=360 + 2 * self.output_dim + 1, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + dropout=self.dropout, + output_dim=self.output_dim, + ) + if fore_optimizer.lower() == "adam": + self.fore_optimizer = optim.Adam(self.fore_model.parameters(), lr=self.fore_lr) + elif fore_optimizer.lower() == "gd": + self.fore_optimizer = optim.SGD(self.fore_model.parameters(), lr=self.fore_lr) + else: + raise NotImplementedError("optimizer {} is not supported!".format(fore_optimizer)) + if weight_optimizer.lower() == "adam": + self.weight_optimizer = optim.Adam(self.weight_model.parameters(), lr=self.weight_lr) + elif weight_optimizer.lower() == "gd": + self.weight_optimizer = optim.SGD(self.weight_model.parameters(), lr=self.weight_lr) + else: + raise NotImplementedError("optimizer {} is not supported!".format(weight_optimizer)) + + self.fitted = False + self.fore_model.to(self.device) + self.weight_model.to(self.device) + + def loss_fn(self, pred, label, weight): + + loc = torch.argmax(weight, 1) + loss = (pred - label[np.arange(weight.shape[0]), loc]) ** 2 + return torch.mean(loss) + + def train_epoch(self, x_train, y_train, x_valid, y_valid): + + x_train_values = x_train.values + y_train_values = np.squeeze(y_train.values) + + indices = np.arange(len(x_train_values)) + np.random.shuffle(indices) + + init_fore_model = copy.deepcopy(self.fore_model) + for p in init_fore_model.parameters(): + p.init_fore_model = False + + self.fore_model.train() + self.weight_model.train() + + for p in self.weight_model.parameters(): + p.requires_grad = False + for p in self.fore_model.parameters(): + p.requires_grad = True + + for i in range(self.steps): + for i in range(len(indices))[:: self.batch_size]: + + if len(indices) - i < self.batch_size: + break + + feature = torch.from_numpy(x_train_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_train_values[indices[i : i + self.batch_size]]).float().to(self.device) + + init_pred = init_fore_model(feature) + pred = self.fore_model(feature) + + dis = init_pred - label.transpose(0, 1) + weight_feature = torch.cat((feature, dis.transpose(0, 1), label, init_pred.view(-1, 1)), 1) + weight = self.weight_model(weight_feature) + + loss = self.loss_fn(pred, label, weight) # hard + + self.fore_optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_value_(self.fore_model.parameters(), 3.0) + self.fore_optimizer.step() + + x_valid_values = x_valid.values + y_valid_values = np.squeeze(y_valid.values) + + indices = np.arange(len(x_valid_values)) + np.random.shuffle(indices) + for p in self.weight_model.parameters(): + p.requires_grad = True + for p in self.fore_model.parameters(): + p.requires_grad = False + + # fix forecasting model and valid weight model + for i in range(len(indices))[:: self.batch_size]: + + if len(indices) - i < self.batch_size: + break + + feature = torch.from_numpy(x_valid_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_valid_values[indices[i : i + self.batch_size]]).float().to(self.device) + + pred = self.fore_model(feature) + dis = pred - label.transpose(0, 1) + weight_feature = torch.cat((feature, dis.transpose(0, 1), label, pred.view(-1, 1)), 1) + weight = self.weight_model(weight_feature) + loc = torch.argmax(weight, 1) + valid_loss = torch.mean((pred - label[:, 0]) ** 2) + loss = torch.mean(-valid_loss * torch.log(weight[np.arange(weight.shape[0]), loc])) + + self.weight_optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_value_(self.weight_model.parameters(), 3.0) + self.weight_optimizer.step() + + def test_epoch(self, data_x, data_y): + + # prepare training data + x_values = data_x.values + y_values = np.squeeze(data_y.values) + + self.fore_model.eval() + + scores = [] + losses = [] + + indices = np.arange(len(x_values)) + + for i in range(len(indices))[:: self.batch_size]: + + if len(indices) - i < self.batch_size: + break + + feature = torch.from_numpy(x_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_values[indices[i : i + self.batch_size]]).float().to(self.device) + + pred = self.fore_model(feature) + loss = torch.mean((pred - label[:, abs(self.target_label)]) ** 2) + losses.append(loss.item()) + + return np.mean(losses) + + def fit( + self, + dataset: DatasetH, + evals_result=dict(), + verbose=True, + save_path=None, + ): + + df_train, df_valid, df_test = dataset.prepare( + ["train", "valid", "test"], + col_set=["feature", "label"], + data_key=DataHandlerLP.DK_L, + ) + + x_train, y_train = df_train["feature"], df_train["label"] + x_valid, y_valid = df_valid["feature"], df_valid["label"] + x_test, y_test = df_test["feature"], df_test["label"] + + if save_path == None: + save_path = create_save_path(save_path) + + best_loss = np.inf + best_epoch = 0 + stop_round = 0 + fore_best_param = copy.deepcopy(self.fore_optimizer.state_dict()) + weight_best_param = copy.deepcopy(self.weight_optimizer.state_dict()) + + for epoch in range(self.n_epochs): + print("Epoch:", epoch) + + print("training...") + self.train_epoch(x_train, y_train, x_valid, y_valid) + print("evaluating...") + val_loss = self.test_epoch(x_valid, y_valid) + test_loss = self.test_epoch(x_test, y_test) + + print("valid %.6f, test %.6f" % (val_loss, test_loss)) + + if val_loss < best_loss: + best_loss = val_loss + stop_round = 0 + best_epoch = epoch + torch.save(copy.deepcopy(self.fore_model.state_dict()), save_path + "_fore_model.bin") + torch.save(copy.deepcopy(self.weight_model.state_dict()), save_path + "_weight_model.bin") + + else: + stop_round += 1 + if stop_round >= self.early_stop: + print("early stop") + break + + print("best loss:", best_loss, "@", best_epoch) + best_param = torch.load(save_path + "_fore_model.bin") + self.fore_model.load_state_dict(best_param) + best_param = torch.load(save_path + "_weight_model.bin") + self.weight_model.load_state_dict(best_param) + self.fitted = True + + if self.use_gpu: + torch.cuda.empty_cache() + + def predict(self, dataset): + if not self.fitted: + raise ValueError("model is not fitted yet!") + + x_test = dataset.prepare("test", col_set="feature") + index = x_test.index + self.fore_model.eval() + x_values = x_test.values + sample_num = x_values.shape[0] + preds = [] + + for begin in range(sample_num)[:: self.batch_size]: + + if sample_num - begin < self.batch_size: + end = sample_num + else: + end = begin + self.batch_size + + x_batch = torch.from_numpy(x_values[begin:end]).float().to(self.device) + + with torch.no_grad(): + if self.use_gpu: + pred = self.fore_model(x_batch).detach().cpu().numpy() + else: + pred = self.fore_model(x_batch).detach().numpy() + + preds.append(pred) + + return pd.Series(np.concatenate(preds), index=index) + + +class MLPModel(nn.Module): + def __init__(self, d_feat, hidden_size=256, num_layers=3, dropout=0.0, output_dim=1): + super().__init__() + + self.mlp = nn.Sequential() + self.softmax = nn.Softmax(dim=1) + + for i in range(num_layers): + if i > 0: + self.mlp.add_module("drop_%d" % i, nn.Dropout(dropout)) + self.mlp.add_module("fc_%d" % i, nn.Linear(d_feat if i == 0 else hidden_size, hidden_size)) + self.mlp.add_module("relu_%d" % i, nn.ReLU()) + + self.mlp.add_module("fc_out", nn.Linear(hidden_size, output_dim)) + + def forward(self, x): + # feature + # [N, F] + out = self.mlp(x).squeeze() + out = self.softmax(out) + return out + + +class GRUModel(nn.Module): + def __init__(self, d_feat=6, hidden_size=64, num_layers=2, dropout=0.0): + super().__init__() + + self.rnn = nn.GRU( + input_size=d_feat, + hidden_size=hidden_size, + num_layers=num_layers, + batch_first=True, + dropout=dropout, + ) + self.fc_out = nn.Linear(hidden_size, 1) + + self.d_feat = d_feat + + def forward(self, x): + # x: [N, F*T] + x = x.reshape(len(x), self.d_feat, -1) # [N, F, T] + x = x.permute(0, 2, 1) # [N, T, F] + out, _ = self.rnn(x) + return self.fc_out(out[:, -1, :]).squeeze() From b24af7fff6311a2ff0e8e5456b359febf5d6099c Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Mon, 24 May 2021 05:07:38 +0000 Subject: [PATCH 02/26] multiprocessing support --- .../model_rolling/task_manager_rolling.py | 10 ++- .../online_srv/online_management_simulate.py | 15 +++- .../online_srv/rolling_online_management.py | 22 ++++- qlib/model/trainer.py | 88 ++++++++++++++++++- qlib/workflow/online/manager.py | 29 +++--- qlib/workflow/task/manage.py | 7 +- 6 files changed, 145 insertions(+), 26 deletions(-) diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index 4f3ac04b1..89233b37b 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -4,6 +4,7 @@ """ This example shows how a TrainerRM works based on TaskManager with rolling tasks. After training, how to collect the rolling results will be shown in task_collecting. +Based on the ability of TaskManager, `worker` method offer a simple way for multiprocessing. """ from pprint import pprint @@ -13,10 +14,10 @@ import qlib from qlib.config import REG_CN from qlib.workflow import R from qlib.workflow.task.gen import RollingGen, task_generator -from qlib.workflow.task.manage import TaskManager +from qlib.workflow.task.manage import TaskManager, run_task from qlib.workflow.task.collect import RecorderCollector from qlib.model.ens.group import RollingGroup -from qlib.model.trainer import TrainerRM +from qlib.model.trainer import TrainerRM, task_train data_handler_config = { @@ -122,6 +123,11 @@ class RollingTaskExample: trainer = TrainerRM(self.experiment_name, self.task_pool) trainer.train(tasks) + def worker(self): + # train tasks by other progress or machines for multiprocessing. It is same as TrainerRM.worker. + print("========== worker ==========") + run_task(task_train, self.task_pool, experiment_name=self.experiment_name) + def task_collecting(self): print("========== task_collecting ==========") diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 4bb5022ee..de6dbcb21 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -78,8 +78,8 @@ class OnlineSimulationExample: provider_uri="~/.qlib/qlib_data/cn_data", region="cn", exp_name="rolling_exp", - task_url="mongodb://10.0.0.4:27017/", - task_db_name="rolling_db", + task_url="mongodb://10.0.0.4:27017/", # not necessary when using TrainerR or DelayTrainerR + task_db_name="rolling_db", # not necessary when using TrainerR or DelayTrainerR task_pool="rolling_task", rolling_step=80, start_time="2018-09-10", @@ -113,7 +113,7 @@ class OnlineSimulationExample: self.rolling_gen = RollingGen( step=rolling_step, rtype=RollingGen.ROLL_SD, ds_extra_mod_func=None ) # The rolling tasks generator, ds_extra_mod_func is None because we just need to simulate to 2018-10-31 and needn't change the handler end time. - self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) # Also can be TrainerR, TrainerRM, DelayTrainerR + self.trainer = TrainerRM(self.exp_name, self.task_pool) # Also can be TrainerR, TrainerRM, DelayTrainerR self.rolling_online_manager = OnlineManager( RollingStrategy(exp_name, task_template=tasks, rolling_gen=self.rolling_gen), trainer=self.trainer, @@ -139,6 +139,15 @@ class OnlineSimulationExample: print("========== signals ==========") print(self.rolling_online_manager.get_signals()) + def worker(self): + # train tasks by other progress or machines for multiprocessing + # FIXME: only can call after finishing simulation when using DelayTrainerRM, or there will be some exception. + print("========== worker ==========") + if isinstance(self.trainer, TrainerRM): + self.trainer.worker() + else: + print(f"{type(self.trainer)} is not supported for worker.") + if __name__ == "__main__": ## to run all workflow automatically with your own parameters, use the command below diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index 25b8b2a0c..40da30db7 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -13,10 +13,12 @@ Finally, the OnlineManager will finish second routine and update all strategies. import os import fire import qlib +from qlib.model.trainer import DelayTrainerR, DelayTrainerRM, TrainerR, TrainerRM, end_task_train, task_train from qlib.workflow import R from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.online.manager import OnlineManager +from qlib.workflow.task.manage import TaskManager, run_task data_handler_config = { "start_time": "2013-01-01", @@ -80,8 +82,9 @@ class RollingOnlineExample: self, provider_uri="~/.qlib/qlib_data/cn_data", region="cn", - task_url="mongodb://10.0.0.4:27017/", - task_db_name="rolling_db", + trainer=DelayTrainerRM(), # you can choose from TrainerR, TrainerRM, DelayTrainerR, DelayTrainerRM + task_url="mongodb://10.0.0.4:27017/", # not necessary when using TrainerR or DelayTrainerR + task_db_name="rolling_db", # not necessary when using TrainerR or DelayTrainerR rolling_step=550, tasks=[task_xgboost_config], add_tasks=[task_lgb_config], @@ -104,17 +107,28 @@ class RollingOnlineExample: RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD), ) ) - - self.rolling_online_manager = OnlineManager(strategies) + self.trainer = trainer + self.rolling_online_manager = OnlineManager(strategies, trainer=self.trainer) _ROLLING_MANAGER_PATH = ( ".RollingOnlineExample" # the OnlineManager will dump to this file, for it can be loaded when calling routine. ) + def worker(self): + # train tasks by other progress or machines for multiprocessing + print("========== worker ==========") + if isinstance(self.trainer, TrainerRM): + for task in self.tasks + self.add_tasks: + name_id = task["model"]["class"] + self.trainer.worker(experiment_name=name_id) + else: + print(f"{type(self.trainer)} is not supported for worker.") + # Reset all things to the first status, be careful to save important data def reset(self): for task in self.tasks + self.add_tasks: name_id = task["model"]["class"] + TaskManager(task_pool=name_id).remove() exp = R.get_exp(experiment_name=name_id) for rid in exp.list_recorders(): exp.delete_recorder(rid) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index fd76e6728..07bb839a2 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -12,9 +12,11 @@ In ``DelayTrainer``, the first step is only to save some necessary info to model """ import socket +import time from typing import Callable, List from qlib.data.dataset import Dataset +from qlib.log import get_module_logger from qlib.model.base import Model from qlib.utils import flatten_dict, get_cls_kwargs, init_instance_by_config from qlib.workflow import R @@ -190,6 +192,8 @@ class TrainerR(Trainer): Returns: List[Recorder]: a list of Recorders """ + if isinstance(tasks, dict): + tasks = [tasks] if len(tasks) == 0: return [] if train_func is None: @@ -213,6 +217,8 @@ class TrainerR(Trainer): Returns: List[Recorder]: the same list as the param. """ + if isinstance(recs, Recorder): + recs = [recs] for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs @@ -250,6 +256,8 @@ class DelayTrainerR(TrainerR): Returns: List[Recorder]: a list of Recorders """ + if isinstance(recs, Recorder): + recs = [recs] if end_train_func is None: end_train_func = self.end_train_func if experiment_name is None: @@ -315,6 +323,8 @@ class TrainerRM(Trainer): Returns: List[Recorder]: a list of Recorders """ + if isinstance(tasks, dict): + tasks = [tasks] if len(tasks) == 0: return [] if train_func is None: @@ -329,12 +339,24 @@ class TrainerRM(Trainer): run_task( train_func, task_pool, + query={"filter": {"$in": tasks}}, # only train these tasks experiment_name=experiment_name, before_status=before_status, after_status=after_status, **kwargs, ) + # FIXME: reset to waiting automatically + for _id in _id_list: + is_prn = False + while tm.re_query(_id)["status"] == "running": + if not is_prn: + get_module_logger("TrainerRM").warn( + f"A task (_id: {_id}) is not being trained by this Trainer. Ignore this message if it is being trained by others." + ) + is_prn = True + time.sleep(10) + recs = [] for _id in _id_list: rec = tm.re_query(_id)["res"] @@ -352,10 +374,33 @@ class TrainerRM(Trainer): Returns: List[Recorder]: the same list as the param. """ + if isinstance(recs, Recorder): + recs = [recs] for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs + def worker( + self, + train_func: Callable = None, + experiment_name: str = None, + ): + """ + The multiprocessing method for `train`. It can share a same task_pool with `train` and can run in other progress or other machines. + + Args: + train_func (Callable): the training method which needs at least `task`s and `experiment_name`. None for the default training method. + experiment_name (str): the experiment name, None for use default name. + """ + if train_func is None: + train_func = self.train_func + if experiment_name is None: + experiment_name = self.experiment_name + task_pool = self.task_pool + if task_pool is None: + task_pool = experiment_name + run_task(train_func, task_pool=task_pool, experiment_name=experiment_name) + class DelayTrainerRM(TrainerRM): """ @@ -395,6 +440,8 @@ class DelayTrainerRM(TrainerRM): Returns: List[Recorder]: a list of Recorders """ + if isinstance(tasks, dict): + tasks = [tasks] if len(tasks) == 0: return [] return super().train( @@ -410,8 +457,6 @@ class DelayTrainerRM(TrainerRM): Given a list of Recorder and return a list of trained Recorder. This class will finish real data loading and model fitting. - NOTE: This method will train all STATUS_PART_DONE tasks in the task pool, not only the ``recs``. - Args: recs (list): a list of Recorder, the tasks have been saved to them. end_train_func (Callable, optional): the end_train method which need at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. @@ -421,7 +466,8 @@ class DelayTrainerRM(TrainerRM): Returns: List[Recorder]: a list of Recorders """ - + if isinstance(recs, Recorder): + recs = [recs] if end_train_func is None: end_train_func = self.end_train_func if experiment_name is None: @@ -441,6 +487,42 @@ class DelayTrainerRM(TrainerRM): before_status=TaskManager.STATUS_PART_DONE, **kwargs, ) + + # FIXME: reset to waiting automatically + tm = TaskManager(task_pool=task_pool) + for query_task in tm.query({"filter": {"$in": tasks}}): + _id = query_task["_id"] + is_prn = False + while tm.re_query(_id)["status"] == "running": + if not is_prn: + get_module_logger("DelayTrainerRM").warn( + f"A task (_id: {_id}) is not being trained by this Trainer. Ignore this message if it is being trained by others." + ) + is_prn = True + time.sleep(10) + for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs + + def worker(self, end_train_func=None, experiment_name: str = None): + """ + The multiprocessing method for `end_train`. It can share a same task_pool with `end_train` and can run in other progress or other machines. + + Args: + end_train_func (Callable, optional): the end_train method which need at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. + experiment_name (str): the experiment name, None for use default name. + """ + if end_train_func is None: + end_train_func = self.end_train_func + if experiment_name is None: + experiment_name = self.experiment_name + task_pool = self.task_pool + if task_pool is None: + task_pool = experiment_name + run_task( + end_train_func, + task_pool=task_pool, + experiment_name=experiment_name, + before_status=TaskManager.STATUS_PART_DONE, + ) diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index 443cd61ad..ef6cb8dfa 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -18,10 +18,12 @@ There are 4 total situations for using different trainers in different situation ========================= =================================================================================== Situations Description ========================= =================================================================================== -Online + Trainer When you REAL want to do a routine, the Trainer will help you train the models. +Online + Trainer When you want to do a REAL routine, the Trainer will help you train the models. It + will train models task by task and strategy by strategy. -Online + DelayTrainer In normal online routine, whether Trainer or DelayTrainer will REAL train models - in this routine. So it is not necessary to use DelayTrainer when do a REAL routine. +Online + DelayTrainer When your models don't have any temporal dependence, the DelayTrainer will train + nothing until all tasks have been prepared. It makes user can train all tasks in + the end of `routine` or `first_train`. Simulation + Trainer When your models have some temporal dependence on the previous models, then you need to consider using Trainer. This means it will REAL train your models in @@ -103,17 +105,21 @@ class OnlineManager(Serializable): """ if strategies is None: strategies = self.strategies - for strategy in strategies: + models_list = [] + for strategy in strategies: self.logger.info(f"Strategy `{strategy.name_id}` begins first training...") tasks = strategy.first_tasks() models = self.trainer.train(tasks, experiment_name=strategy.name_id) - models = self.trainer.end_train(models, experiment_name=strategy.name_id) + models_list.append(models) self.logger.info(f"Finished training {len(models)} models.") - online_models = strategy.prepare_online_models(models, **model_kwargs) self.history.setdefault(self.cur_time, {})[strategy] = online_models + if not self.status == self.STATUS_SIMULATING or not self.trainer.is_delay(): + for strategy, models in zip(strategies, models_list): + models = self.trainer.end_train(models, experiment_name=strategy.name_id) + def routine( self, cur_time: Union[str, pd.Timestamp] = None, @@ -139,20 +145,22 @@ class OnlineManager(Serializable): cur_time = D.calendar(freq=self.freq).max() self.cur_time = pd.Timestamp(cur_time) # None for latest date + models_list = [] for strategy in self.strategies: self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") if self.status == self.STATUS_NORMAL: strategy.tool.update_online_pred() tasks = strategy.prepare_tasks(self.cur_time, **task_kwargs) - models = self.trainer.train(tasks) - if self.status == self.STATUS_NORMAL or not self.trainer.is_delay(): - models = self.trainer.end_train(models, experiment_name=strategy.name_id) + models = self.trainer.train(tasks, experiment_name=strategy.name_id) + models_list.append(models) self.logger.info(f"Finished training {len(models)} models.") online_models = strategy.prepare_online_models(models, **model_kwargs) self.history.setdefault(self.cur_time, {})[strategy] = online_models - if not self.trainer.is_delay(): + if not self.status == self.STATUS_SIMULATING or not self.trainer.is_delay(): + for strategy, models in zip(self.strategies, models_list): + models = self.trainer.end_train(models, experiment_name=strategy.name_id) self.prepare_signals(**signal_kwargs) def get_collector(self) -> MergeCollector: @@ -297,6 +305,7 @@ class OnlineManager(Serializable): # NOTE: Assumption: the predictions of online models need less than next cur_time, or this method will work in a wrong way. self.prepare_signals(**signal_kwargs) if signals_time > cur_time: + # FIXME: if use DelayTrainer and worker (and worker is faster than main progress), there are some possibilities of showing this warning. self.logger.warn( f"The signals have already parpred to {signals_time} by last preparation, but current time is only {cur_time}. This may be because the online models predict more than they should, which can cause signals to be contaminated by the offline models." ) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 658eec4d6..0e495bb0f 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -69,7 +69,7 @@ class TaskManager: ENCODE_FIELDS_PREFIX = ["def", "res"] - def __init__(self, task_pool: str = None): + def __init__(self, task_pool: str): """ Init Task Manager, remember to make the statement of MongoDB url and database name firstly. @@ -79,8 +79,7 @@ class TaskManager: the name of Collection in MongoDB """ self.mdb = get_mongodb() - if task_pool is not None: - self.task_pool = getattr(self.mdb, task_pool) + self.task_pool = getattr(self.mdb, task_pool) self.logger = get_module_logger(self.__class__.__name__) def list(self) -> list: @@ -288,7 +287,7 @@ class TaskManager: for t in self.task_pool.find(query): yield self._decode_task(t) - def re_query(self, _id): + def re_query(self, _id) -> dict: """ Use _id to query task. From ca0363ded804ad97d21d2d151ef823df9336a7c5 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 27 May 2021 06:04:46 +0000 Subject: [PATCH 03/26] update trainer and manage --- qlib/model/trainer.py | 38 ++++++++++++------------------------ qlib/workflow/task/manage.py | 34 ++++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 07bb839a2..ace3031ed 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -283,6 +283,9 @@ class TrainerRM(Trainer): STATUS_BEGIN = "begin_task_train" STATUS_END = "end_task_train" + # This tag is the _id in TaskManager to distinguish tasks. + TM_ID = "_id in TaskManager" + def __init__(self, experiment_name: str = None, task_pool: str = None, train_func=task_train): """ Init TrainerR. @@ -336,31 +339,24 @@ class TrainerRM(Trainer): task_pool = experiment_name tm = TaskManager(task_pool=task_pool) _id_list = tm.create_task(tasks) # all tasks will be saved to MongoDB + query = {"_id": {"$in": _id_list}} run_task( train_func, task_pool, - query={"filter": {"$in": tasks}}, # only train these tasks + query=query, # only train these tasks experiment_name=experiment_name, before_status=before_status, after_status=after_status, **kwargs, ) - # FIXME: reset to waiting automatically - for _id in _id_list: - is_prn = False - while tm.re_query(_id)["status"] == "running": - if not is_prn: - get_module_logger("TrainerRM").warn( - f"A task (_id: {_id}) is not being trained by this Trainer. Ignore this message if it is being trained by others." - ) - is_prn = True - time.sleep(10) + tm.wait(query=query) recs = [] for _id in _id_list: rec = tm.re_query(_id)["res"] rec.set_tags(**{self.STATUS_KEY: self.STATUS_BEGIN}) + rec.set_tags(**{self.TM_ID: _id}) recs.append(rec) return recs @@ -475,31 +471,21 @@ class DelayTrainerRM(TrainerRM): task_pool = self.task_pool if task_pool is None: task_pool = experiment_name - tasks = [] + _id_list = [] for rec in recs: - tasks.append(rec.load_object("task")) + _id_list.append(rec.list_tags()[self.TM_ID]) + query = {"_id": {"$in": _id_list}} run_task( end_train_func, task_pool, - query={"filter": {"$in": tasks}}, # only train these tasks + query=query, # only train these tasks experiment_name=experiment_name, before_status=TaskManager.STATUS_PART_DONE, **kwargs, ) - # FIXME: reset to waiting automatically - tm = TaskManager(task_pool=task_pool) - for query_task in tm.query({"filter": {"$in": tasks}}): - _id = query_task["_id"] - is_prn = False - while tm.re_query(_id)["status"] == "running": - if not is_prn: - get_module_logger("DelayTrainerRM").warn( - f"A task (_id: {_id}) is not being trained by this Trainer. Ignore this message if it is being trained by others." - ) - is_prn = True - time.sleep(10) + TaskManager(task_pool=task_pool).wait(query=query) for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 0e495bb0f..167087260 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -108,6 +108,15 @@ class TaskManager: def _dict_to_str(self, flt): return {k: str(v) for k, v in flt.items()} + def _decode_query(self, query): + if "_id" in query: + if isinstance(query["_id"], dict): + for key in query["_id"]: + query["_id"][key] = [ObjectId(i) for i in query["_id"][key]] + else: + query["_id"] = ObjectId(query["_id"]) + return query + def replace_task(self, task, new_task): """ Use a new task to replace a old one @@ -223,8 +232,7 @@ class TaskManager: dict: a task(document in collection) after decoding """ query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) query.update({"status": status}) task = self.task_pool.find_one_and_update( query, {"$set": {"status": self.STATUS_RUNNING}}, sort=[("priority", pymongo.DESCENDING)] @@ -282,8 +290,7 @@ class TaskManager: dict: a task(document in collection) after decoding """ query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) for t in self.task_pool.find(query): yield self._decode_task(t) @@ -338,8 +345,7 @@ class TaskManager: """ query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) self.task_pool.delete_many(query) def task_stat(self, query={}) -> dict: @@ -353,8 +359,7 @@ class TaskManager: dict """ query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) tasks = self.query(query=query, decode=False) status_stat = {} for t in tasks: @@ -376,8 +381,7 @@ class TaskManager: def reset_status(self, query, status): query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) print(self.task_pool.update_many(query, {"$set": {"status": status}})) def prioritize(self, task, priority: int): @@ -401,9 +405,19 @@ class TaskManager: return sum(task_stat.values()) def wait(self, query={}): + """ + When multiprocessing, the main progress may fetch nothing from TaskManager because there are still some running tasks. + So main progress should wait until all tasks are trained well by other progress or machines. + + Args: + query (dict, optional): the query dict. Defaults to {}. + """ task_stat = self.task_stat(query) total = self._get_total(task_stat) last_undone_n = self._get_undone_n(task_stat) + if last_undone_n == 0: + return + self.logger.warn(f"Waiting for {last_undone_n} undone tasks. Please make sure they are running.") with tqdm(total=total, initial=total - last_undone_n) as pbar: while True: time.sleep(10) From 94ab4bbf3feb5496720c6359dc85cfb1766ed5dd Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 1 Jun 2021 07:45:39 +0000 Subject: [PATCH 04/26] add docs --- qlib/workflow/task/manage.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 167087260..dd42caf65 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -24,7 +24,9 @@ from bson.binary import Binary from bson.objectid import ObjectId from pymongo.errors import InvalidDocument from qlib import auto_init, get_module_logger +import qlib from tqdm.cli import tqdm +import yaml from .utils import get_mongodb @@ -72,24 +74,26 @@ class TaskManager: def __init__(self, task_pool: str): """ Init Task Manager, remember to make the statement of MongoDB url and database name firstly. + A TaskManager instance serves a specific task pool. + The static method of this module serves the whole MongoDB. Parameters ---------- task_pool: str the name of Collection in MongoDB """ - self.mdb = get_mongodb() - self.task_pool = getattr(self.mdb, task_pool) + self.task_pool = getattr(get_mongodb(), task_pool) self.logger = get_module_logger(self.__class__.__name__) - def list(self) -> list: + @staticmethod + def list() -> list: """ - List the all collection(task_pool) of the db + List the all collection(task_pool) of the db. Returns: list """ - return self.mdb.list_collection_names() + return get_mongodb().list_collection_names() def _encode_task(self, task): for prefix in self.ENCODE_FIELDS_PREFIX: @@ -109,6 +113,16 @@ class TaskManager: return {k: str(v) for k, v in flt.items()} def _decode_query(self, query): + """ + If the query includes any `_id`, then it needs `ObjectId` to decode. + For example, when using TrainerRM, it needs query `{"_id": {"$in": _id_list}}`. Then we need to `ObjectId` every `_id` in `_id_list`. + + Args: + query (dict): query dict. Defaults to {}. + + Returns: + dict: the query after decoding. + """ if "_id" in query: if isinstance(query["_id"], dict): for key in query["_id"]: From ab6b88ce14814fe5679e4f5b5f9a016a2397c1a6 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 1 Jun 2021 07:48:14 +0000 Subject: [PATCH 05/26] delete useless import --- qlib/workflow/task/manage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index dd42caf65..7a85036da 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -24,9 +24,7 @@ from bson.binary import Binary from bson.objectid import ObjectId from pymongo.errors import InvalidDocument from qlib import auto_init, get_module_logger -import qlib from tqdm.cli import tqdm -import yaml from .utils import get_mongodb From 8d05cd2dafcb8f8dbf2bcdb453b5d9236d3bd766 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 1 Jun 2021 09:40:53 +0000 Subject: [PATCH 06/26] modify tests.config.py --- .../online_srv/online_management_simulate.py | 64 ++++++++++++- .../online_srv/rolling_online_management.py | 6 +- qlib/tests/config.py | 94 ++++++++++++++----- 3 files changed, 138 insertions(+), 26 deletions(-) diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 5f024192f..8650859ff 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -5,6 +5,7 @@ This example is about how can simulate the OnlineManager based on rolling tasks. """ +from pprint import pprint import fire import qlib from qlib.model.trainer import DelayTrainerR, DelayTrainerRM, TrainerR, TrainerRM @@ -13,7 +14,63 @@ from qlib.workflow.online.manager import OnlineManager from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.task.manage import TaskManager -from qlib.tests.config import CSI100_RECORD_LGB_TASK_CONFIG, CSI100_RECORD_XGBOOST_TASK_CONFIG +from qlib.tests.config import CSI100_RECORD_LGB_TASK_CONFIG_ONLINE, CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE + +data_handler_config = { + "start_time": "2018-01-01", + "end_time": "2018-10-31", + "fit_start_time": "2018-01-01", + "fit_end_time": "2018-03-31", + "instruments": "csi100", +} + +dataset_config = { + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": { + "class": "Alpha158", + "module_path": "qlib.contrib.data.handler", + "kwargs": data_handler_config, + }, + "segments": { + "train": ("2018-01-01", "2018-03-31"), + "valid": ("2018-04-01", "2018-05-31"), + "test": ("2018-06-01", "2018-09-10"), + }, + }, +} + +record_config = [ + { + "class": "SignalRecord", + "module_path": "qlib.workflow.record_temp", + }, + { + "class": "SigAnaRecord", + "module_path": "qlib.workflow.record_temp", + }, +] + +# use lgb model +task_lgb_config = { + "model": { + "class": "LGBModel", + "module_path": "qlib.contrib.model.gbdt", + }, + "dataset": dataset_config, + "record": record_config, +} + +# use xgboost model +task_xgboost_config = { + "model": { + "class": "XGBModel", + "module_path": "qlib.contrib.model.xgboost", + }, + "dataset": dataset_config, + "record": record_config, +} class OnlineSimulationExample: @@ -46,7 +103,10 @@ class OnlineSimulationExample: tasks (dict or list[dict]): a set of the task config waiting for rolling and training """ if tasks is None: - tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG, CSI100_RECORD_LGB_TASK_CONFIG] + #tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE, CSI100_RECORD_LGB_TASK_CONFIG_ONLINE] + tasks = [task_xgboost_config, task_lgb_config] + #pprint(CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE) + #pprint(task_xgboost_config) self.exp_name = exp_name self.task_pool = task_pool self.start_time = start_time diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index b4f7245b7..99a91e027 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -18,7 +18,7 @@ from qlib.workflow import R from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.online.manager import OnlineManager -from qlib.tests.config import CSI100_RECORD_XGBOOST_TASK_CONFIG, CSI100_RECORD_LGB_TASK_CONFIG +from qlib.tests.config import CSI100_RECORD_XGBOOST_TASK_CONFIG_ROLLING, CSI100_RECORD_LGB_TASK_CONFIG_ROLLING class RollingOnlineExample: @@ -34,9 +34,9 @@ class RollingOnlineExample: add_tasks=None, ): if add_tasks is None: - add_tasks = [CSI100_RECORD_LGB_TASK_CONFIG] + add_tasks = [CSI100_RECORD_LGB_TASK_CONFIG_ROLLING] if tasks is None: - tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG] + tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG_ROLLING] mongo_conf = { "task_url": task_url, # your MongoDB url "task_db_name": task_db_name, # database name diff --git a/qlib/tests/config.py b/qlib/tests/config.py index 80461f6f9..c61b5651e 100644 --- a/qlib/tests/config.py +++ b/qlib/tests/config.py @@ -43,17 +43,29 @@ RECORD_CONFIG = [ ] -def get_data_handler_config(market=CSI300_MARKET): +def get_data_handler_config( + start_time="2008-01-01", + end_time="2020-08-01", + fit_start_time="2008-01-01", + fit_end_time="2014-12-31", + instruments=CSI300_MARKET, +): return { - "start_time": "2008-01-01", - "end_time": "2020-08-01", - "fit_start_time": "2008-01-01", - "fit_end_time": "2014-12-31", - "instruments": market, + "start_time": start_time, + "end_time": end_time, + "fit_start_time": fit_start_time, + "fit_end_time": fit_end_time, + "instruments": instruments, } -def get_dataset_config(market=CSI300_MARKET, dataset_class=DATASET_ALPHA158_CLASS): +def get_dataset_config( + dataset_class=DATASET_ALPHA158_CLASS, + train=("2008-01-01", "2014-12-31"), + valid=("2015-01-01", "2016-12-31"), + test=("2017-01-01", "2020-08-01"), + handler_kwargs={"instruments": CSI300_MARKET}, +): return { "class": "DatasetH", "module_path": "qlib.data.dataset", @@ -61,48 +73,88 @@ def get_dataset_config(market=CSI300_MARKET, dataset_class=DATASET_ALPHA158_CLAS "handler": { "class": dataset_class, "module_path": "qlib.contrib.data.handler", - "kwargs": get_data_handler_config(market), + "kwargs": get_data_handler_config(**handler_kwargs), }, "segments": { - "train": ("2008-01-01", "2014-12-31"), - "valid": ("2015-01-01", "2016-12-31"), - "test": ("2017-01-01", "2020-08-01"), + "train": train, + "valid": valid, + "test": test, }, }, } -def get_gbdt_task(market=CSI300_MARKET): +def get_gbdt_task(dataset_kwargs={}, handler_kwargs={"instruments": CSI300_MARKET}): return { "model": GBDT_MODEL, - "dataset": get_dataset_config(market), + "dataset": get_dataset_config(**dataset_kwargs, handler_kwargs=handler_kwargs), } -def get_record_lgb_config(market=CSI300_MARKET): +def get_record_lgb_config(dataset_kwargs={}, handler_kwargs={"instruments": CSI300_MARKET}): return { "model": { "class": "LGBModel", "module_path": "qlib.contrib.model.gbdt", }, - "dataset": get_dataset_config(market), + "dataset": get_dataset_config(**dataset_kwargs, handler_kwargs=handler_kwargs), "record": RECORD_CONFIG, } -def get_record_xgboost_config(market=CSI300_MARKET): +def get_record_xgboost_config(dataset_kwargs={}, handler_kwargs={"instruments": CSI300_MARKET}): return { "model": { "class": "XGBModel", "module_path": "qlib.contrib.model.xgboost", }, - "dataset": get_dataset_config(market), + "dataset": get_dataset_config(**dataset_kwargs, handler_kwargs=handler_kwargs), "record": RECORD_CONFIG, } -CSI300_DATASET_CONFIG = get_dataset_config(market=CSI300_MARKET) -CSI300_GBDT_TASK = get_gbdt_task(market=CSI300_MARKET) +CSI300_DATASET_CONFIG = get_dataset_config(handler_kwargs={"instruments": CSI300_MARKET}) +CSI300_GBDT_TASK = get_gbdt_task(handler_kwargs={"instruments": CSI300_MARKET}) -CSI100_RECORD_XGBOOST_TASK_CONFIG = get_record_xgboost_config(market=CSI100_MARKET) -CSI100_RECORD_LGB_TASK_CONFIG = get_record_lgb_config(market=CSI100_MARKET) +CSI100_RECORD_XGBOOST_TASK_CONFIG = get_record_xgboost_config(handler_kwargs={"instruments": CSI100_MARKET}) +CSI100_RECORD_LGB_TASK_CONFIG = get_record_lgb_config(handler_kwargs={"instruments": CSI100_MARKET}) + +# use for rolling_online_managment.py +ROLLING_HANDLER_CONFIG = { + "start_time": "2013-01-01", + "end_time": "2020-09-25", + "fit_start_time": "2013-01-01", + "fit_end_time": "2014-12-31", + "instruments": CSI100_MARKET, +} +ROLLING_DATASET_CONFIG = { + "train": ("2013-01-01", "2014-12-31"), + "valid": ("2015-01-01", "2015-12-31"), + "test": ("2016-01-01", "2020-07-10"), +} +CSI100_RECORD_XGBOOST_TASK_CONFIG_ROLLING = get_record_xgboost_config( + dataset_kwargs=ROLLING_DATASET_CONFIG, handler_kwargs=ROLLING_HANDLER_CONFIG +) +CSI100_RECORD_LGB_TASK_CONFIG_ROLLING = get_record_lgb_config( + dataset_kwargs=ROLLING_DATASET_CONFIG, handler_kwargs=ROLLING_HANDLER_CONFIG +) + +# use for online_management_simulate.py +ONLINE_HANDLER_CONFIG = { + "start_time": "2018-01-01", + "end_time": "2018-10-31", + "fit_start_time": "2018-01-01", + "fit_end_time": "2018-03-31", + "instruments": CSI100_MARKET, +} +ONLINE_DATASET_CONFIG = { + "train": ("2018-01-01", "2018-03-31"), + "valid": ("2018-04-01", "2018-05-31"), + "test": ("2018-06-01", "2018-09-10"), +} +CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE = get_record_xgboost_config( + dataset_kwargs=ONLINE_DATASET_CONFIG, handler_kwargs=ONLINE_HANDLER_CONFIG +) +CSI100_RECORD_LGB_TASK_CONFIG_ONLINE = get_record_lgb_config( + dataset_kwargs=ONLINE_DATASET_CONFIG, handler_kwargs=ONLINE_HANDLER_CONFIG +) From b2fe2385d561e0169abed9a3d859e3e0af789ac2 Mon Sep 17 00:00:00 2001 From: zhupr Date: Tue, 1 Jun 2021 21:02:32 +0800 Subject: [PATCH 07/26] fix XGBoost predict error --- qlib/contrib/model/xgboost.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qlib/contrib/model/xgboost.py b/qlib/contrib/model/xgboost.py index 2a38f4fe1..300326143 100755 --- a/qlib/contrib/model/xgboost.py +++ b/qlib/contrib/model/xgboost.py @@ -62,7 +62,7 @@ class XGBModel(Model, FeatureInt): if self.model is None: raise ValueError("model is not fitted yet!") x_test = dataset.prepare(segment, col_set="feature", data_key=DataHandlerLP.DK_I) - return pd.Series(self.model.predict(xgb.DMatrix(x_test.values)), index=x_test.index) + return pd.Series(self.model.predict(xgb.DMatrix(x_test)), index=x_test.index) def get_feature_importance(self, *args, **kwargs) -> pd.Series: """get feature importance From 811d2c975e4c277651e3e87220ac4c36eb63d8d4 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 2 Jun 2021 08:56:15 +0000 Subject: [PATCH 08/26] update & fix --- .../online_srv/online_management_simulate.py | 61 +------------------ .../online_srv/rolling_online_management.py | 1 + qlib/workflow/online/manager.py | 7 ++- 3 files changed, 7 insertions(+), 62 deletions(-) diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 8650859ff..bd7c4675d 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -16,62 +16,6 @@ from qlib.workflow.task.gen import RollingGen from qlib.workflow.task.manage import TaskManager from qlib.tests.config import CSI100_RECORD_LGB_TASK_CONFIG_ONLINE, CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE -data_handler_config = { - "start_time": "2018-01-01", - "end_time": "2018-10-31", - "fit_start_time": "2018-01-01", - "fit_end_time": "2018-03-31", - "instruments": "csi100", -} - -dataset_config = { - "class": "DatasetH", - "module_path": "qlib.data.dataset", - "kwargs": { - "handler": { - "class": "Alpha158", - "module_path": "qlib.contrib.data.handler", - "kwargs": data_handler_config, - }, - "segments": { - "train": ("2018-01-01", "2018-03-31"), - "valid": ("2018-04-01", "2018-05-31"), - "test": ("2018-06-01", "2018-09-10"), - }, - }, -} - -record_config = [ - { - "class": "SignalRecord", - "module_path": "qlib.workflow.record_temp", - }, - { - "class": "SigAnaRecord", - "module_path": "qlib.workflow.record_temp", - }, -] - -# use lgb model -task_lgb_config = { - "model": { - "class": "LGBModel", - "module_path": "qlib.contrib.model.gbdt", - }, - "dataset": dataset_config, - "record": record_config, -} - -# use xgboost model -task_xgboost_config = { - "model": { - "class": "XGBModel", - "module_path": "qlib.contrib.model.xgboost", - }, - "dataset": dataset_config, - "record": record_config, -} - class OnlineSimulationExample: def __init__( @@ -103,10 +47,7 @@ class OnlineSimulationExample: tasks (dict or list[dict]): a set of the task config waiting for rolling and training """ if tasks is None: - #tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE, CSI100_RECORD_LGB_TASK_CONFIG_ONLINE] - tasks = [task_xgboost_config, task_lgb_config] - #pprint(CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE) - #pprint(task_xgboost_config) + tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE, CSI100_RECORD_LGB_TASK_CONFIG_ONLINE] self.exp_name = exp_name self.task_pool = task_pool self.start_time = start_time diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index 99a91e027..6abbbfb0e 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -19,6 +19,7 @@ from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.online.manager import OnlineManager from qlib.tests.config import CSI100_RECORD_XGBOOST_TASK_CONFIG_ROLLING, CSI100_RECORD_LGB_TASK_CONFIG_ROLLING +from qlib.workflow.task.manage import TaskManager class RollingOnlineExample: diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index ef6cb8dfa..dc1186038 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -163,17 +163,20 @@ class OnlineManager(Serializable): models = self.trainer.end_train(models, experiment_name=strategy.name_id) self.prepare_signals(**signal_kwargs) - def get_collector(self) -> MergeCollector: + def get_collector(self, **kwargs) -> MergeCollector: """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. This collector can be a basis as the signals preparation. + + Args: + **kwargs: the params for get_collector. Returns: MergeCollector: the collector to merge other collectors. """ collector_dict = {} for strategy in self.strategies: - collector_dict[strategy.name_id] = strategy.get_collector() + collector_dict[strategy.name_id] = strategy.get_collector(**kwargs) return MergeCollector(collector_dict, process_list=[]) def add_strategy(self, strategies: Union[OnlineStrategy, List[OnlineStrategy]]): From 8222795ac4ae51f504aa1ace432fe0ff69b3bbf1 Mon Sep 17 00:00:00 2001 From: Young Date: Wed, 2 Jun 2021 09:16:46 +0000 Subject: [PATCH 09/26] fix format with black --- qlib/workflow/online/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index dc1186038..d3cc0cbf8 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -167,7 +167,7 @@ class OnlineManager(Serializable): """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. This collector can be a basis as the signals preparation. - + Args: **kwargs: the params for get_collector. From 1320e53f8172a7abc54dc8a6753f417edc47430a Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 3 Jun 2021 03:23:48 +0000 Subject: [PATCH 10/26] fix DelayTrainerRM --- qlib/model/trainer.py | 3 ++- qlib/workflow/online/manager.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index ace3031ed..28d854477 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -350,7 +350,8 @@ class TrainerRM(Trainer): **kwargs, ) - tm.wait(query=query) + if not self.is_delay(): + tm.wait(query=query) recs = [] for _id in _id_list: diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index dc1186038..d3cc0cbf8 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -167,7 +167,7 @@ class OnlineManager(Serializable): """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. This collector can be a basis as the signals preparation. - + Args: **kwargs: the params for get_collector. From ed54f1213c717950b38ca9da7be95f64b20c9ddc Mon Sep 17 00:00:00 2001 From: Jactus Date: Mon, 7 Jun 2021 17:13:36 +0800 Subject: [PATCH 11/26] Fix exception hook bug --- qlib/workflow/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/qlib/workflow/utils.py b/qlib/workflow/utils.py index 596ff0927..5a93eacca 100644 --- a/qlib/workflow/utils.py +++ b/qlib/workflow/utils.py @@ -17,7 +17,6 @@ def experiment_exit_handler(): Thus, if any exception or user interuption occurs beforehead, we should handle them first. Once `R` is ended, another call of `R.end_exp` will not take effect. """ - signal.signal(signal.SIGINT, experiment_kill_signal_handler) # handle user keyboard interupt sys.excepthook = experiment_exception_hook # handle uncaught exception atexit.register(R.end_exp, recorder_status=Recorder.STATUS_FI) # will not take effect if experiment ends @@ -39,10 +38,3 @@ def experiment_exception_hook(type, value, tb): print(f"{type.__name__}: {value}") R.end_exp(recorder_status=Recorder.STATUS_FA) - - -def experiment_kill_signal_handler(signum, frame): - """ - End an experiment when user kill the program through keyboard (CTRL+C, etc.). - """ - R.end_exp(recorder_status=Recorder.STATUS_FA) From 7d9544fb91fc350f29b2dba4bec5d6424f75e8ef Mon Sep 17 00:00:00 2001 From: al Date: Tue, 8 Jun 2021 09:35:36 +0800 Subject: [PATCH 12/26] Remove non-existing parameter from doc Remove non-existing TradeExchange parameter from generate_target_weight_position doc --- qlib/contrib/strategy/strategy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qlib/contrib/strategy/strategy.py b/qlib/contrib/strategy/strategy.py index 4f8eb0ab1..74290b7d6 100644 --- a/qlib/contrib/strategy/strategy.py +++ b/qlib/contrib/strategy/strategy.py @@ -148,7 +148,6 @@ class WeightStrategyBase(BaseStrategy, AdjustTimer): pred score for this trade date, index is stock_id, contain 'score' column. current : Position() current position. - trade_exchange : Exchange() trade_date : pd.Timestamp trade date. """ From 492a62a569e2a64dfb9e79e61e61cffcce8ae61f Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 17:32:24 +0800 Subject: [PATCH 13/26] tcts demo page --- examples/benchmarks/TCTS/TCTS.md | 50 ++++++++++++++++++ examples/benchmarks/TCTS/task_description.png | Bin 0 -> 25240 bytes examples/benchmarks/TCTS/workflow.png | Bin 0 -> 29877 bytes 3 files changed, 50 insertions(+) create mode 100644 examples/benchmarks/TCTS/TCTS.md create mode 100644 examples/benchmarks/TCTS/task_description.png create mode 100644 examples/benchmarks/TCTS/workflow.png diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md new file mode 100644 index 000000000..e8cbc8005 --- /dev/null +++ b/examples/benchmarks/TCTS/TCTS.md @@ -0,0 +1,50 @@ +# Temporally Correlated Task Scheduling for Sequence Learning +We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://www.overleaf.com/project/5eb8efb42dcf710001d781d6). + +### Background +Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. + +![Temporally Correlated Tasks.](task_description.png) + + +### Method +Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. +![The optimization workflow of one episode.](workflow.png) + + +At step $$s$$, with training data $$(x_s,y_s)$$, the scheduler $$\varphi(\cdots;\omega)$$ chooses a suitable task $$T_{i_s}$$ (green solid lines) to update the model $$f(\cdots;\theta)$$ (blue solid lines). After $$S$$ steps, we evaluate the model $$f$$ on the validation set $$\Ddev$$ and update the scheduler $$\varphi$$ (green dashed lines). + +### DataSet +* We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. +* We split the data into training (01/01/2008-12/31/2013), validation (01/01/2014-12/31/2015), and test sets (01/01/2016-08/01/2020) based on the transaction time. + +### Experiments +#### Task Description +* The main tasks $$T_k$$ ($$task_k$$ in Figure1) refers to forecasting return of stock $$i$$ as following, + +```math +$$ +r_{i}^k = \frac{\price_i^{t+k}}{\price_i^{t+k-1}} - 1 +$$ +``` +* Temporally correlated task sets $$\domT_k = {T_1, T_2, ... , T_k}$$, in this paper, $$\domT_3$$, $$\domT_5$$ and $$\domT_10$$ are used. +#### Baselines +* GRU/MLP/LightGBM (LGB)/Graph Attention Networks (GAT) +* Multi-task learning (MTL): In multi-task learning, multiple tasks are jointly trained and mutually boosted. Each task is treated equally, while in our setting, we focus on the main task. +* Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task T_1, then T_2, and gradually move to the last one. +#### Result +| Methods | $$T_1$$ | $$T_2$$ | $$T_3$$ | +| :----: | :----: | :----: | :----: | +| GRU | 0.049 / 1.903 | 0.018 / 1.972 | 0.014 / 1.989 | +| MLP | 0.023 / 1.961 | 0.022 / 1.962 | 0.015 / 1.978 | +| LGB | 0.038 / 1.883 | 0.023 / 1.952 | 0.007 / 1.987 | +| GAT | 0.052 / 1.898 | 0.024 / 1.954 | 0.015 / 1.973 | +| MTL($$\domT_3$$) | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | +| CL($$\domT_3$$) | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | +| Ours($$\domT_3$$) | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | +| MTL($$\domT_5$$) | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | +| CL($$\domT_5$$) | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | +| Ours($$\domT_5$$) | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | +| MTL($$\domT_{10}$$) | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | +| CL($$\domT_{10}$$) | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | +| Ours($$\domT_{10}$$) | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file diff --git a/examples/benchmarks/TCTS/task_description.png b/examples/benchmarks/TCTS/task_description.png new file mode 100644 index 0000000000000000000000000000000000000000..7a9005bf24869c6c88bec34513a67704743492ee GIT binary patch literal 25240 zcmdpd*I!f57cHI8n+ix5Lk(RafOLc}M7q*@N4oS9x*$po5J0+!fItYnccch}CN)&) zz4vzG??1Q?_vJq1BzfXKEME9+&t#>GaC++Lz0;c#_fP<^MNrEB-yN3! zp1hg5kEHB0aL<5ka924m`C9BfFS=3l=?G2jR0Q8LJ@)~GCf2NzGE#G@zqIh#>pi;-;mjy#ILrTJKT=}l1 zsGTp9VHISHWzQqrl&s-UFTYS&Kyneo--@kt` zHSNVmN)(Yh@p28~?&ZqeaDH%CT(f_4?%TaDtS@(ajZ%G0jbu>gSoxijSkKldM*=ti0?^50IG` zCTSe+d8ntlxl|kGowS8c)rSL557QwSXUxPQj(yyaaV6a*Cc32SHb7ZP_POC9Z#7F5 z!Si|UG2H_sTC!2JgX%pm#6spRY-uTy!L%S^U#g;;dWxF)<_bCj#{~1b z>vX&uZZB!je%P@6oO;-UtL^3Q247OJ{;FQ)X$Cs)+P z&kS^{D7^oqLa58yWP_4K}8N7B?*1Dj{B7WI3AOq`+5R09L0J zm&u@zbw0>u$$21&tFguY+i%b}f!0^>+>CwH_d1MS(K5Dw#! zyZcgZI^WludWm3N<~ivA&Mv3wmcywA0}>8P%1q}!t?4e_!6!;6JEN1HU6iOTPC9#| z^-+zkDgtPIe`F@cJcIBaE2clJSH`-_KS$fH)(zBSk35HA8>TH1{_t^LxW29$_Hk@q z$hoVjzpP{aopj50kwAE>bVawqf6AtZvx{n>zuqm`Anu@@HEko_7d%@9GID=XQ+>6A ztTH5%F9%n-Bix9ew>QpuwtxT6h$+!*<>5Cq1?!%T0P4;9wty<#Y7Z=rb-)K<$)qeZ zCVyD0`(MJAotbp?cgIS+cSNNw=F_Mgonv<((`r0jZ)Mnl&a;HON?17$5ymq+FN+yb>bYn2)CNCuc&J{y^x>m= z)7KeOqfkj1w5$rHQKshUuv?7B!@Oz~ zb3J5CkE4KDEe*f>;>Nh0i$0k!T&HtTuw=Q@d=V<$o6BSqNn?p5XN%C0|M9wKEkpHB zce7Q$lcyNI{yxXs44a1KpUc;aO^WTS7fTBTiX4tiL`av<%DVMc#}J5gQ+GbD(%shz zPv_zt6=3;xEYdJkbF-Yx24VeU62EBnfFIaXB`Jb$^?9-BP%475(<@Tz3EgFE)hISG z#~jnflWRmF(GJEH!C|;!k9`N*c>~xPGkmxb8}374$)7Q^U~T@<`Et}Qw1Wwv1f_E7 z|Le;PH!P)8$MsUU5ICxm*zc*vBWT>{s9P2UIR-qLgrqq65B>*OmZiotJXK)k1I7$Q zeU6>zD9YFhJ3unMG|Yf^cB)r9SNr_#{7XpCx5FO(PMjm}`iT_OJddDVk$C)<#H=S*n(qo%|z7@%&fq)MvV+ zRkCtge1vrHqkz~j?T$Z}9uh&p!%UtOe14zAI-dpk9Da$rH(8|uW-z~f8dv!?$M9Kr z0ai9Ze?mhVX+20BNvWQIr*FYB7|AEz*A1Y&Hc+yVnsuB9SG`DDOFy%(Q#+G$Xq%_9 zPn>*B@2~>?zuqzR@YUq;!B zF6i0D6Xl0U+}wc5BDM&TXV|*`_VZ4nBrM*4Pr4kT67p}4re&+mh&#zT!jxv`ei=XZ-9cy7 z)&|Vps&S|^{pCWyF9Q}(YWfj+bp$)$x!Rs6L3Xp3-_7^vs50vxmjNF(V=2AauqS7Z z8G&uu$HXOi7;{j(@mvi)Dq*Ea8X_d>$=<^pP}l%-9l@f}a_++r&vR7|Gvm7w_y<2Q z=G)x+e7W`PcTBOy!FVBC1d<&h_xLIu)&yVWK>qIzsdU_u9A3Y0STgE=O&CArs;W&0 zz~_b-{s)PceQ&k<=i~3m{&&Wf8BjUN$S;}wd(H>@?sdNm;~hh1!_qHE_Ri)a0|wL_ z#QpD1l-OroDLT!TUjhE?j3hEl$5p^K;|~~6WaWMXTNKF4VOfsF3~!-lonYFw`b)PQ zA=9SY8Jabf^SseM(4_w%IdEiqVy%5xe!gK9RC&Fqe>@BVs1CZafly=9q2Ez0s zB|jPW2a3X2P?Xtus@G9~sfIwu1$opFrT+b3M!*irCEueEO01Va^?hv|ANf+9(IRk~WH+FG=F1ayn7j9?!&`u5|%Pc>Wnuh{mZViH(Tm9?Twi&M1uS;etfkuPE`lW-NmHyPn>Yr=BxFZBgKV-ouc&|EPgz&*4xnml(;CGe*dkE*n@7Ak) zs^A+9qZWyux0sB-PE{#VDA_m9$TG<~|GJe?tqZxZ5l`3;=5IZz;c=zLx_A>mK(k z)XRc>3?6mFzX9~3g`d?QH?+%%DnyC_~Vi7Y--i`*K9RDT5Re6sXAz*y*d01h?~ zCOoYyc`*F2Gf~1`UD4kq`Ol~10VIJRh&E2)C3<4YCh);tjpS;PFq=*p{))FEc1_ax z5x7ZS4tScCr0~F}K77BQ>7y!LLji<#273jn{Lh4W5hjt=L{nS-__^gPvDgN*2gBGq zg{i*5rHM4MfV6dQZGa2fw3B-9PCt5W>Wgpc`krca;hP<59CkVBEc)`{%AC^Tn8m>P z10(s3MccD&CA|0WM+{IK?U8{mIL#LVR}-+RB0W+}I1EJ&K5r5tkY*3Vo=FlXIqU?6 zv5VZMi}7gL0VQp=T<57jm&QVko~$52!^VGPb1QbYXD5F0Eq}$AHEZ$rV&9Gc(NI6c z1H$m>!-i3(d~BOFp1{7bb>_df^@0e6Int|w6_h)FyDrF@PQ-5-&Tby1zd zA6eEt)^1Z<8%z+!8#KiFzO}V`h^{gjGxeQ@VCDhsZ4LO+_dtZ8iTlB)0_o7src%@S z%$t;=jH5dZzC^7wE%0DyKXKrG&BJ41*OeL7?1RlkkB>1?vm%fPX3YjX!aa$N0tra4 zuuwcG9T#Vy;(ZMs_4bVEcWWYunXpJacSY89emTfMUNetuv3YeFh+{;-U1RKLI+DHk z))EJakm3UlAW>d6pT}!{^>{hm<~EN}3n zCQ7vOMNzF<*%Y;_;Ws6f#JV*L1OCv#i`3E*uf~PGRqo$DBEPhkIAy;Z!HbnQd9o&- zxV+MjBAuv90c09qFn)&yMML#}B)>~BseT345FM%UfzwE3E12nj^E~;~%JtDs@N#(D!Lp)>F1KK9?rfC>zNeCX&w?dZ5I| znm((UD65cP%Z|ehEnm^e_0sWmbhx0PunXbVt|us@6BR(>0|@-XHcf0gH>vr4!x7?{ok% z1mEUet_u#bb-onDZ&>p3DB;3j@`0!DXt2pk&Ak`a>39a23520R=KbZV00PiQ_x1PcuNb~0C2?0caZzp$ zekzT_g48;aHiJHjpZgj$H_{&%M7o2pr?knw|hX|1pI66mF-8R zC3A80#jMomEc1nz2L3t}Yom|$s~L9=HW5JSc=`&^hJl4}0en6C?YBu)vOK%@a!R{Y zK&gh@q-zMJ7(poqS*6v=5MWjxX;VDHR8d(Dr?C4oNaU@?{D%uA=Y_#N5OTsEIrmPu z4(KuJ3`jfzjg53cFqxOS9oemZ_G{P9%B+$LH}P@=XlYHypf?Zt>aLrp{&s3V1_BSz>? z5_Ra|*s&AFLSC#w3(;YEfuZV_4H5b{lvBq8{L(fGBum41IT*BY@bY}??oCOm{FV37)>bGcFybi|+IFl65rtI3pY|OM#<~zZ1rbrlA;rG=ehV81_8Ax{UCs zzfnJYJ`N@vj%`}Cw&}UfJCWb%l8}<$LHj~y?Nns;y3Wt*RAl&QuLJK(hwV&Qw*!RR zt`*2qi6RH}Fwf_>%Bzj-2{N%;1E^SZD9RC91tUh2UQT}I{$MMO5vi)|cw|f1qDRYgh z!pEJ!9PQ41I^{aPnaHEdC}5k(hXB`15$Wdcn!pK6Yjfet79flEF+K3X_}CFx$4#`X z`i$h*6=+t0!Tc8O50x0kah-JNUGJMipz1whwH(rewF87?8y;_!=ta5r!t5{%pMJ9! zQ5QcArFy&`w|K0Oe0fm8>V1ixa-Zd$LO(;PsY!5l;i^sCz0_ko9AW7;1IrIcNQhS* z?18`H?jT?#(|v;)>U}U#QnjoF8nUWzwqk$_^-L;n=)t(?M? z=D*wTz>lSiB1U~rFbT|;x0O~Idl5F-n!<$oy|2yB)PMp=f3HA(h{=-QYwrt5h`Lq7 znmPvfAN!Z+ZG>Yo$<+K{jTBeDnV0R<$x!?Ls1&#QaxzVeBhRIrYn2bJ`U?K>^F5rw7sPSRcnj*>2mZ z>3hdz*R@19nh!03PGcfvrdcc?pZ?>AH~m)4R4$$Ks{Ck0h~`}f@WJpnnK~#oQ;EX> z4Grx7Qfitd-9!IScB`lovh#Obz=&mkNCjViENm=M;O((8uubzDvubPyOY)1S_Zbjo z6HEr!rgz*5)Bs(L!WxRS%Q`iZi&$Vm)fye>x@5q}QzmXe)t|G%1+M!7MO6a{xX$Pm z=SOc3#Bhs$4byMiVjP06Yj0jxK=xeG+4ulzXGsab)O&`5l85Q)7Z*;z>j?V;FL0gL z)H7di(2d(%q=oMQHFV__)TK#{e6pd*Wn(RPN*v{DTsS4oGOE5%7u@>{<)~z2PlMhQ zX+MiTk*{mGH`z#QIq*RD&kZS!x$m9mV%*5N1iF4Q%4z!*+6t-n-mqmr^VgKu9;V;L zqa~wyD!w1d)F|F~OYT`N?Ip!c2dShs4qZQObY4Gdj_??bY`K%}NT}UBuVU%-k(%v* zT`E9a^uDn;jYH?AYva}0;5}C3==WmI6>4?613}94eZT?8{rxySW14cjrD zH=fB$mO4s87reiF`7X3Bn!WEe@jHOI3M4w^;>uun{j^L^wRF+jPmbS)_7<<=8#k^` zTP|vxcLv!oS!aBTJxi#rS?YDe$%~oB!?`>~PyR~-dCnl8=$gB!v!0ipgTe4VKoRW0qS^tBhsuaO*Bjy6Yny(H_U(46P*_3>QW3@iKD1bc*CQA5ICN#|XY z>vgdg^-@n?>UfF)E^iAVq&>Z<)3qYuF>r$f&r+PX9i^Sa1*9Pa-c)>Ol>H;6}}f=j?wXVSTK*DTl80=saICAkQ+x5uMffr zIM(rFtkMA!Fae%X{;VC};) zwf_$dJ`MfGt6G@&p~YO=kBF1=h8_1gZ(`Efn-p+sP^Ga8?C1d4vMOh{kk@_DTDfWm z*+ejq_nZ_km}ws$!d$toPN*g3eF7!K#FB5VqYWx{Msqkor|a&(7_QB1=^wZerkmx? zap9&Ct?qkYAKQ6QTB%K#Kz4UhRT=e+{O^Q-JDChI>?j!}@1Apqs*R90tl?HnShX}aS1-N}098?g#{;rBvui)GzjV}u(k#x);3YRcRl z7Z)B%scGgoQ<254hJQ1eHUHia4=PW;Jy&jW(1`ES3Rn+dN2RPdXH^JDRF&oCkRcb9 z;b5P}<(dlDxd#3fovbp+Uo+BHULX>oQJ)~E5(b~sl8W&aSxGt)r__oYT_Akdn)>SL zfrEhDSD;j{8^4QXy4F|t%#t!z>jILh$@LkJ{F6t% z{f@1EZ^wu4F;9CNxZ5iqv4cnWW!`oA3Azvna`uAhu+BS#`l?kbgr8-y;r80AUT6n- zG0c|RImNIQ-dq6pJdY4S9PYZ`UH3!w)oXO-4!y$y?m<=|1;Nq{5uxKUBXKd6tc|H3 zhVJPk3BG!mMcr|$uk|qwyfmtONAg_O>Xxp7MPQV}D#R;?8eM}*dweI`4^E>uiYvEU zWlRr&sXP5_TXvF0bc+ii8S`o3#%5Fg>%X^>q*}e^+Z7#D zs1dW>xZC~7+~DDTylCqwMu>zrEOav9?%#XPH~mR-OY4bu@~IPmPQY;x&VWsPoY>Y1 zft4a53wPOKPI%qf%1ACh&23%IKk@LxR< zYX88vGTDm$L5#xJ=utJRJJ0jLX_=(oQ>$aj=X{poX57nyPK~|zdY{o8Xb~;>Zsxn` z&=Ti#(vgsqtFa}HJ@wl3>?k|ev{fUAFU{_@ne;Xb#5mr4AZLUMvbfvQzqILdF2|Sa z=$|uDGX85?SX#}uC%2z&bOTZ1H5Q;7y%}wOL%BPCq<&%x2pRd{P7NX-V>`2$&d<*R^j=_F{N75=opdS;9EjIiHefV|H zdosXG2(Mj(9nN(^E4;2r@D|73nW1?lm) zXp8=IvMr4T^~uiEWDgPc-p!xkZ2y3GPrF6R#F#gZ(9o3Fx68pApw3q5p?gz7HruX0 zLm&m8cZez^Q#-E3*YANzKN=j~G(!)3n$B@%ifWAmgk%=>tBVlmBNV(n7BiqRT2S^R z(eox7#f?fLG`O|&4;>yLuBCkO20CMUKqPz(UZG|)+QH1bC5dqrX#5iqp;W-xL`l9% zL<~BUKEbcIe-K zE@}m}p1f`A-=4T1Ve?Lpm2rmSfP!NQ?;{VMCK_0GO(*)59)!h+OEPe%^I+xWl1i_M zg{LhmlF9abj-lH{DM&zj-k7`{64DIh9uflETZ;@HTZ?j0uc#%G<*K=nqZ}8#DJcz6 zZY^kJ8Mfn-SEiD^oM4Mj>))&#oYm>zZ^bo4gnASy`+e*5)u|Zinh_>4_W1|A^g@ys zfLN1+bYvNpMZ~hlE4g9!CE2ss+GXpSVTnP+pG$WFVMau()y8wSqNTT6{nPxyX0soq z<03!w_)%VqoEJL!IMIt>Gi^LCs)~5-ySJi$ZDY*n=pAM|bd3*BW>SXB63BX&xZmOM zL=Jn&KY=ME;%`}9<*>crZ}k37l@{+6nI%Lgl5xYzy1Gdlu=j9*cM_A=KgSnZYI)$J;mN>9_}cMed~vSo=(WYnZpT ze)H7&W!@g+0L*!ZPbesc%Idy;gz1_bJKRys;+-yAeTI;dGrRrvAX$uZQk4tt>4h3d zsp-4xbfOR)`tgjdXt)0RrdGTr^ADEk2P5uH=7Q3Xaq^A;_QnHZqqhNVAhptCU|M057P%o7S@@bjySu zkSGkf6IbHB!l6-W&idpOx>L*es`d*g&AUOlUv#57j{Gm4#OWKs!O+-VwlI#~4?$>q zT+uoUyZ1N7_?2*(KhRTm>#f1uJ%#n97gYN8MiR(JSVjvpf7}pJ=508#=~Nwfg_dxc zojD0>c~Hq$-&@OQ*)QSqlT%r-#$y(8f-S=NcI==+_n#-yEt7g4OGE?qo?fWa-7rTx zimkK)Q+oP=Z4OEZ6iLGPbE)@i(4YXC5WCa=_fvF6?Ph!Q!f!!?R)*eB{X{Jj(uHyY z7yM@>OIjf|!fsD$Tf>cL>{=a&ax&`saExldk4M=Kz(*c*znm_P(XBL9K+gTT&(E)l zCQ8mmq%^RQPEZ?gF%~2wAKQimE>Ui$?%@s5Y^O_C8A>ms+PQu2Z7ogrcF|!ls0Gd@ zJbhQw((&rYZ-y$DGlR+QAyF|KG4+gWGS$U{Fu8BiijX3I!%mW=G3-TWG3rx;Bhr1OmwyRixKTE^A+A?isZlC zYqH$uC0;(Gqq09WX}Ym=A}+En16vJvpewo9dm47d<`X`e(>|ISc<=lz{Fx8eFhI$; z>fZZqUe8KxSQL+@r)pMuqgtSqSM!m+qtR5CH1l+1Ug82{(=4TS*HYBc= zym20@q~M_T9BdF+_VqMC2c(^`K@w=;NyNxqxjKVzq@@F{!Mm zvxaW5jmVHseW2@gqBeiRbtBGxIJ?IGb+VyA^!o)|G%rYgU7lawgej=z{0SKtWm`4s&;gPTh#xt7=2PU(${ z^_ZM1e!0tTnls^DwksISZ5>dNEWWy$^<`-pOoLF^0v&9+X4);vT4$C7taWY1dQ_7- zPguU;U?jX*vrE>e8yC4IaDh(~uJPfN@|O!TzloI8DN@ouYT8{kz^k+S{R)fx*V2UV z=QNvwYh6c&0KvE2!XaF`*8Am6TXcfIe_8xi6-=|0W&9*JTbH{1ALRMieEzp{rpT3m z&t_f{`QOz$g{_A10vEL{39X`7bG6v5h?X_!?5DVn4^DSEK#$%0i8ZCX^T=8~s65^8 z5rvL3cu&gRP7l;ZveDNjwsK1w>a2G&Xpf)!(-o#|aY@;EzJKHw8D)E|S?F122AhYCkT@B=Aia;Ma&Gmb zTz-oOoL||LBjP~pb1`+E3%Liw&h^b6qQlnulaVU`vL{NE;b7N$YN?2JoCWSSbFj}I zH%~WXBkTv*cX{D`$z&Czh}EaQJ*D7$ufhs=?2Cj1YF`i<$;T_PtOw`pG$n?Meu$&pMJYsV$cfM@qjg4Fbb4C9*>5*ZpQ2Zov zj4Ro**MbnRZ)!F8N4>_w_7UBSP|B{q9!v)=3Es6!OL#gl+xlE6P4{k;a_E{b;VdAk zXgbh=$eM;!1pfmcdh!%MB=eK^mZTXgT@Btyomja05)wpIoyfG+|Ij{tb>XXlc~Gp?|;Rwp<2i|NVfcs{%ub2JFE6;P6c$mH|n=2+;`4l+=qs* z!(e(+oUa4xaxT}&8UZGhm;}>Kji+smV=or)WDX!>17o%|i{qK9f3F znZ?wvP(lkRs=q|}ikgU2?Yn@zKA^@$P2LIP%y89~`2JqiL$19az$ zlEMPfp0~0xdXtXp&Sy1qGsuWhAGWD&+tA3ZqP> zR0T2%NiU|$PY1-)72`I8d$U>OEWfQ7HuZS1yiu}l-8cEbt zI|GoMIM-wtlbI2`ejnDs5*u$VXnHofY2;JD;-PR!A|Vs^g(^p2ez2B&#&{1%(8tco z&l*LbpUKb8Tf|7XN6lo;o#staLkJ-j#81u)U^kM#qSvwX%L~4@>lRKnIuqBZ>Nsl9 zC*yp!GlE^ln#}&VUVKB{faDpbt}lB>D*|krYfXyb-7`g=G6h(*;XI{}_c{|m4VS72 zl|l`EQi9?;C^k4*Pa8_LSRj)2v=FS!kGm7Ns_Wi$PCpJausLFG?U9qC*l=&buE&Af zEJ}W)h^LK~@SabSdXQ&U9NmiY<`d5Ezagy|Ppt?AW_978MbZ-Sk*}O2gL`|m>;nkY zSZ<7ara=ZHPNLgVEBy@RRHfZ;QVAGL`S({Cw& zOO(kwymI)3>aKhT7 z{;Ysn7VMp(rZ9(&ypZ9I*}eu6EWDTn_EU$T-S(K;$<|$%(84vpZw_av2$Ok(`oQN| z#q+@((|aNEnnX~mhpjryU<x$&&+fY7afihp$(1shkCrKzo5es^tg5YgB6^pRQl}xUM1RSN z5KrlOz26thD4mlbH~BI+(-vQnFruh1-f=U64R7bjl#yts?*8bewS~1qxxs>ZB3C+i zs-Hi|PqcZuV`H(1wbNO}cSHS;vmfuOM3Ao4J8?T5d_lgWa09s>PIuL(H=w62wT%i=uWg3-@V zasImz`7>E7J_RsK{f=fEpf)v|f~M0?8LCA)#ao23KKh%7QfF(FFbI7dVC{D}(xg00 zvj6?vMX^#+_VuE$_u^e}alh;&yMp=Je|}=U?WmQE zcvRmA(Kay!&|>>cHjMtV;(Sz5a`Cpkj4n$(>I<&SC`Fs-mDzPc$d|~8yn>9_Os-xu5*6K8*U|RzfYhH4n)g9#;)& zTdmyI0vHK(CA_}A{wFM8UvfW>|{y)MI#ng9nR#<#JP|9!L1AjnTH=zhsd6 z{nAI6k&~UNd|1sQGzMs*YiB|6HBH7J)`8Gc%sA%;C2V1$$GkePREL!Q> zRTGQ^^{%zXdI}4o$#EFiOPyZsrh}*TfYF*t^irdkSq^Iby&C&Y@67wph>_BDs|OI% z4`)23f;wPs@GONug~ouVfR9>sopq$U-pR^|=|*rXtl*{hO~5@Ym9L`Z^qM4fP}8*P z_y9BU%aL|GTL+ZIw{Dem%`Dm<{+0PJ2*IT*tW#N+C>N?%LBDY)EyjQ%0Wk0V=a+TV za{DP8##0U#;)|R&y{@JdZ|IW00+679C>l&X4zt!XiH8m9 zt$8zns~Xr8j{{eFb=7b|z+DaEUJbbY!gH*TPgS}=as4$k1-h+R)fz8Ae^pCO50ZOK zcO8|Q_Kw<%niC)YNp-Q<48EMT^7(cb-g5yACHscm_fg=}PsDc4l1~Z(7Vnz{_?jNI zixRsypL_Iq&5>ZnNRS zSSRJi9M|xoqPpy23;4%=d#b0#&(%cNFvr16q7JbzA!MzU84eYQoua>~xwA{PAJAw# z|GMW8^H57KrE8Z(e`cxERw=W~Fgy8Fa^?K)-W}e$dvcG_vO1OTKlBfq(K!3UHiKJH zs$;InMMoa@`Q@W+*9$kEPPKJ6xp!?1iotgZDrU67r7WF97P7>M$pWC)&jJN>pgPg7 z+AOTyHdW6NC(`dD+@EtxnL@Ae59U42wgEX(KNLqqC46z;xvvtYg78J6zODaf&Sq{P z=G(y<42i~4q;ycbRkD16U4YzIXNUr%P~J_pq&~p z@B~Dr3w71~NoITz0LPRl<{+Zw_Wj$cx;LzHhs>wXyHw}W&p!fZ*Y<(2JJ22K4X(o= z>*!~ecWuSNbgkw5S6^T(+==*)1MNNhuRVaIy8xy;1U0w2^AD-u;_u_N{`GKnB3egh%~c7dK^DxD?TFeG~jZeSbrfjD#i)oY+_BBJBG z2M89wI}rn-lo$P5h5*<18gP5gMo#wKQHy3m*#TbbZs^5U>`9)_xy*J9QHMr~y~WH&V6Wmn+l9Nl=C) zRAFze6?)Py(7k^V9;h3AWNDN%9*~T5rf50O-aQ3m>`AKP%(b@%(cA4GGFXJ45oTG8 zSMz1jbD|-el6TgD;yxu*ri+D&O%i%HS_c?o%Qnok#^%Z*S{t?2JAk!QKC_Q!M~_Rf zmRKOuu|8bOk}+E6QNh&JM4toO&{|E*I7Y?$sv*BbiRg?(W8)P@dUjv7j7wi{wF1im zoFeZIs~kRrJixyei6;@%*Z&V+W5w7PYK5-9@ogw!|70+=rxH zy`w`jj|m(Dhj$lS#Z^@0(@FgaZeHw5hStB|56{Ng=KG6HjKlY z#0jkgN^PuqCQXWR)lq>4F2vgZWV8$_xBswv{E{+)nIqG*FG%TNM82f zYY&60I!S0#5bdkj+W3}$x6$si62)zA4yOfc&na8KN0pfh(ZhOSDesYgZ9&Y(bjSi~i! zncmFR5lfJm^kaw2FllRZcMzQnSfH0CBF-%)W|_{sW8UPS_L222lgWk!B1pDT3kjJJ zTyGurrN6W$ttC4YZx?2;x$Rw3k$7GdR?560PAgoeUSCpe=L$)i`TdP3!M_3Py0Zb` z54Hwe4n->Gzow;w_Wpg_9_`Bq-j`LvS_Iy4WhU>i-btPMh1%+Vg-K_ank# z=*ab;!^NH2ZN%FdlE!|Ll~ho1P+`Id&b;OTfv6#pIQ4 z>Oz7%#Vx$W#Z-X)(*CyBR;}1JrWW`9m2zL)R*6tM11`w2xlO5G+8)vo-tI(NWSu{S zuOAy7u`-|^FL5FbToB81%yduSO&!g1`$Tb^AWq;EP~r6TuQ$r_{UDAnzAeYbWFP) zo^05vp~CaCsXPt736rckf$z1p7(}Sad(3dVh2ev^?5`AvOxq$(1VvsnCGwjPlTKNf zYriFTVyOM-k#`!)`r9dt?JFS@?qC0Q`V0pHbqiKL+TWAaeYg5w6Ms*N3M*vt;>vUa zo-Q?++iuQ+9%2FJX9c+%;8VW6@8up$I5u(#?VVLA)&!tvue_q#l~VDv3X|qZeqn{U zsyJy15vMUhI?wpl5nl37R+ce?m)GD*CDgBq%?E(V)c#@mihkrOh`z1oCJdskfJHd4 z5~dp06u%biuUNO?o!^@uFrFjzgW&Zc?34*?I-f1-?b@MaNh$NRThU%c}Fd zZ*hXdfr6eS-J+xRJ}6&z1@K%^&Wh*tZBzb@b5qMKH-F9<+eTw#`25ck9W8@bVG$Dtn3Du`S*w4 z18fJi@iq47=sY$Yyd`SCV7+eA$C-O^C^kwyCPZh8jgV%DS73}98FsG5&Oncd3vb`G z77+C;XkYk(;!qWTT(dY34I=JU%QySv{?pfa2MO=Yl65=*klo8@fmLir$3;Wy?XZjE#HNzABtz- zfY22yE_9L(m!jzp1=10v?2WfpdYc?9AOj@c1OKJ=#YH1<`;CnQen4tAhNvHey{?3_ zz+=eQI_HSK8Eyqs%@oiB!P+9PnFM63PG9ohiSx6&64>TS5?1-W9%pPP?ak%Vh^4XO z$>pOKAv0U2dP2;NvR?3F?}E@w?>T(n=I&(z4d2uuWZifTv&yz?7S=+SFLsa zQoY3s`g66t4}f_ATi@6f`;`K#J)C((0ZAA6rFruE1S=-OxFnr#%8->$`G;4vF{2VO zM)cw}mK^v&4(1LYFl-i<7atClm`9(i$D5VoW9qDeM6>3qyxK|PEc8JAM4jT) zzy92On^dkfBVhAZK^MFw`G+i~}7~1#^QMbJ^7)r(2ANt?Bt_@py0?tB-Cy$;oY7f=2 z5H<|P;au}K+dMDWbRPc|x3a6jgC6(n4c)&j$kcE=EYIHw1`)Y*vGaiR0HSwHqj1|( zM3UC|l<;Prrwq!creo=aCW3bC`udkwI=+WtX3kE59%^DEN2_x6MZ5=Lu_1nMrBl~` zC0YqW71mPhbZd|5MzN#)tF;6y2+&>vq)+ZNd4*j!?i3v5yg!-}6HL5_s|hEQ5}fpR zus1s`*{wGBGoGD$Ov82OqVjHtHtT!I%6MBU<*TcOu^|DI;rupnd3r_k*N?;M{k7M& z%q+FEm!q`57IuWUAk&4iqtbss=^;Pwj=rvBote|dP(e2X@eHHBsV)78y?O<@^6ZIQ zkjQ?sFei#rKJJhCBTHoU4C&%`rpxpKymkln`Vhl0TPKD4{|;64{KueU0MHe^!JCeS znASYIuy*K@9iBeiFhdA|{AO*?iVNvA$E6ruBk7%{?3K$84PSmvr|$4zy7TyP2ALsB zxJy&NY+t2OzZcTG<5?VpQs_R=HK@AX2dHcR7?ifj`-;edMZfni?s3v?T$~R$Z-_Y2 zqI9N*QO)A3{`9tH;S{TTY4!fc_N1GSq>I9N59y!dtT6A;g7)9-o?^_nMJ(1UbX4hb zd#`G9JeELp_>7fAwoKQfOBYOX0$s(j^WyPf0t-zagZ(+e;?+^s&4F@0hxuc@vPUgd zr;bVmCYe=|;H`0frJC}*Bbi>FkX()PJePPO_nW#wlB^FPlwnh8#;@2-7Jm*dK?1x1 z&ko_H()Uw!+(lxdAbPHZ0*|;gf4rGqq3*ITnDVC#8nD$=xA>4;cF-gWntH%?I`4f- zj4IA~uzfI%KNA-DTyWELXpfx)IujQuJ3zd<`RNLa&P+YRBSzGHIolF1bSnM>E0zB6 zrqTA6H(LMqC;ewSyUu1kKlgk(&a+<~xUwT*a!fn8VnTK49=aEHL{J3PT-1Fl`by1Z zv#fT+_U@8g--6ONgX?lDnVG!@xpuFBd+e;Qz9v%&2ec|TZx}bl{4KSIzRa`hZIm7x zs}FgrTkANWjDN%ghs0opGP7dYf1s*d6 z+sft}*(NkR4c>`4+TBpoq6_V$CXmT@dLqR$CyJ*_%9~%;PZT07E~BKUJIrb&%iUC( zEL)t&B-C#02p_9|Pv5xEAwBf8y1414=fn={Cap3erbYR=+ivU5>?lU4eo6hHh= z-wQ9>Ft^GG<~Y=QBK3K%F)@*{eHJm*w8uWISfL3Z9wu+by<&<$Go>FE)`1AOWQz=; z{QuM1x&Jfy`2RmbXbDN=oY#=^Ea&q%%^@8fYs}f?d-<%vgI0;`sIpUNX4=Ow3&DD06h zhxKD^Eq_w_$)`u8uX^jg-;4#E^R<;U}{gYZpBXNg=owuE_ii)_Du1+ay)-XU%k2I*GO%q zPhlSExDLgc(s?%tM0k^`a@z#IiXoi=;-`RoEd>+jYo+v52ndCc?{BBvh9;dz!Rihg zEgZi;>54QTOEeoZKaC{+AUuk(l=6o=lFuKndAuZbTiYiy5*rNmPWqfwD6_jCM4D2d zR?>w#zzQ^?U=2MFYkvLi_~}Xe`1S?ISSM#=(q$Px>l`FOF721#i~rocpySG$oIhEl zVdg6TAqP4;`|5Im=i8$dH}n(*6B2@604VE&k;(0wZE|DrS=^n?r9UonIpe(43?LzK z?qTi^)*@6S_+oX`&Bo;kOOj92AHDzfs-CoRy-8b%)dtZ0JUCNfMOnTSban1-QbO>%(do87|v}P^Tk$TE^$i zwsa)CB$1k=e&zPnws}Erut#_+q06x`93;RPDDIA^WHHdg>U4C%#hG5a^Ew4`YpPfI z)+%c|toBHECMbgL^NaLUj3O(QX2k8Q9UIlg#2jRu*|*asWU5mzS?ENcixQS7pSvU) z+YCMX zQ|aU{1#Qq^ji|L1#Zg<&SrO}G4CwFJ!>|a(2o%{3noS2FPyF*;?9J=m7b2b*L2;vO zTIO3kr~Zx(o{Q@c1CLI8coxc;GhMqo6CpX5^burB7QYq69)x_j)}M?o4$K@dqv#h4 znJ1j!i+o{BFUOmSe|6eu9FT}( zpXfs@2)r*xgfZ3@2ZB^qWT+txe*}MhK;Z7V?{Bk+D}i5W>bjh#=db5GBBr-fN;KXZQ4Lp}P3NvO+TTT!cD2E~s zy~CZ(*_9myoRArvGGDK0#n}&_J~2gpX+dbAw570BE^94JIhJDo=A(0%Q)6CD#id5o zo{RDo_AUQIMS6$CNeaGF#&F7={joo%io;1)-qZ*{+Zt^Zl8NROgB5>co)XY zb#ni9oY;c#wtA%|riaOyY%w#F822vY9AKZOnMhLTHqidPh_>RL1I|628fnhZlYpv{ zlULYBvIQ5GJ7FA^v1`HDWne)pUw%#gI&+lz-J8tS47aPWEPsm%1I6dvB?f2C)lDc6tSnONFE*|p~mFThUUFzKQUoy7k!6yqfO#P$z-W<~Hb&&kD& zofp93$A3J5-E3TA;$e}&8h4m_IphqlqS$m`YF6u+_g+o=Fm(%dBdZj zQ?lY)dwePk)W2=iwNSPq&)r>jvFcgeZ{u5Y`|G^W&sM3aNgK`IpZ1xobwRN3z2ESO z%8+xkJ?&d$ylKlutL$NCW-CK1f+#wkTcq~XWHY)z2!sE>OZozNv_xK{a>7oufwA}h z{99mGU(SP^RJ(E2iNk1R=BJw%%;~4Mtjop)lfv8f)eE?oQR51rN7iq06rM&730P8w zTV*xVNE^M>?7^f0wMB>j4Kvq(U3Py4Gf%g1nk4&MEKCgbQ0TEs7i)xAQ)DQMZrQ^t znu(z^yoP0G_M7l2%thaFZz@OxWbawVg}xBTIK30Z->5QxHj9Z)6}0)JX(MCLS%*rj z7+93od|9HkK2AmJw5;4<7PWUX#8DZum2SaKdUog0@9&fbo4D_tZbSYInuu+o(Zmru z^FNf|WGblY40U=z!+q2C&w1dCWH;|WhDU`(4?w16t)$&zQP*!~@HwyIB|MTDUJc4_ zFYw~^rwjjETb3ufFe^>nvRQ1<)7`$NuJ8tbp{L0Mg1+1Kdh~8_k zN_CHb4`k?h4pqIj`}#I6SU0|fK9ZU8`t}2ee@pWS-Y)qmbSNEtd*$f0PWTWKDSB?# z8JI5fbc=3f3D*>e58-)iM+7whqa$uOzOR(2g?v{L+Y<*i`ccoh75cYTjdVl^Rn*w zgvleY_foffV_Emd%yDDbJwKSd!}}fQ88;mt-(&4^=fyS?w%!%|-y0b7RRH>UxQKEt zY^M|mJ41g4R8a+EsLl`#g#p|2zpOx5@wkR(mSdzVT(EV~>F4a+n)|J*#Gr24yj@w% zqFvbqGV?K_f_J7p`w;eWPw?mf1`L6FU*Ay#egtDq&q7-ED1=n_}!yy z-kC}TeQdv7=*9)&YUf@G>HWOx=|GICl#?WJknNkh`k|K+37@7P&>3^u1B{-MsH4K! zv@K#cH3ls_b2lIKPTRRQyHy?2?%O1!nd@ciPsVoL`(q1C!5A=Dm-&h}Mv?d#Q=486 z36Y^{@WScs-nxGQsVI>PJiycKOlgK~q(2P(Q6T0kR9_OX{V@CA)1mTP>$~rO!u_5( zNSVpXZa+&I-m{Be-WL3^=vlrEUF%``m?#-kh#HR*>MYPt-1;1=`1FYPN25uU#$(jf z8S>WEAIjUT;>_pF3^|V%Au%m(H@eww^ufdu>{0o~&0C=ENht)tEIP@Sew?YU$kW*w zIi&w6Y8)D>lQ?;>cX7a@1DNRIIA+61sMjkqf&RkC)owZcygDDcDm+TNYM4WX-Bvhc zoNUj3ev9V^$e!JZ?MdnYQV5mtm<1{xI2ON5Vo@Oci%i}9JDyq3m2{$Vt=Q@3Tw#`8 zbD_bN1d*c#o0-8OJEOh1`4H~)^I^_aEp8d)y)85{8D$;NPah8pDO%tNDV;}Z3pzHZ zpvin}M+GRtv|jf+c9h6NakpZ+@1>A@RDelCUxB&l;_=HkAm8%u=C2pGe2!M*;5#oIi7 zK%>&Rv4yXI;?I;igX58&A7MfVa|7xeG5%aPX5101_d?8Wo3kKr5$C>w7p|=6RP^sS z?{2*wToIC90FP-Hk#5JHIHVp}Ik4o$Mr<_4eOs|ISZ*!}UGamEQA-w4Z&vDDnDgpu zzxyMsEU3u!v>5lZ>dXJu56U6Ji*E_Oi0j*GlzK6y&eZoDl{3sC8t3R+NAtC~^qBi~ z{{QBRu&7uDjTDQ~rzU zUd7)nuCEQr6GK1KVu|cOK#?R0l(j6YRGc8z=pe9{R___FM3<$zK3~;6>qfQ#;!IXT zx=ca?Abw62lY(n&`qOBEd!mNo0mK_w3s+vetKM1wLT0qC83bZIV8XJV1|vu(aMgtM z_ipZzAZYe|p`lgf34lu9-#K6!Yaj{C9so@*+^6wx$a}HrOw;Pw6Yq8$G+vDbd7ov=;t^ZECqf;7 z;I_s(eNxX@Y)cOh7t z{KE~%fQ5zs|0LES^~Bo^Fw0E%q-JeeI>%r{ifk9{+jci+`e@X(As6y71 zRG})}xr^4sm;n$2USazt)AI$iBcC38dt6Ai5xq##YbKQ*Q z9Z_BALo##JZ*6kL2L`?34ELsbmW#2(722wC@Rg*sI1%fear=sU0gTxu_yXD@9`t2= zB6jUa3Mds`l8zhsQOj&_JeQ@i)r{f9Xa%nykDah{v%&&SR9rB3#bepmkG&+VFI{5t z>wlDfVxbJYzO?nco1iN6lK79n8=c0nF}t~(RBvhEGAfT&hOB)UF*1YoR0jmaGT7O> zGMc75!pPjA^s~7x9v`}*7-E=H^ngH(arvJtBXm#qjl05u^9hAq;ytg1RzS4I1_v%Y zKjtOL7bMf9ZR`^;s6$xOIieBnywW!QbyVKCR9zS>gjrRE$fxBAx-Om>o~(x{D5Q1s zE($}3?kICISBITR$i_G;8&~QxFED!iFnMs)S{oO^Ao9AiUfE}-90dD~y%h21AVEem zyckv>D*&%`kMT(SlG}#k%6Oyw+U_UiLq>OIdE{cI9%}zv%D%!N+%29)%{7fHvGu57 z04gffW|0(C|KVl!PBO{Pn2}w8+Zo=$^A|jaCOSHv2uaDSfah1C((T7}d{BzNp_7t! znt-<6CBTHL;WKPmj^uh8wn3Q40B`1Z53yV+R^;_uV8sd6;#Uh|2t7W-c}zlFQeR`s zy0OJ^GpD3nPt5P84N9BrWhw3+OGiNF8;o%y8fIF)O2qoKP$vF-B}Yu`YTRA#1Z>u` zB8mkDX>Bv*&grMGuhMg<-2#|?&hcSmVB`=()@Ia8g<0%XJ&bjx{o@%yFfF`4!h=et zW7B(}LSZ6}4^e>+!kpwKN)rosI4iln?8s+5UlAI6&U>?O%ue=OJUZR%OE#wJLz+En zN;i|rHMYF&WC=~Ba5aG@$p-G4mo1BUAaXj}r?wY=E&gamZ@b(5%>Yb-*y{;IRLIWQ+<%}?q zvzMNo^?Z&aajz}ep)Z*^t^L?Ue%>u6W}M9wR54B|A-;v_pn&HQB)j0ax~Z-O?GU6; zj!Jle?0|}qJ|#ii+j(?ejm+-|efl-17ziIu@3;n&jFREL4F7DNI%Lths5P=ckc2dV z8(M_3b_OK^8|Xd@RWbO7XbVrvdu}DPalJ?`zmNa*vh*gUCQeC(AZ8Gwk>WL5NA&m7 zW}|v*1DM?YvG(qI&V2-%VM9wrP5CD0Kk|m+a2L^Tau3s(>7EZ_av3N87OoyUa%lKG zV2YmjoirpWQV80?V?&ZZsw_9s7MXS|#KkMd;n#JA0!O12oMSKeuQQ1a<8&Bi!`0mF z!G)brTqhNLm@B=NlbHpmdy5j32CoaU^;v!&!2u#o0-7+1Zf7(U8UX|u*63_#@*M$# z7tgyo`$tmx-dyHr5Jbni#S|zpuEC)>_6HWOP%t^;!fZ0tgXUZ{(koxcq?5|mB!~+8 zjl|o#w)h;zE6Oa~e>hMd`W{O(jZ$nrUmhkSQqZ~^AaCYR(Qvy$`jLm#yrfLB^C-i8 zo2+-+v%Yk!RQ_{oJsT=t4rWdfpS`iE=@$?46bUyxMibNqp!Q$yT=dlU?C@+q&Upd= zaap<2-!$pFS%g2!y79yB=>K%jjE{S9e0oK*s&};WQYOlcywI5W1j@l{{Gz6y=3HM zZT$PLK=v;01(S{h;JX-nZAmp)AkBAT$b;>gdvUNHLV6I{g#GFNb+fZ#Q9-ESi(ds_ zRxnyiF^!RGF0q2b?=i1F~-g;L0}-M}k3;)O+n9|GLQDiXzt z(ZusHGFTKbt3k;e3h`r>RhyE{QjrMrbIj{~w)U1;o~w$gjpqCG`qA-M_Aht;4L@%J zIravn%P0S0fut39NXqhC2D0C&@$gOHdG#kVi0%}*2q%P3OBtk97o2xpu-9X z&zPjxGoQ}=a!Gta5Sf{`#(X>)iz*1Z@*|&MEb?t~gsA3r3DJt`n8axt3*Y6peIF|# zU2!*!j@mS5>J;LaFQk!l1+GC-(05p7VbfN{U%f;lt>^CJGWxPMvKa=c7J!~&t48G; zuEu|U%0+VzYmaX*>UZ z{d1c~qX`B1f*lT0&DtHmi!D?Br08*ZN=Lknf_)_F1%$05k-78=V@kev!Ju9ic$C+L{q3H z71|b<3ay?vl}rg_Y(}_rhp>;#B2t2`=N5kZ`8jE6tQIG}U!)y&HStFjn^4H4mP|t4 zje%BbPznHbRT}?SjYKfrWWx(&q%NJdyxkg192sf97G66)^t0XVj(jQ&?7|{gcuif7 zJ1*eooZ|!2xgYRZ-sfB3xp}D~&@Wwxf7`PbAI zOI0RAU8xMU?WES~tW?5PwrIHLv7DX1`O2mVo5{3c@r1FeW5WIz`^WN^KUmw!i*mis zPYn>?Yu>m1s>;6qotJ!;X*R;VkR!zp{hJ5Ril-YxS1Xwjrsi~ETGbLUEJ1|sXDI*% zpj%SZxL3$k1a1xuoP$y^aXL`&9-{fWIFH@z^hgvyhSH^$e}3=#jK<3Df#qujTb-&?1;TD!A0TgY>sw%A4pP?1{0_yYlQ{cF%#=r|e-BDZiNcpt@zU*A4pYobUVAojF zeL#ofhV}&ae|Dm-a5M8+{Te6{*B!2xaM4~R(RWk)tJG_M!mGZ8sxhN&ukMl7NF@e~ zu!~;JLAZ7cJZ3T{y@{G|d=sdDue@ji|l7=-S`J@cE(<;ySRN<`}`gwPD26c*GDpyzrQxm=pE(7J+} zU3Vz<0Y`3Z;_(t=WF;-g8ci@?lBn)c%Gyrr))bUxG#U0s0j?o)v$wZ?D%OB9;Oj(< z2C+3?Be3-Xi+n<0@%bd-9BcYw6y0k1sHOK2$x*veEeS0&oVi66_&AdAlXc^iTD4*#@)(cCOu(~H`0_dG)e#`r9aC#~m!@UfbU+n3&AJN0Oq=VO){ekb{hFJ7^ciuk=*IgP7;8g+9YVA3H>8 z_><(F84@a9H~vKwE`%=u@yaISWCWg{M~HK@xk*a=K;v9 z&^lG@%*;E4NX_26oi>DQ9Y0H2ZnpqLiZvnPr!YE$!(%0CBy^(~@hO>Stt@SI{oVe| zMe2k)eeTn@%11&qA^~utIAH$L6Wx7Wn~#2E#rOm)bnKd7t8NjCAg!a+m-d!;-;v_m zLa>`!g&doBHbqenK7b(;DWIsC2}xMy1p${C8UY#D6R-UcmRl%tbU#2xw67W8+EzK% zEC*43gxhG{BbBmSvdT3$`EcaGDVR@T@o%wf{^rqA5mV~_R1rlR6z}d6A^(JX8{T_w z5qK@~8!R}c_J-E*(nK7D;`n0tW*|E(G6U}+kBIFTUz1m@8^7K^iA0x!hs$nABJ}b- zVgOuc<@0gu!d1%AB5cxUZ(!kkcN6{} zD_%vpNst#nKHES4vVUfYGff{#F+KPMU`W7c?UuPCLIQr(x=(BRhU2%v)|c}wX?$@6E>PGDkVUZtfeo< zyzIIx#oip?Bh$g|?EM_L8VZ@d!Wv_fis}t(9`M0WjlVnl=wU*j_owxDgFtzg_pQGj zSX#s7_U~5z2p212@!@vsH`%@xx@R~&_5Gqf3gG!N?(4j+N80a~Q-17qQ_SxbIF_%K z@EPgWXj^vtNH}>fqIKPo-}*|p)caresw%lVO+oFhM{StxYs)G0%gu5-Jj(6-&9gqE zin@M?X{Z0D33*oP?Ej~; hQm=PV--S~#V@Ssl9}NGJ@i&Px7N*uF7$a!H{{yKCe^dYf literal 0 HcmV?d00001 diff --git a/examples/benchmarks/TCTS/workflow.png b/examples/benchmarks/TCTS/workflow.png new file mode 100644 index 0000000000000000000000000000000000000000..403a17de3923713e395c076b611f7e154f370d6b GIT binary patch literal 29877 zcmc$_WpEt9)+}i8h?y)LF*7qu7E6{z7Be$5Gue{G%*@OT7Be$5Gwt}^Z})BN#{2z# zbWBWl&x!6cRj2Y~W`!%tOCrL3h5Pd53*t8^G375`z~VqhF8~^Jrsp|}3G@Nxs4OY+ zrE&uQ81x0wR9H^<%a@v1_&0qh(05onDGkRjUy%C#9biMY#YSJgI7xpK6IOB4IbVm; zLpS||xw2h5p{Iw6XlvtT)$0vnc6HT%k>6l`$YOySkqPBJwOP`1PklAo?dMwuyo!gs}+Y(V##tNsyMut?Kze z&-Z1V)gt;xNQ$(;KHD^;-E&Vy(NL^ux)ml+t}c6F_iph}pZ`kO(@PU$8|pAautfW# z0j{a3{S#8l-A+Ps=+Kg7Re3Ytyl{0|tTD(@1SJlIK#84$CkG`IcrOAAN~lzaA~-Mz zMVV1YfjFQdYW+q{9^k^;CI?Di;uL2od~SUMle==iHUP;j~Duj)bn(1xMe8WXE*K`SMv5Gr6EOxQ`ioraUg<| znBFVI(?^}HL1$ocJ?N65p@LM8A2m&Ji&P^(2 z_oq*GG8S*X`o&HN_X~^UCM$g-C_d^wVD+Y7f8)iAw{X6R^R)}b4 ze3?Sc=#KiM(V_ZtZdfH!etNFetNZzmC=xDho%va--M2V@rH(2+5FWJaw%JlO;^OoM zwqpf3`!?#?ZH`fnR}`f%sa2%8SsWt=Z1I7a^29L)>P>Hm;!WXkWg1baXGQvN`#KoD zY+nn?eC&tDN|=0{AtFo#kS3gb3`z)keuK}xy(+$aC?-@sg^Pm;?Aj)@bE1h~$_3OV z9$xQXzh{f_lz*e96OpoGDJt43GBk<8Tg3AS<{iA|ix{O-q}A|);yFYr z9`Z%jZbG6vIam@z59>m$jdxUR0~CyE63ZK7qk4>a()aZLWj29qCqDgUWMwGeuu&?Nr2shGm~8F zBl%FMgh&5u+~5~9c24cFmtP>+Xw-WbPNrw!bM9lpD#?;7TiE_=Ok@816CU=xO^{U8 zkucN46IL_21>v(L_Yds>y}$C`{#qBTry2YOI3^l;Cqts(HIxx>=fi2p-oq-fs|HwbSDSRH{yc|v1x1~A$~VHBl|^>&8DKK+jYwXbUJcBhISV`Z{fF=83`s& z+z69T>{gghe-2?%AMnZSy)@J8j&0v5GB)R`(N&;&M%?lJg$p}AUS`O(5Na<`3UW&# zl%=pVaig~kaXiOge?M1|c^f8#_zfelNm{nUty{TCQEIhXz;z=_9RdBl+SWBKUM*># z;lUZy(`^(4s>gkouCMSJXf~a>z16HTm1q-m;@_iC!R9+HU~oCgSF7s%YW%Ss`v&Q5 z>b>#=X*3&%zmot^WnC9@} zm}=DxMSlmc44_nVG|=KfY``*$)R7_nmbN=)v*hv~QyqFx7j{^HML6`9@fkWK>C;Wp zzxuJ{gri!9rl2DjA%2nvh*i*rG_Lw^d$~#cBs>)8FvZ?jHW+Z7trDkQ6!7<}H6V?! z!dLlZHX@1=KccMuKpxQ%OLzUK`(ziPXG|%} zaP`%KWHv;EaGH(@E6}tv0h=_57NsCIZy$}G#y$UOy5c=Xp*Kuw0)u74OEjuU67B+X+IG94q-r5AZFEU^iOK;@U2lYZQ*{Z|7 z|5<$Z+>!5}RIplb z50*4%4kP>Q)Ki~35hA$tAb#Z&&67w=zP8!F%AH6ObzSq!KVO*p29c`%e)!9SESIu$ z&IxN_B5Y{fpj2pE<$35ht*gOxA(_{W3<(bcUNH7h`e{#^^f(iZ(NrB{gh)=PJZ<)p zku!{x;?)_DP8`Q6uQJ%~%$6(akm2mRlP@&tWToiQ)3*!6x>oMq%on`vN8gaCvdfTi ztUxD7``w8&cEyIvagrbQEtJ_OkHk&{nsrj^cBYL1vuvS--r$iaXz8tY=h2gdxB;+z zJi95pLpzCJ_Qeh+!Mplpc48VYHZr%#@mt%=kR-pGTA*45n5*g1UN4W~Ql#PCn4@u%Cx`*)}QZcDE3CWE^B?>4Eg#USHp(DHZ5M5W=b*@b+LBZ45o3z8A?BccDyM}^FVo}gG9Dh{_<4smmnMkOlNN11QIdZ8(T7AxXK zVcu(Wsupdg_ji7`nzEm?;4qy}Z0VFPeocja_AfY3zi;|}&=gC&!vGB_RHcyum@x;5 zi2R0Th^!gu%h_4~q)fb+aMaGpbfkkismz%ESaWwpgXGH+A}duqD)J(Ez*4EUdMI)x+Vhym6)uiu*cjKO3C{WaEdmo!ku4l|Q~d$)6}5 zm^PaUbEEW^O2|=kX+;bPSD3F%Z<7UcuYUrQJ$k;(4T~^#RaL;6h6lP%`unUQ#+@=& zpXS8`)GoJVm04|w%krJ&5zgR+@MeaT{&=5ALqRKd+E#MAdlv!L%lvND|t-1 zeI0Txp< z5(Y4rrSgX)z?B`{40S%N+KOb=*L6*XzpRl?Sq*y^xPmJb%p2hvmRrUDNxePkbG@uW>hJ6;O zOx#;)G!hy$Gm1=3+j*UIS;p@CZEcNy-ZLcbxl+x{)pD4!^>D_dvu@u@LTs{z>OIA1 z52Bo1kLGo!vRTsj^>JF3{Of zp|ZKkcf|5ZSb`)>?OYU|z4XAk(w)&SpRv7w;DV_Sv<>#@Rbf6^eyp8Jh{nas3|iI+ zxwsCr-`wJ)_3jLFrQ7tur*)<9yzgjS-XH#pEL`@aTy^nNx-T@V!yE_Apid)T|Bc6g zxPD6hF;)6K9DIbnC|-~NAJI)Zk3%0*0Z5{B}w|Gp5 z99D;R>K?21DB^!tWRb5~gpW)#>(Qi}!)VRB;oX3hh7*6p2qKM%?4)42IqXTA5uC5{ z8|(~TgS)v{hcu;+uVlkJok9WBC&VaCLH`iABnVJEyj(8EiU zWMg;br6%Jid~60@?P-WHQlfks3DzPtu1BJH3?2H{)I6}nxaJQ3K3vgTgi6YUyjY?r z9^$9*fos2YWvtmVoS!DLchs$+{5eV(BX2AeMiJ0pF=@`(qqX98aIz5GFI;dx*!r1BKl8<8N-}b~vO+pBpT& zKWPuB$Po*36RXyOFEf2YG2CFH{&=Z}`uU&E#Y@U0EQo~!D`s!}kA%jElXXs;0pHVi z!9E{~KjQEI>2a=s8d9#q^?d2L8gwLscjGy2$E3rzAO7~&d;kXsm5f1=b!WPSam`j~ z@e-bxkk(Pa@O?wF!idbjmNP7Nxc+)7szrAVg%?blEurF7nDx#lXzhc+h`THA?r^ai z*ZPYs)FgZ8c8H{Zu9sik7v>~Uf+GFznorQ64ofOS%2DFmVw&4cZlfM~Sa^5m3vPm! zJN8*g2}%@{9icAlEOKm5{fjB!;U=C0zd63ES7!<1Z=uC;GqwsnLiO3QOQl5%^1t>B z^$8z;l?ky;tt0ULRG@cb-qvuk^mvJK9m}vyO}_8g=#vl2o7!9jZmEF!>tE9-(tFDB z9|>8??$pC5y!aB0BH|oNu2n%MTh@y8_z)ToBzDFP>9+rQ&E6}hw1GWlL>E!){w6%X z-h5=cohe~9b3rt@R115D-88E0Vk7Qrg2QcNp;M))2SS+B3vId+-UK0lzO-@^u|c;f z4$m8Oe}fs6XSm;0CUUlTnZMb_xqg2?88JuCt+6hkguch?OQR7t#+!tl34G)W5-2&z zaSx38YwQ=1I2A3SN?VHA8f^@fjarrBO_shwR$*n|!Y0)}+i?qIKJ@4}b0I|-{3J`L zai0&PZ|rz(ep%BQSIF3|rtI(BaMHz`#Go%`@#JzlSv*=hjwoH7V3}WdbedYRrjg>U7rCmLloZqn~aiHX7Fq5SD1XPFX&EDCvX}Bhhin++@3^HB7oF-t5K4}t8_&D zbHbjbkysI2u3$6RYdC4*{D}5mGBfrH)WU^Uh#1!4hA-FK`?rN@aoQH4B<18@%v97X zQyAZ01R}8q$0e!``f`-bmiRN3Qe6Q7Iq;esB_1Dr>2-v(8$>Z|d6%JzXgih(LQl^Eee zU7^4So+yij5dPentzRvVGC0npQ;k0y@z>g*CvIaRG~nWMVlE&n0x}3t)2aA60A+%B z^qqniD4_1(Uu=yxjJ3^IhcmthC@hN4Xs~zPA=bRY9IAcVxAsrGVuPZs&d#RpsUrb? z2e#8!ItkfKN(RSi-zm46q`nJ-n@Jl9x6yA@m+4EakKHAw)Rl*6!14Yj3^`tr6ylF} z#_8O!(=RES$i|U`^BcNF1@4?e%iWpvYz5~D)ZZa94}*pDWHf0Ld_I$j?c}jwZMX|e zgC7_8$PzT+G|22K_+H|#UO!UGe{h;hCc(;>AQ{kKetYq5t~Q)R&7nH?>#nInxTM15 z@M2aOrgCh;v$$Ui7rD7;Dc|>XfUD@Wwmrue%C83WRjK>@|%9<($V{> zw56&t@{Tnh^$(&jY)~AJ8T*NhY_liBN`sWymRf9Fx8%RGG=jc5@>}jwv$U{;tw_Of zLjUfC-a6yvYPhrva>WS^Sl7+XpBuga=3j&6k`x5hoc$4#8O@8%GlxfvJN~wQbk|0k ze4gJt!+on%bspplKq^#T_CXoPnjEZyD;<$Gs@G9oUCte>>nhzB8P?@b4Yyiy6eM#B zTa;^G2%>tm9+(jrQ+dCS5rI`3Ht;THazLS$u`=+1(R~nMZ26`16sKK9ed;Gy4{UIG z>r6oA?%w%P5);zmgPWhY1>$s$dTUdnB%sZt1gIaRj3JpqnrIvX73j#G75@XsV)0Lq5$*bPvBY4CD8JS9iLDL zmaO|4Je1{UO<8-(BS5|R@jHSIKVd~KPhGOBS%T(~in9WE`jTf&>W#*zr%iHV#HNlV zZ17s!CT28)vTO+S!VP1Bkk`;A8bDi0ih-u$DCp`f3q9Dj@m=jL9dcp7FwZlzlV5YE>h?|o_?Qw<1S1>AJoX9z zpc7ig+;Gpb9iM9!t$#U|C!t)QX5Kd`$NGq~Fq`YZWlfuz@HZAqRLdX6ilIO)>~T#S z@dGV&e2RrqX-!=ZEpGE>hfeA;Wsx$emp6&y-E5`IG=TI8{eGhH+>+&Q6OII(co83) zpp?1U-@o$}$Gd|hP0xmsChx_)+(xB0vD=<>_VHLX%4iC-_EUvxNa{Ke1`<;h-YkE3 zES{DW&uJcgz0rz4**Fw&L@GyOwfy1y@k;kgC-N?})Uf8-&E5nWt6ku2GybUkt_hWh zw8M4&*G3tRC~Gmsq5PZlsdVfw!k+}ePwz3zQod=9n~rU>5#WxEr?$Hvo}A8z?Zt>Z z;TP@+Lc`+UM1!!2l6R#kSnLB=Wo5~l?mR1J2KC%$`0yuICNzIIN1O+qdhi{2w2Q5{ zy|q5F=3{0#HB3@^qrud-A{g?l?IDHbTWb3(JrhdzN1Hk4PhLJTxaLaM<J(iI8UzC8hj z+QikSaNCxR(9}B_oWM~YIeq#jMh*o`bgg@}c^SM*!*B1`^Sh6qX+rr%PxFn7mR?x} zV%~;TLsJ}uypd9&lHad(FOUZ^ZSF_3%VxG?J{Vg}vgusuOpz1uUk2|}=+FgW)Lgre z1ApocTy!u=6K8_z#au*v598*oT7>86p#tg8>yHzu5s4M}DyfrgE_X`H9|c&xk^`Ck z{Rj1P%|kTa9`idPnk(vm;yKu63`3Pt+b>vX0|(B)-}%Pbc(Ia9$FkVq*UM&dI`dIwR0Eu zGU*(9H|kv3<|&=fS{vqfg-4gIZc4m7yBuFqZ+w_9!W^yD6NNN&EO;M^D}rCQJ8rUx zFbJ(s9jibuDzD@3VP00*aQo&_{PK}^K22miq(ly{7cQ~UHlL1=8Q25<;t<(Ii#X4SuxdjJk^XfQR3g1BayNUOX+IUM|odP^RafDbQ>QkwqmB%Wb^EB zym45{YcO+L@h=(;;*P}Un_Q@;q3*w69a)E$+0y5&AJ+L@w$Gb|t%q_9Ak862ORQ$v zU+p6)s7O%o$U^XOKv*`z8}Lj%pG8VcrVawGdn6}C_f}RF@NP)^arV( z15waGy)=pgs`B!0-byJyiSv(Esq?lf3)q6(n{7-7^JoiSh?RjmBl;b95YQ|MHeuF` zI$iB|Zc`WnILVs58Lz%)u;rHR9C14wgMe8+RzBLn_rV6c@nHMceb`DJ0I(+}VJzZJJGG&Qp3vc6Y{D1#(!o<+Cju#mf8cJ}hYYa}+jhts z{iRN7>&5t8SoZ4%rW)c|H_69m2XyHadjlb(h^-FX5Yrr}Ye7LD<^}>J>x9en{BXLli zpOJlnaO%MCr+|EI@*-ur)~8Vf}g#jrVW*W;_yt8|FhM+*HHWW4LFDmBB83#Th= zk1xPz&~u2C0@Y=htrwKrxomkOO0$fR`7XO>22Pn7m;Hwdqq@^v?|TFz8x1^t#vN@@ zCY5OB;zo}&nRG$mT8h1PmeszuKd+h?n-ab(i|S|F( zxBLYcJ=|38$b++W2wDF>|BY1l_RV)OAgYHrW2|a zq@(D@oIj5N)-flhH}GOJT2$eo@I610`9^ic@ODrOqYeHiPz{lm|L@7lQ0v7@6nu6C z*q^#ch71eo>|34F6?Pim5s@g2ciuLC|49iqG?vy{;Q= zS>)BtZ!u%2C%t7QzUw=J?@Z$ARxjqe(YK5rj|K zUT4;2MkV5NMu%TWg2?jypyXL|f*9odo=e6E5$oNGB}$QO`eP108It(T z_{`H4Fto8d@N~+*Nf8HH&{W{qMc9AQIp~9QM#e2P03=~R$4h)F8J9YzH~i>=yTMMJ zDC(lpnLl4p#{<@re&00oqmWffk=-nE(bXw@k|?DO9aIq%(U?|%bt%Ww`L(~DxA8Me z`_&4`QfN$Xky_J7Q~hhL;OAiP8I#xJl^vT=+sg^3B~!DEjMhRGKPZ29&NfHo;kfRq ze76I_b_U?bD?A9U+JF4yVRM=f3ar)7b_s`(c{fE@DfpSn$HlN({#E?Lwk{vOWl+iw{E9C z%=Tg5#H6W$RK&6XXm)$s_Jnfy`?W|!gA8q8P?QP=0}s2O3_lgrQF8aI$*I>nayR@Y zfP5@)>xOmsjUrx+bhu4&I00PRhSEeAj)ZL9j3uz3ib2*6LHl3=X6*j-wpj60-FtJr zv(9A+E*}X?w%SmO^|&DbY07aiu=eBi>Vgr5Y40Vxxd~Z2UMS!nxkBcEdC_HywlL9L zb+DfSOGOkqUoYqBb$6z|<}Gg9?z4y^dx5GoB3ksL*Q($axhDoB+()s&Df#>Y)%g2K zco?W-vDUtC*qG2DYzQib`D&pe)Q|6q3wjbDT^cQ}^86a*r5!v7Mn(cBuU0VRVEq~=dvN+dRf>lxzUi!;_kOzuv|TI|&jkxyA^n`_}IlXni< zB^R_QecNS_s^Wna6G`A)PZB?DV zJ`cdR!x;H6xbLJCBGD-81z8Zr7Z1lrD68;&s|%HW6yzwApgsdof_dEAQzXFamrjyK znKHCpD^HY|1v359E3H+Lm2j<^zGLq>kaZp|_%Y$Zu zgF1M{vP+PW4N?CQ9{&P9^*8kGPLuU!Fy)VwUbM4-InXrX)c?zdg`D))#aO4TdcrC4 z8OqSgu+QRW{4nBfA18>QS}iPMY|>eAq4jqbW_sNh^oSrWni;-R( z)vm@P{4l{RF>JSl3Gb!*Dn7ChsBdzRj?8W{B~PIl@F@`wf5U8z_L6q?t5j9ebsgq-aXMZB4PlIKeEtCOP$iB}=VFO2nD^w7xYM_HsuYBq zVVkM-JK)^OF4gNZJV0KWBuItqT%Ca51xp4!R{y&)k`aOg%&QIOrhw;E2`OQ zlSz*dFz}!j8_4)XP?!;_5rtwC$0mvc%C9`1^6Nx80inKB9}! znOWLYD||=6Hn1)Vicw-6V+D+KYwecbt*vc`3Md@yxD^UT`)j71GLv8$OP_{6q|r7a z14h2PM+pwrg;16lJpJ`EDEkb_*a{HIoQROzR2gz`A_+9I@xSi|+Cy}KAG~|!5O~w7 zR9JH6ssGryTze(w8rLiS+2rSZD{^%v*4)0ouP9#XU+u4${^-OYHf;m0Cjf0~j|vjx z+Gpe`49!y!Vi9a%q1tp*X*w&q?Fuo$uQRlWs-@?YrWD-3OAq?8El^=D z#yr#;-yr(fZH3l935(OjYA{cw{c>mNr|yX;A3Q=LvQGN!_G)gi<@1)fl|*?@py!M; z=~{sDZ@zfVy=bo4NRY?0`xss7V6}clj{JKfkX9QWDBIKYQzScUKeh`fer9xj&%Z7e zr!Q}_P=g;$LEcU0?WW`lsy?;R>$wHUFR2N@818(-$xKF;J98KRI)Iqk{VlYqAF)OY zHYFG7hjj(W?Q!w0sPKaLB5>Y)Xcclji3w=&<=LYNKD`IXhc@Rv)fw+hsxDmBMN=yA zof1+(dxuR;i>hDk;1{{}K8?aFvYkVwtZl*|NkIZ&X+Z`!O1RHb48Vlx6^|x zHd%CQkx*3|lO%|GV|~`PjG|SRRfDV$BpN8{cS8#T=0~p@>GZZLx)m5IjgK|SC4<~3;4TnBidc07hS(^|w=fGVDeL)}m7wRn0 zd9k5|W*pk6EX3fc?N1aQeS9lle#D%^)2YUAY+pA9I^QvHqGkCr2h3idJrt)!-!IOQ zW)Dknf6=A7jsYFz56&+5)Sh}EYm`$S5gK|{g!ntQEuEkBVHKd`5^q?vGq_Lvk9zpJE-6RmLZlq`Nyb#!LF_AmgwRC-+ea3%;~wK+wyUmUY!NEBM2 z{0v!|!a5?Wcek@bBO( zKzrbNk171Zd6|YNid7jAT7+*pW%^^px^xVj_$3l{lxJ+M&^ed#101>T2E&yLeB=|S zsZXim+jBIyTIvuA56FTsVT1|C*{O8vTFroTu|}cwutFpfQS%l_ z=B-c{#GCal@pPex8yS}6-KMeB?9)^&P!J@}w_9Y0#goB1^IqCfr4s~eUyHhqxGQXCDQ!^fHQlI*%Z&O)p>891cQTMH7rrCLFlIii3yOb|zt zhBxFF>39&U|KdlCk;_{PqlTeesF)xjVeNctlo zYfbNOW^^{&L9jWv@81?B#R=V?di||4q^h@Pib?^0O?c! z`!g9kRA;`Ybx+rxi`Pyej1!vhgKQXh>4l9D$lH52n`VcFTR@PsQiI*|HiWIH`%H$x zf)`W^axZ;;|B68Ifk3YEP8hNsaY{>cQM=z)`@m_}VzBdk3}Hsm30{zsTFGV`>O2CY z7~>8Hhe=6M%BH^fPI950(Uv8nL-!#65%N*!njKXv&;z0k*!1PDCJYTxpwYS95p+E4 z@I#uevW`jayT7IsYsGUJ>WkzX4%sxztlwqqCvvZmMWaYO)m0OWyg23VXWY9ytvwn!`F{==5u zO5&?!`pJCAsU{7phDuJJUw_R$kd*eK`8-0*$0X2DMiTmksWp-Agg<86Yy^mx{U%@2 z+BcQtdmwpTb|?-NgzxKr&50_%#JmoIWZW3Z88eamH8Md$rv5b4BP${$^q1QiLziwd zv^a5G(c=a3fNnXI&xOeR4dhe6?a_ojTz=nTi50O^?=dC|KZh__bKI9WOUI;!lGKWf zcO|ZPf}Fr0>#?e`)-Xb&+F5b#ndL+pnxiIVxA^_^meGpjd=J|~_0{sy5}U42{o&Set2W-hRb}9w!#=&`FxEceyd>>0p&$DOXu0$8h;wQ_sgs)oHqjw|EkLT$xu)-} zXOrq#Rj!S^x{4Y&S|}93C1#8xRbJDk!NR^gXDcwm-klirTvV5}pIp8Vo8lC1OC|($ z#VWGXw-)lmBt6P-{_)b;ugE<%630nRq-WDv!0)eW&yXovU@6U^L;z2T19n#Z^_z^c z^fs4aYq}GSH2IJ40hrIw9BT?ahb+ngz&W!=+O&bB!HmYjO{RwL!S)1g9!lZk#ngs8 zeOk70x#`Ias9gy7T6XE@*K*hkRQ_p|M zwUWoMX!mqe=IIBPiBA5b*<}eh@o2=?A1eRCP3j$w`B$eFI4b#9iFSE18Qi#n6d)TP zM@kccS$GhqfpzUW2XW0OO$;V4O-xkji4Rirq7#2*sIxH@V3O}HdS~snFze2NX#7lG zu%yj8gpjLLMjAxOqKEge^n?ln@RU)Kk0By7z-6T)L4r zEJybVTV0sx(XJ;c%b51JWi1DU^js)3O|iBWmf)}Fp=(f|m38=9E1bu{CfnBk6A}b{ zcI<9K)!klj>i_A#tj%5$ij~b*bJOpHtac1l&5&ouv*7urFCANWMi2uk zHyz15DUnRAtL(W`T!(+;GlT3^2cV^+uBrVj!qdSfBA|YpNh~Hr^QU$Pkk$%|Cq*8_ zW-14ddtr2uS@#!j#FDDY5`KwgZzCK^^87Rkt+)$A{E^4?$;782C4%CXd|t5R=70)9 zhN`=@uV6w9C?cWBQ93HaE0g zvwgWUqC_;`EPMPKW$9h=WH4#cceS>}XyiCu1YYwa(gXUYN>_#QoRUVc#zNyPlybg*=hzXKr79s;4dOh8 zz8rI4n4hnB7A#2S0x_;Vw}z%WnZlpKj*HHdD6rc=l=#Yb6{x`XzY$>|&nG_QY7 z0-nZH@CJI^HdZ7OdQbkwJxk_JZ(1w<4v3QHw(HT(tf`V+RvZ-EQ9^t3k%4@r)o(#3 zA(Af>pQ09(=sBeZXB(Z66Sqi_%Ww+9gtMp~1-$;r%07LA!LIs;zYl9Z4qf#QkNN?y zg#RS}ZPdW4qpL;K5~^Tv)WFkM!7%&v%${r}a_Ms!B&sMN#(+&U_|Ayx!F4(0~E@=aM6rXHy)nchj&fA93l*s>&GP-poJG1yIt> zjQ7q$0G}Z3P9#|xK9~f7VCN-IvJNN(h(=sNmjCvkAH;qMRaDZ=?nhksslpZ>0Ul(K zkZ_EOD`C3u?J66@qZ178NrTjF9Le~{?M=;q0CbI20rCFo6f=YDDhyb{J9El>Bp&TH zv|Zb@=uB@0!_cer;bm4tQ)X~)k9)@oQ3k|G)~7)T*cs6|Gw*IxV`I#|LBEIM50`&4(|*OHKxPTB+;&H01|negb+{C=rs}- zm!}Y;{KU48@IBf5z)m`ceGT+oZ{%fg*Q_}P(<%rO@GhrwI|d=*ap?5~K*f!``2uqF zSaaM4A;o01!BSc+O2YL(!uONix}hR%{S;NlPYv zY~LaPR?FSk1Ir>*f##3%Mu`9S?i&B06dyog!DnbSOjqbwWY+CUE+pDkK3pTnjME=o z^+&vY*XaYe2q~|fhGc+x8TVUnh@#_^uXoU#6L@+jQFYnj+eic6$E!J26@7R}Tr!L@ zjmA9w?y8a`NEVm%I3|=<%z!gv0v@S_^Iz_dK{~a_@Zg}NlCZbVu*sU@&$m;) z_Xor-Q?439BqXJ;6tQEIf@tW{2F}KV=6^{q$wQF3OE4kbe;hz&l#o8?e@9Oz4S;n4 ziM~+F+;|Iwy2F~?FMGP%(bPt;K>tuQKudOh`MDIWya_#V-CdOBi@FFP$B{T+Vg~kQ z-Td^I?U+bnGFSeFRN3%E7;}G~?kF==<6d~juhh_u5VKx=#n75WHYn;?@b8JInzu4; zKyOr0ntv-Wy>sewo<{na2|-re*?b$%= zxIwC_gkr{%u48dOzw4*}Q*vsd=bucuWRBLEBXV=?c(GD}L=E(D1&#* z3*8>}AU7eWLT+uevb=&jSY(6U>|*`Pmq1)Wr|7jKjBlhfZy>Vp7J{%zYsw{(`FkoSV{$)Z5)KJ)2bx( zQ_dy48n4Zx%mlxI80zYy|A(Q5<ahwT*9i|cpCvjx3Ti@VQhiFg;4%Q)OI^8iO?ylYmP2{`;qnZ%1Y`maJ zS9K2`7yYFcWeG-?y?Y{sEi1~upsr%&Jl(wV{T4D1O(g}^l3LCa{F5n+FcY^^QbwoC zwE3J)x-&7E=&K;*@W1(|KtB%`Z2)JZw)}C>@GBWYHaOJ2xpLk z*%{P{t&!rFf;IY^Kn*^5D?+?gA(3;H@eACRwFUqTh|^G{S$p5O6p{b+3#6d@%^2x@ zh3BYC^!0d8Bh&BO&{}A{lWkPq&iL^N`&0gXfCRhAYMDjgzm6j)cFL9pllvxyUyOF7uE0%}D5c@*%Q>nx5vRU2hYTqKg@)<0MjppCLkkD79 zXx!uam6j2bkb6=^Ji+aZ_SwV&HMzzEx8BDNh>em|`@xJwz+hmBqAbXZEI%+xn<*9p zqTN2i(EzYq0qZ-ZQd@H2tH`;l-9CWBadpqyWN}(MO(xqB8?ncu=nLjQqp1n0{ED@` z3ReMsXCkx~$;XSP`PA0P#x|lJ*Wa+r`M8Ftt>nIitpARVH@Wngz>D}7SxcBk+GPDK zl}NZ+RWQ_5)UJ`FT!5jEba&NIJ>e_w?^?T$^37+BWvMZ!%2OiOi(bKrfHh^I0xLdU z+R*Raw4XAdEm4_b7LCh)6yN;k^?PgcTcoJ@{*pvXS`6}-rRhXMMicN(ocRk9p&*Va zv_1i`WW~44VgHIVCHXN<1RPH`DZb2WR}-i+P@s{%+Oe&bkQV7j+lCzQK!iBOikbg~ zO{Xb&V;69KBjd`zLPg>Q_0Vgh-|Sf#;<$Ayt5|x<k-zmwxU9C|XVqoM;`=~2k|;5% zUq)dqd?_qsY-Vsb)ZJG^z>Lr4G*?o8(43FQk+!nMo8;n*!RJ5=^6hDS^63LI)NZ|{ zHySC8z!-^NAk&4^K(xA@8Ta7hJ2Fquj}3k5G2Osr7|z}4ZChf^3WMp=4}lwgIs=7D-~a{$gTGYs zn@abxTMTP_tNlSn$Z?a}8e$9h$wZ#g%(+B+BOk$~AWNq(tQgg|q3Eo$p~Wf%v3e0` zowMb)!#Z*x@$({K0XF)bBM)>tsaZi zR{RO!o|Pw~B%Qloh#Op?%mVPfw1)J981(d@PGBAvde_Pq-Lb z%B{|eyndZ~^S8~`!))6Rw#-xhkP3*U@u92Zkftfm?$>6LMsh9&B1@zRak09eOk51Y zTHAC2>in4NFFsM>qhNfX|m>bF1$H0p}5bZ`_t{FDQ3 z^20?P<*^x8UcJVy#%@5y7q;k}&WIZ0!q>f$$xspqFLl=IN#xDgUb|-$JhU)LBUn0U z+2Gil!08G%>E$D|UMF#8-x*RqHU+B zoAL9lNK9a`SemOKN&Dv{XTTrFCVFzqAhpkCGkhy zGJ;M7&nFnr&H(BJSpI=9Q;!Mo%kGs9TpZ5gz~-j$0E5q5-!2YC``{(l)QoCZE+rbO zX`Tly+~}D-ka(6qP|>%d>Snd|J%YdFb8Q}$mGIJkn?EndOqNkT!E~&9Wf#8ZdLtw3 zYzH#Gh*4Phh{^4&YBlm-V;t9%eYl^fvj90En=dgGC0J#!}0c^r`#+wQUC0g z$|t76z*a(5^r8_PwBmzF5@Nv{-`bS#a}Nx~SLe_n$Tb__%Yo4Ch*J^JK_t7&`J0q{&# zo8G3=n|C4HHBsODL>|GKwR7e|QDshDtjtJ1pH>1$DH@v0apCvx3B^dbY8=$QLK-sYWOd+Aj`1|LU!6mzGs8*oxM4k4{<&{H zf3aC=IzVY8%NR(CL_=NhV&I+ z4&4gXzTBRTRxtGZD{Ut-pBXx>8czsyiWui@Lnf7)2s4nj2;ZetYx2aO_5s6hOi%6sh3j zoZT)-pN~6jGc=3m!n!5|EnUr2Og=N}Z+9m~W1h!_e@90-w?oxz2V)gtg`OcR2|OPL z>pea4yl16kQ<&fsV&(j3!aC(|}=;8M_-f#d3{6~n(F;`8cP1{msui7KFn zWFWvT2+1kumunqWx!gP(BDJ;FM8jy&7_M+a%N9v1l_7YM3atW0Qo`NwF8c0%JgTY_ z_QmYrb7fH4^D)A+Xsc;0x56M%_=bHCmaIAKkNv+IJL{;pw&l;?5D4z>?ykXI8x8L6 z?lkV&Xh?t%AP^+M-Q6u{g1fuB&*9$p=FOY+o3-Z8UTnJ0-e*hgs&9SDdg=L1O&xJZ zSxYjn-@S6l2vgURM3zKe(Kfnv7Di%j1svZuF4}uWqw+pjRHLa5la{u6pBeZ=ky%XX#<65;?5QtOtR;R}(;^GcVxBe((?ohfxqQuN z1#w-F)RAD-<4ok^UKtaY0aitH%z?YqVofTEv>a>-Zr#%suGY#KOyjyjo-L{p3@$ZCAO&S_2jqd$z?MTKE&+-~2HYY}UB~>dI8v@TQ zpuSXJi8Q#a4o6M-yUVABf8Uvo2BIbYTsuh5H82?hPQi{N+=J`@P$42Jw{I zh$c~)NARf0$AHBNtgN{2?`^8*;E~2&F}2kI;3LR4RTELngdTduh!j1vXxiis{RP2_ zb~6rvJH^lxQyswVF6xD+an0K$~G3@>Y>c};pr_u7DGf(Oi(-iish>XPgfim#uWSdisN zFkT;?AAgn)fmknM_56(s9JhW`6+X5Ok@REL4JLbZhGkM;`7h0|ksmnCj^+IKn1>l7 z9t)|vXgK6?Q4^;9KiXeHyEqUbV3`l}*)#CtKUdMnr}_I0{HK*!+|wmBR-+oV>j@9WW+V`scehxLMZ-b_3ZR1u%{uGu*41 z9TdDOXC^v7v&~C-Nge!T5mx+rVE&tb0?#{Hi;d9Ob>+nNVd%5{zQGHxks_d-a&-v_ zn*5G&D3BEWWE0LNptC5G$M};Jj~{u2a=ud5If|!e<1C5v_Hj{$?{%~hF?eZt%0WGZ z7dK>lGhrIVb*A&{xvS81Aqio&6#CRlq0wBbbvarIzOUD!0T?*m1_uGlW7b}^o7kRD zCd8+QVaKzzjPSIF$mK_pP=v3(_&Cr-)#+KxZ#>~L()oyH$W7rVgn|gfS57ccmtJvpMzeT`zT>* z9t}$QiTiTF^VJs)FmgbFh;_pr9H|nyl9#LUP7)2$hbTJ9$I_yyBiqluGW5JI->E7b zto5KInftvB@cD+w8JnKr-P2#6KIF(IJ^;2=FU(UmA=c&V4p8(tB^mWcc7I?NLBiO# zmcClQ>mKOKgbz6q)x31Q(5BTo6;I9xDfV-a^3>zr?L5D0YYZePreLg&HitJvFrL#i zVPN@y0Ccn>8JX611fM5s=|G&7oJ-LpOM2quo}!Zpql<&UtVB;fyP?kc%+zc&xiN z^+t?on4X~&#YFeUvd4r~)HaKRvgvj#+_9$_=)-1b%dR3XhrISzv~}hqjCB(hmd>?U z)il@Y)4Rt~YaVMf5+aPTY2HY$*|QC&7vt5q?ulVE{j0(dogiv7>dhSXbw^ZxQTVgE z@ok6GJ>pgucPEUth4k>?4It|}nphR43?LCm zc^$5aa|=duf|c?hAq{KEY^$06>pwANU8ouX{1hA@4JbxHUOEmW4VMatZ z0hP@A;I%snhKZo!Wc&NcCOXcCDg(WK%O&9yO2H6P3s2tm$8G@4?{XLHcY3N(w$&}u zx6#M4c4#;`3p~YO&|Yz~i9EfzgSaf)IvC|O0th4%r&%i#wtFyZMc7t zl=Od7-`D@9^ETu7Q~!)?kSfo)oRKK1YG1Ry_hp!^#Fw!r%(wgpODF*p?^#L>xPLHK z+#EM$ZKI#2Vt~)m7vIv!8$bC$Fk16Nrp{$o3ae54-tQlj?CgLqK-qIo$FKg0xrNP< zM;!qfa<2sR{*AZ`3mOUGUt^yW<;UzjjF*dA%Ouwo`_;DKCfEQ&lvnKmhMWmh6ou3KiO>qk?@e0kc_5N0?x#p%tcSYp{pK27MU(^`^$SA-!cX+|1abE^M%yCwpJ-)} z2wi-AN{%brx9MSAk^0-DFcvnXA;4MQRx+5UX??ajZWD!SHv;Sv3WP^KOV6s|X~5?4 zFV<8cXTJEm)|Bn7S0^kSxySYGl>>W4v9F;!M$79Kv9GWnJm3#rN(1tj)|Qo%oQ6!3 zR*kDtn+)Ig3+VLJzqWyjSO3gYx(lsPhXK4ji)qu0ArWZ9Q$zSKO@Q0$-#Dv%`<)32 zEW)SUJlOb8It%eZ(hSDbP z^x$rpGLcT=!wu=Ej&k}xOeK%&jeRaW=4W0@%NFu!_M`vqZlbP>GnBBtGZdgv#nmxAF`I8O13zv&nNs9r{S%rk#Et{S zFR<8k>N@)KOJ+N?1OqYKS3X!_t(ZaXc)rHZgF4O9j#f-Pv7E%l`P7e^7NRdHX6P1h zWxCoQBD7=eMc@${W2F2buMaY9Zd#VcMX~JRD>-Ie6Z^Z-(a6^I3sJ7XVw8%av&M!7 zV-4V^Czv_7f&D)hSr4jAxDdQpki&}6DuhEPNI#Z|ieUcvJru$VU9Cb$Lh?qE*5MN5 zfN1>$d==Qwc9l{W1(~e+=@o|0#bzy}=4N+yzLh5I&yRtLsT0Qa2L=N!VX_fbq6|4$ z@H;-@$MjPR?^T4~XxAP31M169s_3f8)s4O2^9u{O-*(zP0z^uDpFkwbf?$g}e&>_h zwgw+rD;ygsI~JSI@fKK@ohgVe}%lf->U#6jjEgir+6l zp32xmrP<6i&5IF5*5-&Z=|}e*G#ZRHIWW*m+&O?|(80=JF_KI(m#O)9@@?jyQ#NNj z12Xnr2-v#d$@ALskblk@ZiA>gdyZ>5*V$CrrGSYZ9QU7v1x(&TER~UGBmXOt+<5J(#`4Ayqa2?#RmVIA8qjC`DEJB!iH-HiA ze6vubW&GY#bYNsme2xmouyy`tjyVCHZhY8ULF(G#B?42wQndgOWNhYf>U}tHb9wpH z+0ddazsOgBIQ{H~xAzB`9C!5$y32|=-1|fhugtgQJUlu;31|7uJ)>igN3Y&0lP3;! z-WlhXN5JlqGZIx&5w!bx=Z}@|a@k@v8#ThroQsJGJ<`|+GQGUhq zQ^~;&!3?XV*jZs7v(#X7_lL`6D&Ar3Q;JW-ETM}T2)-a32-@T=^>Z~LiUd|)J)#wQ zc0~`g*rK4dpjyPxhB*hHPdy%d>-{6L;zq%z;)&urxYtQe;*S%;hdR2bPKVJ1oCl+n zE8^B}0+(+iyHXnPT3X>!+Koa@2RO&CnsLv^@6FXo;tjsNEjg{t4n{4<;z+MhFu+XWyTc zch6Xc-$+oFC_ql~Owj(*PH#F-wmFS|ymJT>lJ+b4;yP#i(+Hr_L;#K4Iylg@1wPZ` zG*(Q;VlCd!`8b1+#iw>KqLv&Z1q=8B0(^+vh0{(XI2@keU0P#mnK73%0ZBYrJ9E$< z?=eAb-`;k=4U||aiC79?A++G65w(6|5~p4HaOVYL>RpYMD*MWz0k^rvUb{dhkdR}3 z@1Uaeh#&DLv-1aYq|4EVu*ECE%}Ly#9r91+L01lJIJlpbC^E}rtG^if@p0G{tLLU4R;@|2NS2y6 zV}O;X^B1DZfJGD>qEu9YBP<({C!qog32MfA38=N+BiHEL zCdx}iTLa0-;yh>$iy^Gw9dDP&N~kPgT>M2VNEd!V;r8c0(qXl6>sMHOs*>eC926hC z3=khIUx{uH?ZsVFxD4vAvuJ z-3p%1`YUzT-AQydJnD4@xJDX3AAQJi_qZw_ZadS-Iqe<{sjjJOpBZ`0MZY`u46fJH z?rG<{PzzrlbHZwvJii&ZuvaQRU1@j1Js;xG4g)WIDG^`mdmmt03R$T2PG)nt#~>g3 zRT*_VH^4`!;BzLhc9VR3@|P7S{L4c?=IK)eW>?k*ewQ!j3tHQ<+GbNZ57*gGSUXD);>MB6+%$WJFZ;i-GM{qyaXR~Vg;8Psr#^c*@i#*@vwC;QF z-d}mFc#(9fLVe*!;RqD+m5qXRQ9{wJ#TYv8R$ssITDms>nO@%=R(E_NdcTJ`t3wb% zC{r)95ut@ms?CZE>7w2?Yu!QCUhH+GBIM#$3Pl61FF-wgt5qS|@_Ph==K2!=d^@ z^z*QqBkTM7-b=Z)9$1#v9L!SDraT*&tXXZ#!FNn4`Zxo-Jn^^+!WF zlH7f`4ihw?`jfWT;(zFwicbh__)p~?%e zmp*VHgK*U1I>gK5AuZ;dI1YcKpb^pHGtJ;JxFz5i~h>bxtdd}`+ zg>^59UnM&stB?=3G?&2p8E;4(v9sAXl@=Z|_U)r#fPN{MA({;r>bNQC+#DFv?S5=3 zxD_;mel~=G7g3#jw!aPMZOcbtXb9u~@_d{odDw|}oDORslEMF2P@=wp2QB$#IyPHqCU9T(CzS{wn$Xcv8$m0vYI3O-Rd8^e9 zul#5ccrS1!1fb@UT8>J`ucIO?u1)nO~8vmt4SAmxSbyMwIRuwt)Euy6ec&=fDI$MEx%hAaNacr`>n)K^OKP ziw%;D<-#qt^nc2yw##O znsYYg;6u{6AW2N6#oIfbwkQ`DanEZu82t$I+v!_ZQVUd{QG2B0d0>yehPUiD&ON;} z%=(4|Kfai?U;Kz3sc()23^G;`lZ~I(_y&AhnXaRJs(H!HFVYpYJ-`3*_%mb8BoXsP z=7r}9@0wXyc*Z_F)YguhAxoJILlN&%O*pu&6)MGHMPgOEg~l{QB_jHO#a~VVnzcJc zKPFL58IqZF-b%jL6vwvr4FoMu({PAfaVU(q^a(X5rpU)qxd6kL2_K>_GE;NwKOi84 z=+JEKSv~;9OS!VkZae10lQz=4YQho^(RR1SyLP;Tb$Y2FF=7#j9j1-#PHbp$;Q0lE zpH}b8$YU|G@``%a3OG$XdM_d%&rw7q`ZbrnJanh4am`H6U;`lLYQeucZ(!UTtAb zu$eTZ8_H^kq_XvG%bXjg?OcW>(by6Mv+-edm6+Ecanm+cX9@cqoDJO`!{Kekv`X`J zUudNMp?7a&y$kSd02yOIHgZ=#^5Xfiyf`EJtX`{U!O7?HE&Va|TEbKHsp5K|N!4fL z&Mf>}lO5kg$dZ$|?1l?qlI%OCicK}$7uWeqRiu;s-pwy1humgyUf`7o1c=Q(t%w(# zpM21tX1*{sdviy1Tu<4f!NW zl3s~NjM<__R-Ms{@h-DY;Sr_D3$2A*s2tkMXrg`|nj&f;xMm%*L}jNDLc*HIPK8$J zE)RsHYhn-8y?u`;xoOdoiVLQsB2^4@Lq=jhvO0?2CI?2fCeHgiv*5Gm4F2)>dA2H% zGP2TH^(>E9_JqpiPh0uZ>PW#L6vkH=$Cvc|>O2W*72O#PWF%O^swJ@Y)>kBNW8&Z| zMaGEnE_3NR1sbl7N7EZfchsDoVc)CXg{Po)0uMvhQHQ^6;Tb23Q~;47{N#YyfGzlz zveQ!hEEYa0t%<^zXL^~6n1X?0CnOa?ar%u>3gXVXS9XgH!6yIMweZj!ALI5>TD;!9 za3q@p7Q`@3J-t~tzbGG~q)2UJ3R~-O52&n>a$EC8&DOh*h(Ldy-Awh0DhP_A zvD%4@roB&!N=}hdMv16sR3mTsP!21}Gk!<#2+^ALY&J5j2hs)KMzm8WrkB4xQ)`(5 z%pgwHRR%Qdssvo4V5nEXQA?>zNWQYQH2CwYxAcGwCvW~~MTjH}NUn6ut znDRe`Gte-c^Co=A)1BM#hJ5G?u^7%gWNt@9(O>-$L6io9;GK@t39b@s2y`T!zuMya zTV%Q|`yxcrS8oF4X(MmlnZu#ZQ{Zg7=&AHpBWYIE~FGf4Xt z*^E0Z2Dao^EsrpO*8#81nA5Y|50S@!wx|9evMcY#WqmwAvYs$LO`BWE9{iSCi2+Vd z_SG%TMqT(7oqd%PiP(j5UAHEpRZIA@?e5Q_V z-gcRBusb%v;>EI_E`t;^r`k;%rQb!B>ZU~B2C3H?JI%jKJH-9Q(ac`y#&f8+vipy7 zBN1`+gDpt9ZKQABQAtL)Dz5Ee9R#xl2#UAQ;EVpCkK~wz`uw7n=`7aE_D+WL5-pST z`!R8vpqARRUlH{!c;$ofs~aFadlOSRpOvDsn%VVMPwX z#Ctdb5Pe7~U zs_bHaYV3vYQV-;T0Cgh9VE@6lo<23m+NCzLiom&QeK#5(itWhV)({_7ekuA*(32Tz zBHAks0^pBXopvviPBd;%{=*-;-5Gah;;kBOU7AJ_ z5Tla-b>jEP+?bH!@nLb`gWZHn%Id5JEL>+a>M)y^Aa=fjDjOU{PgiAEQy^yPy02kB zWCBB?Gf#qsB=CDawxUXtLo4dmS)fZR7%KVVMTpd5ocX+D)ukoM1CgvkigGr1T}8XF z!a36h4T$G4~iX& z(gG3QA6qs*7afC^v;oKC3ocU$OzocJk^%lII9CU_Kj#;1Bt>-AQ_^_rk>Af3?LuZb z=`HKw{}DMmtoL`AFGIL;(KCby?M5@iFI2EO;$T)ARH;Z_NFr!vOw_A+4$LTygnO$r+qcSW}Z&i|EcL&sR#z! zBFb&52Z4qlU-QuAeXeOtParqsTgZctQN&SyTL)`@s+s7^%-OF28#>S(g2>6syCUtg z=Yaq4$%v#E1yL!TvLMf)eGozh&(t1oxG&yE;}mr*E4>@`>v~ap_e4zzV1&0$_jn)p z$B03c+{vSXO8_^nC%o1?X%($7q1QijB@g)NglOOQN9ju;&F@-57JL^I;_p@HxX&8u z#)=!NSKRP0R$%u3N)uVxKv3$yn8&JG|4mmZpIc!L2RR7CY!1NjwYQY4BY}2cYhx9r zaFDbh5xl{s%Tv962C!@7wHDS9Z2jv=IqSFuI?Kpsdi_#d54UH_hB*SR{wes_ms*>1Ai2VRFw8@*m#-d)L0G zp~>vr+=y~^Ll2$fKt=^`I(eOiU~?pxuwS5Ze^wa~+g8guCl;{+0ytQu(}ZP~*Z=B$ zBc%?T!N)~NTOSzQu=abLB9?`vMcrNr5yi?;MlO*$k;9H1#nK+;)tLrw&QP;^#c>}G zh9{0_3)3tmSV*)KXL*g6z4~}ZzHrLSR+zlSVbWercbZxGPg2h-UxJ_|qlFl_N#fD? z;$vZ>j>D40@919$wX+?||HS$%KVYPwvxd)<$v5C(I`tJ-hj^7XC;aiR?=BHif(dTA z6HWbbcL zWc)6cwIl%=nH(htf+F84%n32(=RQB8f!Y1^UzYBXgsuGNh6EcVg$czMw*gsh9ol|4 zA!%VPdq?4sE;=U#c$4%IQY{jt$e}DX^mccoAhNbKCwOS!qRL?r({a&)1_v(wK<8(b z*Ri0$B;GPI$QOox@D1&`c^##5zoNq0*=H}1F){Kxo^Zfe$`CWBwY zH9K&;(*4)Xd+1;1q_Nliacg}u=UX61)wzpMA{weJCqkDXbx1<2x9}^}FqP&>}mIByKju+k)IiU6U+Aqn< zo1|a5qHXk;0%}^-IOJ{Q)QNz!F)|J9CeNxn!dC3Qp z(z&MGsy3IJZS-&bkn@@IcJR%tx6{umGLow)l@%hmlXz4nB5pyn7x`H;2N$>|k_pqh zD{|>QMI3t`vgJQ0l1riU>nH$v+B{iUjJy=}G}MgTdb;Fpy66v%iWI#ZzMT*uiM^|D zB2~;|X&*Kon4*zwKI}O$wYE>1frGpM=tQ3GXJ(>ddmdYXPBiSTAt2v=lrw)8w3jUI z&nqf}a}W9^3uc%itl{fB1vJMue3~s9^+y_JywS%1Fd|X+!R97XmQ}bHAetqRhD9$O zqdPT<$V~R_JrKX_fZ(|CC$4Rguv1+vrYU|c+{FV*>?&CcRg8wp88d6b- zEf17uHyR=sip-9@C-|4?#aNz=n~VP1e!Gn9CK~M~zl$!x=CcByKk5M-;9G~A1h^@i z>={jBOD1ZSJ@%b88K`H$Tah!bBden#UCEkmK;2->OdpJJYv}jH_o~d+IN7Fjwn$yj zk)N+{OKZH70*G@sg|_Hc{+b$^=KXtbT|_f@^g%~{6u z6^Vxbue+1#mLYy@+&w1)KrECuCi&X@N{%Eh+#T%pJnF^Tpyo;s1im&Y4wX9bJj5J_BLKR|NxZ&Ifw-pYl+X=OE)+fe$LS z5xv_TqU_SLY=A*6%M9?%aE{WYDsh~rg5sm#a?X#a8i3!(!%+vpI1m`GDd8`M;@<7|eLbXY0rXLH{r(@H&Y&f*tMF}sj; zt<{&AsmTp~@>(b;PoImlmG$Y7s>jV-0P9Ygyto2MtzAEuKG&B_VX*_Y>g|szV>f5; zSql0;Hx_B7FKMC}KTL`kgKx!HwLxU+qnXf6zd}^kj&A~RU1MUIFd8(E6mXV}D!$Y{ zCYKL4&d8Y2?KoBvhEK@^;_xmHQnX3E7M`IJmBn>6T)vyOWktIBA5t{mY>M~NvvLDx zGa3e2p10BV;{_>0fh-WOb39b~YWI-?_z?9Cqk-eG$&a>fQ~&n6Wb;HU^UGoX*Ol`A zuYt#XgWuD4K;(JsfRQ%fJxU@FT0TSC6{0ctlRjKq9)bO+ScBIZsrxM^tntq#ymFRV z__FQ0FK{;Qz&Hd^&W@Dz>P=IVNGIQ-<>h6HMdvq3z<54BUig?x*LOap)=P;i(3Kr7 zVcgME{cIuz;~9|5TD{7YLBv)M+0j&!ddx_>c?!pkIOZA%fp5Xk;uW^O*(X2 zhXPs1U(4EbEHE|DYZPhn9jRO0Ev69yN%lsoG29tJLd{Cw<@z;uCkkNJv3Qu4GUJ8- zFC`&n!gsmf;L9jJE!d(41p?i|pr(EM>MOF)CGzZ_zt%4Dwe^XV-VokwZOpNvtk*il zlhNnL3Mpe8LY1d|?~Va(`X5K#W0+q8+jw(Z7cX98%1CdOWW zkjHe+?tR;;f6{O~-fcMDvI`Mae`|zt(Y@B;+%_H38%F!>MMi>;wGkg3y)$s^ zf4hQiJn?k{eWK%JV!f0;ty%DGj?W97x;EK$7ue-TX@+63Gxg^a!()I}!djV8#ihl) zTxj-`n`YWUjTUW8|GDi52wul=3gBC;i0n5pqee$Gjd9Z*yE8vuTsXG84BP_L@tUXR z(?2wf0scPFdGS48%pT4*TPVTDX9sLNoll&c3Dj$4#W76AYEOH3!-_Sq%YfklXmB&W&hg)a;(!Af;T<^6RyRgQ67hR-hcPcwqAm zMB>d)Bol7Q*RSVMQOIC)0fhn)zD!t`mkeUTXHrCcmpZ&PLjAZ@ulaDNwkNgQ^>0i# z=+J5bDb5Fh+X!l@nK*%=3uTI5!z;^KFjPx+igM-^V7fM({1{s*4>#KnxWhy=Hfrbj z*0Np}m|KKSemYYu(eY*;YspT5qr;TGCkN$5IYysG1_V71S&^eAU{WHV@O_Q*p7oiJ zFflOIfgYEUHqp+&p+(6M_kgy&K5Ytw^Cgl?4exxgW&~CtJ@%NdaJFMG<@e}W)lSF|yJ2)~L}K2!EDMC^Bc(>exFoXV3=m0#T%!xt(Poy` zn4gl54<62@9+PZg6JjI4_ILV&zVugi`zEE8cgnxN{A=W+Qi~!Q^e0_LB(gJh;9+*L zxAQ6Z`a?YIyw*ilN(*aR9u^Eg&3fA}`r=G}CK2*=7w7a9wq#yOFv4}}+|e64oi|I( z)e0wX1!P&ldO(y*MtD--+WRGt2Obkdw)oR02|%sxBWD0;!NM zB?-YiPOc1cua)WfE@83T6Vcy>rapwreUTKn!-^@>q)Oy!iu(pj<$iqe4bjh0A>*P! zs!U>-k+T&4TU_z~fH}&GPIpmk`ssvQq}^{JEC%SXh7r~<5u{yb4us5YcR=c60k*x{ z#aDqsoD@_QAd23SUGXPG`Q;YYvY4xf{q*W&k*Gd)Z%`M2Wa$_EI?SrO=|sI=TZ(KZ zr_o1lm1ZzId4c650Ag`4BsNlm*7#HK93pfXfc^$ht{N>J1_#T{^WN&+p;gCqSAQdw zbWB3V5rB;Yy(!g=MwpB$2I36P76IK9$WIhOgeR5d*#NuWt%dVEkgB5zzhq1gh* z2NwnXoD|q|dgXTW3?4Y0+Vrb1Lz}5q@6io_0uyw>;D5_BWpUM?pQjywCIuraK_m|A zKpBNqC-^Wx{s1UT{)dWlZ^T)WqaihO*9jL>uc8?$Q7(9!VAu$SP*9HPzqxJ8w*A2y zR+aWQazq1M_k*?Lol*q!gqRqtORlhfv4AD8`5j?$@)W2XV`fo>?gBy~eR0Cqw9=bx zrTTk7^%F~}P|KfBQTYvpp;`5pu;2Q4b;T{!YFC{@6a7Jkw|)I(ZcN_CT6<0etcivu z8K&n_4ptyhA-(ehRAU}d1~M#1ohaI)*)bt78i@ot2p1J}g-A~kH{xnXjT^_k(vSnN z;8a;d4Z;ZDt-ziyulLQb#I{%No@(x&Pt|l0D5>?R!Apacy)o5%IAJnU;(NW!QToSW z6JLTa*VU$2s8jsvw)t=(;qj+R7?~~nNG{hjcFMu<=(;tlj z`hWN;$Xa6{cz9P;=(rShgq*!s?m`UB)fvB7emRh5_jU*u)APt>jppI%m5^A85HB-F zH`bZ-n*G)mUdzDwRZ%6q>vWpf2qN^uZK3I#S76w07`YOSsi2e%z*hlf1eiK3A_yTLbKjSN`$d5y&?ISQO%IGMC zBLJdaUluFZ@>yyF>QE#~_f*68gzP!J#BjO@v0vl}n~L->Ukr#X-YH@%-<^!$5f%J( z;%vOrX2xr9h;_5goERy+as7cf$7%K2eE#vOB+c|aj!ha+1aU3X>lt;S&cO_QQpFt( zf~G1|4D&-#@VhSe-Ft7@I%1HRwYb_PyX;p~lwzFHJ_0xB0{v-1)YyZlP?b;_mpk|ETD@?KkJKxT)Ed^I{h)2w@6%o_)a?8M>h zaT%bnV5HC0BHS72r4>z>x^l6!T0Rod$tV?+Q~yV5-w)G8LL?+aJI91{wYw|g;iM@f zqM$GCC5G>6hi>|}2lD|0JMgey-e@7)MF@SpSv8k_qsg=!D^gXP=5m6zJM0#w5PeWa z8uwvM3Va6GA0G1q$7G#;<9{QDp#iTL;bNDGAi3>WT%49b1v^%?`uO-mXrxZUYCc@8WsXlT=GV< zk=N?5)It*W1dbZ3Q#VkF@b4!y6#u^NJhr(8KCTl3Ul!yvSjaN&3VF}HJM(@0a%2f9 z>`)xQ(et|J11L%x$vj Date: Wed, 9 Jun 2021 17:47:24 +0800 Subject: [PATCH 14/26] asdf --- examples/benchmarks/TCTS/TCTS.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index e8cbc8005..aa707b26c 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -9,10 +9,12 @@ Sequence learning has attracted much research attention from the machine learnin ### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. + + ![The optimization workflow of one episode.](workflow.png) -At step $$s$$, with training data $$(x_s,y_s)$$, the scheduler $$\varphi(\cdots;\omega)$$ chooses a suitable task $$T_{i_s}$$ (green solid lines) to update the model $$f(\cdots;\theta)$$ (blue solid lines). After $$S$$ steps, we evaluate the model $$f$$ on the validation set $$\Ddev$$ and update the scheduler $$\varphi$$ (green dashed lines). +At step $s$, with training data $(x_s,y_s)$, the scheduler $\varphi(\cdots;\omega)$ chooses a suitable task $T_{i_s}$ (green solid lines) to update the model $f(\cdots;\theta)$ (blue solid lines). After $S$ steps, we evaluate the model $f$ on the validation set $\Ddev$ and update the scheduler $\varphi$ (green dashed lines). ### DataSet * We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. @@ -20,31 +22,31 @@ At step $$s$$, with training data $$(x_s,y_s)$$, the scheduler $$\varphi(\cdots; ### Experiments #### Task Description -* The main tasks $$T_k$$ ($$task_k$$ in Figure1) refers to forecasting return of stock $$i$$ as following, +* The main tasks $T_k$ ($task_k$ in Figure1) refers to forecasting return of stock $i$ as following, ```math -$$ +$ r_{i}^k = \frac{\price_i^{t+k}}{\price_i^{t+k-1}} - 1 -$$ +$ ``` -* Temporally correlated task sets $$\domT_k = {T_1, T_2, ... , T_k}$$, in this paper, $$\domT_3$$, $$\domT_5$$ and $$\domT_10$$ are used. +* Temporally correlated task sets $\domT_k = {T_1, T_2, ... , T_k}$, in this paper, $\domT_3$, $\domT_5$ and $\domT_10$ are used. #### Baselines * GRU/MLP/LightGBM (LGB)/Graph Attention Networks (GAT) * Multi-task learning (MTL): In multi-task learning, multiple tasks are jointly trained and mutually boosted. Each task is treated equally, while in our setting, we focus on the main task. * Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task T_1, then T_2, and gradually move to the last one. #### Result -| Methods | $$T_1$$ | $$T_2$$ | $$T_3$$ | +| Methods | $T_1$ | $T_2$ | $T_3$ | | :----: | :----: | :----: | :----: | | GRU | 0.049 / 1.903 | 0.018 / 1.972 | 0.014 / 1.989 | | MLP | 0.023 / 1.961 | 0.022 / 1.962 | 0.015 / 1.978 | | LGB | 0.038 / 1.883 | 0.023 / 1.952 | 0.007 / 1.987 | | GAT | 0.052 / 1.898 | 0.024 / 1.954 | 0.015 / 1.973 | -| MTL($$\domT_3$$) | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | -| CL($$\domT_3$$) | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | -| Ours($$\domT_3$$) | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | -| MTL($$\domT_5$$) | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | -| CL($$\domT_5$$) | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | -| Ours($$\domT_5$$) | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | -| MTL($$\domT_{10}$$) | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | -| CL($$\domT_{10}$$) | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | -| Ours($$\domT_{10}$$) | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file +| MTL($\domT_3$) | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | +| CL($\domT_3$) | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | +| Ours($\domT_3$) | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | +| MTL($\domT_5$) | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | +| CL($\domT_5$) | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | +| Ours($\domT_5$) | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | +| MTL($\domT_{10}$) | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | +| CL($\domT_{10}$) | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | +| Ours($\domT_{10}$) | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file From 5a3dde93a893f91c69e54f495a67cebb3f2f36ab Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:15:06 +0800 Subject: [PATCH 15/26] update --- examples/benchmarks/TCTS/TCTS.md | 40 +++++++++++++++----------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index aa707b26c..b0389dc9b 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -4,17 +4,17 @@ We provide the code for reproducing the stock trend forecasting experiments in [ ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -![Temporally Correlated Tasks.](task_description.png) +
![Temporally Correlated Tasks.](task_description.png)
### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. -![The optimization workflow of one episode.](workflow.png) +
![The optimization workflow of one episode.](workflow.png)
-At step $s$, with training data $(x_s,y_s)$, the scheduler $\varphi(\cdots;\omega)$ chooses a suitable task $T_{i_s}$ (green solid lines) to update the model $f(\cdots;\theta)$ (blue solid lines). After $S$ steps, we evaluate the model $f$ on the validation set $\Ddev$ and update the scheduler $\varphi$ (green dashed lines). +At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). ### DataSet * We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. @@ -22,31 +22,29 @@ At step $s$, with training data $(x_s,y_s)$, the scheduler $\varphi(\cdots;\omeg ### Experiments #### Task Description -* The main tasks $T_k$ ($task_k$ in Figure1) refers to forecasting return of stock $i$ as following, +* The main tasks ( in Figure1) refers to forecasting return of stock as following, +
+ +
-```math -$ -r_{i}^k = \frac{\price_i^{t+k}}{\price_i^{t+k-1}} - 1 -$ -``` -* Temporally correlated task sets $\domT_k = {T_1, T_2, ... , T_k}$, in this paper, $\domT_3$, $\domT_5$ and $\domT_10$ are used. +* Temporally correlated task sets , in this paper, , and are used. #### Baselines * GRU/MLP/LightGBM (LGB)/Graph Attention Networks (GAT) * Multi-task learning (MTL): In multi-task learning, multiple tasks are jointly trained and mutually boosted. Each task is treated equally, while in our setting, we focus on the main task. -* Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task T_1, then T_2, and gradually move to the last one. +* Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task , then , and gradually move to the last one. #### Result -| Methods | $T_1$ | $T_2$ | $T_3$ | +| Methods | | | | | :----: | :----: | :----: | :----: | | GRU | 0.049 / 1.903 | 0.018 / 1.972 | 0.014 / 1.989 | | MLP | 0.023 / 1.961 | 0.022 / 1.962 | 0.015 / 1.978 | | LGB | 0.038 / 1.883 | 0.023 / 1.952 | 0.007 / 1.987 | | GAT | 0.052 / 1.898 | 0.024 / 1.954 | 0.015 / 1.973 | -| MTL($\domT_3$) | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | -| CL($\domT_3$) | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | -| Ours($\domT_3$) | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | -| MTL($\domT_5$) | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | -| CL($\domT_5$) | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | -| Ours($\domT_5$) | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | -| MTL($\domT_{10}$) | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | -| CL($\domT_{10}$) | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | -| Ours($\domT_{10}$) | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file +| MTL() | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | +| CL() | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | +| Ours() | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | +| MTL() | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | +| CL() | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | +| Ours() | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | +| MTL() | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | +| CL() | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | +| Ours() | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file From 02d0eedd68e75b437e38c32c3f52022a73a62ccc Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:21:16 +0800 Subject: [PATCH 16/26] update --- examples/benchmarks/TCTS/TCTS.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index b0389dc9b..f500fc155 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -1,17 +1,18 @@ # Temporally Correlated Task Scheduling for Sequence Learning -We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://www.overleaf.com/project/5eb8efb42dcf710001d781d6). +We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_nn.py). ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -
![Temporally Correlated Tasks.](task_description.png)
+ +

![Temporally Correlated Tasks.](task_description.png)

### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. -
![The optimization workflow of one episode.](workflow.png)
+

![The optimization workflow of one episode.](workflow.png)

At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). From 38c7b7303adbb1d5061e5f556870aed24d10e159 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:26:50 +0800 Subject: [PATCH 17/26] dsaf --- examples/benchmarks/TCTS/TCTS.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index f500fc155..d450cfff4 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -5,14 +5,16 @@ We provide the code for reproducing the stock trend forecasting experiments in [ Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -

![Temporally Correlated Tasks.](task_description.png)

+
+ +
### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. -

![The optimization workflow of one episode.](workflow.png)

+
![The optimization workflow of one episode.](workflow.png)
At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). From 4c4e77b11ffe3c6e36f6c3b7c642b5b17efd0329 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:28:31 +0800 Subject: [PATCH 18/26] asdf --- examples/benchmarks/TCTS/TCTS.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index d450cfff4..f3113f56b 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -4,9 +4,12 @@ We provide the code for reproducing the stock trend forecasting experiments in [ ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -
- + +
+ +
+
From bb6c1572cadc111af94e012eae2756aa0980e433 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:29:55 +0800 Subject: [PATCH 19/26] asdf --- examples/benchmarks/TCTS/TCTS.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index f3113f56b..eb644ff4a 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -4,13 +4,14 @@ We provide the code for reproducing the stock trend forecasting experiments in [ ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -
+

-

+

+ +

+ +

-
- -
### Method From 89d53853e5630fa339b3c463f6010d6bce69f953 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:30:42 +0800 Subject: [PATCH 20/26] asdf --- examples/benchmarks/TCTS/TCTS.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index eb644ff4a..eab8199e4 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -8,18 +8,13 @@ Sequence learning has attracted much research attention from the machine learnin

-

- -

- - ### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. - -
![The optimization workflow of one episode.](workflow.png)
- +

+ +

At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). From 073fe4668e1cd970a0b3bd89711012823359b623 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:34:31 +0800 Subject: [PATCH 21/26] asdf --- examples/benchmarks/TCTS/TCTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index eab8199e4..581d34f90 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -5,7 +5,7 @@ We provide the code for reproducing the stock trend forecasting experiments in [ Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other.

- +

@@ -13,7 +13,7 @@ Sequence learning has attracted much research attention from the machine learnin Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2.

- +

At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). From d199256d3489368fc03054987b1f3453ef95a0ba Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:35:14 +0800 Subject: [PATCH 22/26] asdf --- examples/benchmarks/TCTS/TCTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index 581d34f90..66c569734 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -5,7 +5,7 @@ We provide the code for reproducing the stock trend forecasting experiments in [ Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other.

- +

From 65ddca133f3d3ae28fe6051134eb1e850a5290d1 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:36:12 +0800 Subject: [PATCH 23/26] asdf --- examples/benchmarks/TCTS/TCTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index 66c569734..5579c42a8 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -1,5 +1,5 @@ # Temporally Correlated Task Scheduling for Sequence Learning -We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_nn.py). +We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py). ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. From 567e42840c5d1a3bf0565ab016a20ea2bf6f3a6e Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:37:25 +0800 Subject: [PATCH 24/26] asdf --- examples/benchmarks/TCTS/TCTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index 5579c42a8..ee67ffbeb 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -1,5 +1,5 @@ # Temporally Correlated Task Scheduling for Sequence Learning -We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py). +We provide the [code](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py) for reproducing the stock trend forecasting experiments. ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. From 9e45528165cd1bf011b74a595e98ae0a7c0f6313 Mon Sep 17 00:00:00 2001 From: bxdd Date: Mon, 14 Jun 2021 22:31:31 +0800 Subject: [PATCH 25/26] update backtest time range --- examples/nested_decision_execution/workflow.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/nested_decision_execution/workflow.py b/examples/nested_decision_execution/workflow.py index 2286f4f12..910011887 100644 --- a/examples/nested_decision_execution/workflow.py +++ b/examples/nested_decision_execution/workflow.py @@ -19,10 +19,10 @@ class NestedDecisonExecutionWorkflow: benchmark = "SH000300" data_handler_config = { - "start_time": "2010-01-01", + "start_time": "2008-01-01", "end_time": "2021-05-28", - "fit_start_time": "2010-01-01", - "fit_end_time": "2017-12-31", + "fit_start_time": "2008-01-01", + "fit_end_time": "2014-12-31", "instruments": market, } @@ -52,9 +52,9 @@ class NestedDecisonExecutionWorkflow: "kwargs": data_handler_config, }, "segments": { - "train": ("2010-01-01", "2017-12-31"), - "valid": ("2018-01-01", "2019-12-31"), - "test": ("2020-01-01", "2021-05-28"), + "train": ("2007-01-01", "2014-12-31"), + "valid": ("2015-01-01", "2016-12-31"), + "test": ("2020-09-01", "2021-05-28"), }, }, }, From 4ac6e6e246c8f1a542c21ab288f6c8eb77c5180e Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 22 Jun 2021 02:42:09 +0800 Subject: [PATCH 26/26] fix account bug & update indicator_analysis & fix some comments --- .../nested_decision_execution/workflow.py | 47 ++-- qlib/backtest/__init__.py | 8 +- qlib/backtest/account.py | 164 +++++++++--- qlib/backtest/backtest.py | 53 ++-- qlib/backtest/exchange.py | 7 +- qlib/backtest/executor.py | 167 ++++++------ qlib/backtest/report.py | 253 +++++++++++------- qlib/contrib/evaluate.py | 58 +++- qlib/workflow/record_temp.py | 69 +++-- 9 files changed, 524 insertions(+), 302 deletions(-) diff --git a/examples/nested_decision_execution/workflow.py b/examples/nested_decision_execution/workflow.py index 910011887..689602013 100644 --- a/examples/nested_decision_execution/workflow.py +++ b/examples/nested_decision_execution/workflow.py @@ -17,7 +17,6 @@ class NestedDecisonExecutionWorkflow: market = "csi300" benchmark = "SH000300" - data_handler_config = { "start_time": "2008-01-01", "end_time": "2021-05-28", @@ -67,28 +66,19 @@ class NestedDecisonExecutionWorkflow: "kwargs": { "time_per_step": "week", "inner_executor": { - "class": "NestedExecutor", + "class": "SimulatorExecutor", "module_path": "qlib.backtest.executor", "kwargs": { "time_per_step": "day", - "inner_executor": { - "class": "SimulatorExecutor", - "module_path": "qlib.backtest.executor", - "kwargs": { - "time_per_step": "15min", - "generate_report": True, - "verbose": True, - }, + "generate_report": True, + "verbose": True, + "indicator_config": { + "show_indicator": True, }, - "inner_strategy": { - "class": "TWAPStrategy", - "module_path": "qlib.contrib.strategy.rule_strategy", - }, - "show_indicator": True, }, }, "inner_strategy": { - "class": "VAStrategy", + "class": "SBBStrategyEMA", "module_path": "qlib.contrib.strategy.rule_strategy", "kwargs": { "freq": "day", @@ -96,7 +86,10 @@ class NestedDecisonExecutionWorkflow: }, }, "track_data": True, - "show_indicator": True, + "generate_report": True, + "indicator_config": { + "show_indicator": True, + }, }, }, "backtest": { @@ -105,7 +98,7 @@ class NestedDecisonExecutionWorkflow: "account": 100000000, "benchmark": benchmark, "exchange_kwargs": { - "freq": "1min", + "freq": "day", "limit_threshold": 0.095, "deal_price": "close", "open_cost": 0.0005, @@ -124,7 +117,7 @@ class NestedDecisonExecutionWorkflow: GetData().qlib_data( target_dir=provider_uri_1min, interval="1min", region=REG_CN, version="v2", exists_skip=True ) - + provider_uri_day = "/data/csdesign/qlib" provider_uri_map = {"1min": provider_uri_1min, "day": provider_uri_day} client_config = { "calendar_provider": { @@ -179,12 +172,25 @@ class NestedDecisonExecutionWorkflow: }, } self.port_analysis_config["strategy"] = strategy_config + self.port_analysis_config["backtest"]["benchmark"] = D.list_instruments( + instruments=D.instruments(market=self.market), as_list=True + ) with R.start(experiment_name="backtest"): recorder = R.get_recorder() - par = PortAnaRecord(recorder, self.port_analysis_config, "15minute") + par = PortAnaRecord( + recorder, + self.port_analysis_config, + risk_analysis_freq=["week", "day"], + indicator_analysis_freq=["week", "day"], + indicator_analysis_method="value_weighted", + ) par.generate() + # report_normal_df = recorder.load_object("portfolio_analysis/report_normal_1day.pkl") + # from qlib.contrib.report import analysis_position + # analysis_position.report_graph(report_normal_df) + def collect_data(self): self._init_qlib() model = init_instance_by_config(self.task["model"]) @@ -192,6 +198,7 @@ class NestedDecisonExecutionWorkflow: self._train_model(model, dataset) executor_config = self.port_analysis_config["executor"] backtest_config = self.port_analysis_config["backtest"] + backtest_config["benchmark"] = D.list_instruments(instruments=D.instruments(market=self.market), as_list=True) strategy_config = { "class": "TopkDropoutStrategy", "module_path": "qlib.contrib.strategy.model_strategy", diff --git a/qlib/backtest/__init__.py b/qlib/backtest/__init__.py index a3706008a..f8f30f183 100644 --- a/qlib/backtest/__init__.py +++ b/qlib/backtest/__init__.py @@ -116,9 +116,9 @@ def backtest(start_time, end_time, strategy, executor, benchmark="SH000300", acc trade_strategy, trade_executor = get_strategy_executor( start_time, end_time, strategy, executor, benchmark, account, exchange_kwargs ) - report_dict = backtest_loop(start_time, end_time, trade_strategy, trade_executor) + report_dict, indicator_dict = backtest_loop(start_time, end_time, trade_strategy, trade_executor) - return report_dict + return report_dict, indicator_dict def collect_data(start_time, end_time, strategy, executor, benchmark="SH000300", account=1e9, exchange_kwargs={}): @@ -126,6 +126,4 @@ def collect_data(start_time, end_time, strategy, executor, benchmark="SH000300", trade_strategy, trade_executor = get_strategy_executor( start_time, end_time, strategy, executor, benchmark, account, exchange_kwargs ) - report_dict = yield from collect_data_loop(start_time, end_time, trade_strategy, trade_executor) - - return report_dict + yield from collect_data_loop(start_time, end_time, trade_strategy, trade_executor) diff --git a/qlib/backtest/account.py b/qlib/backtest/account.py index 71214036a..85ca57fa5 100644 --- a/qlib/backtest/account.py +++ b/qlib/backtest/account.py @@ -9,7 +9,7 @@ import pandas as pd from .position import Position from .report import Report, Indicator from .order import Order - +from .exchange import Exchange """ rtn & earning in the Account @@ -25,10 +25,42 @@ rtn & earning in the Account while earning is the difference of two position value, so it considers cost, it is the true return rate in the specific accomplishment for rtn, it does not consider cost, in other words, rtn - cost = earning -Now rtn has been removed in the hierarchical backtest implemention. """ +class AccumulatedInfo: + """accumulated trading info, including accumulated return\cost\turnover""" + + def __init__(self): + self.reset() + + def reset(self): + self.rtn = 0 # accumulated return, do not consider cost + self.cost = 0 # accumulated cost + self.to = 0 # accumulated turnover + + def add_return_value(self, value): + self.rtn += value + + def add_cost(self, value): + self.cost += value + + def add_turnover(self, value): + self.to += value + + @property + def get_return(self): + return self.rtn + + @property + def get_cost(self): + return self.cost + + @property + def get_turnover(self): + return self.to + + class Account: def __init__(self, init_cash, freq: str = "day", benchmark_config: dict = {}): self.init_vars(init_cash, freq, benchmark_config) @@ -38,17 +70,13 @@ class Account: # init cash self.init_cash = init_cash self.current = Position(cash=init_cash) + self.accum_info = AccumulatedInfo() self.reset(freq=freq, benchmark_config=benchmark_config, init_report=True) def reset_report(self, freq, benchmark_config): self.report = Report(freq, benchmark_config) self.indicator = Indicator() self.positions = {} - self.rtn = 0 - self.ct = 0 - self.to = 0 - self.val = 0 - self.earning = 0 def reset(self, freq=None, benchmark_config=None, init_report=False): """reset freq and report of account @@ -78,21 +106,22 @@ class Account: def _update_state_from_order(self, order, trade_val, cost, trade_price): # update turnover - self.to += trade_val + self.accum_info.add_turnover(trade_val) # update cost - self.ct += cost - # update return - # update self.rtn from order + self.accum_info.add_cost(cost) + + # update return from order trade_amount = trade_val / trade_price if order.direction == Order.SELL: # 0 for sell # when sell stock, get profit from price change profit = trade_val - self.current.get_stock_price(order.stock_id) * trade_amount - self.rtn += profit # note here do not consider cost + self.accum_info.add_return_value(profit) # note here do not consider cost + elif order.direction == Order.BUY: # 1 for buy # when buy stock, we get return for the rtn computing method - # profit in buy order is to make self.rtn is consistent with self.earning at the end of date + # profit in buy order is to make rtn is consistent with earning at the end of bar profit = self.current.get_stock_price(order.stock_id) * trade_amount - trade_val - self.rtn += profit + self.accum_info.add_return_value(profit) # note here do not consider cost def update_order(self, order, trade_val, cost, trade_price): # if stock is sold out, no stock price information in Position, then we should update account first, then update current position @@ -111,23 +140,12 @@ class Account: self._update_state_from_order(order, trade_val, cost, trade_price) def update_bar_count(self): + """at the end of the trading bar, update holding bar, count of stock""" + # update holding day count self.current.add_count_all(bar=self.freq) - def update_bar_report(self, trade_start_time, trade_end_time, trade_exchange): - """ - trade_start_time: pd.TimeStamp - trade_end_time: pd.TimeStamp - quote: pd.DataFrame (code, date), collumns - when the end of trade date - - update rtn - - update price for each asset - - update value for this account - - update earning (2nd view of return ) - - update holding day, count of stock - - update position hitory - - update report - :return: None - """ + def update_current(self, trade_start_time, trade_end_time, trade_exchange): + """update current to make rtn consistent with earning at the end of bar""" # update price for stock in the position and the profit from changed_price stock_list = self.current.get_stock_list() for code in stock_list: @@ -136,22 +154,28 @@ class Account: continue bar_close = trade_exchange.get_close(code, trade_start_time, trade_end_time) self.current.update_stock_price(stock_id=code, price=bar_close) - # update holding day count - # update value - self.val = self.current.calculate_value() - # update earning + def update_report(self, trade_start_time, trade_end_time): + """update position history, report""" + # calculate earning # account_value - last_account_value # for the first trade date, account_value - init_cash # self.report.is_empty() to judge is_first_trade_date - # get last_account_value, now_account_value, now_stock_value + # get last_account_value, last_total_cost, last_total_turnover if self.report.is_empty(): last_account_value = self.init_cash + last_total_cost = 0 + last_total_turnover = 0 else: last_account_value = self.report.get_latest_account_value() + last_total_cost = self.report.get_latest_total_cost() + last_total_turnover = self.report.get_latest_total_turnover() + # get now_account_value, now_stock_value, now_earning, now_cost, now_turnover now_account_value = self.current.calculate_value() now_stock_value = self.current.calculate_stock_value() - self.earning = now_account_value - last_account_value + now_earning = now_account_value - last_account_value + now_cost = self.accum_info.get_cost - last_total_cost + now_turnover = self.accum_info.get_turnover - last_total_turnover # update report for today # judge whether the the trading is begin. # and don't add init account state into report, due to we don't have excess return in those days. @@ -160,11 +184,13 @@ class Account: trade_end_time=trade_end_time, account_value=now_account_value, cash=self.current.position["cash"], - return_rate=(self.earning + self.ct) / last_account_value, + return_rate=(now_earning + now_cost) / last_account_value, # here use earning to calculate return, position's view, earning consider cost, true return # in order to make same definition with original backtest in evaluate.py - turnover_rate=self.to / last_account_value, - cost_rate=self.ct / last_account_value, + total_turnover=self.accum_info.get_turnover, + turnover_rate=now_turnover / last_account_value, + total_cost=self.accum_info.get_cost, + cost_rate=now_cost / last_account_value, stock_value=now_stock_value, ) # set now_account_value to position @@ -174,8 +200,60 @@ class Account: # note use deepcopy self.positions[trade_start_time] = copy.deepcopy(self.current) - # finish today's updation - # reset the bar variables - self.rtn = 0 - self.ct = 0 - self.to = 0 + def update_bar_end( + self, + trade_start_time: pd.Timestamp, + trade_end_time: pd.Timestamp, + trade_exchange: Exchange, + atomic: bool, + generate_report: bool = False, + trade_info: list = None, + inner_order_indicators: Indicator = None, + indicator_config: dict = {}, + ): + """update account at each trading bar step + + Parameters + ---------- + trade_start_time : pd.Timestamp + closed start time of step + trade_end_time : pd.Timestamp + closed end time of step + trade_exchange : Exchange + trading exchange, used to update current + atomic : bool + whether the trading executor is atomic, which means there is no higher-frequency trading executor inside it + - if atomic is True, calculate the indicators with trade_info + - else, aggregate indicators with inner indicators + generate_report : bool, optional + whether to generate report, by default False + trade_info : List[(Order, float, float, float)], optional + trading information, by default None + - necessary if atomic is True + - list of tuple(order, trade_val, trade_cost, trade_price) + inner_order_indicators : Indicator, optional + indicators of inner executor, by default None + - necessary if atomic is False + - used to aggregate outer indicators + indicator_config : dict, optional + config of calculating indicators, by default {} + """ + if atomic is True and trade_info is None: + raise ValueError("trade_info is necessary in atomic executor") + elif atomic is False and inner_order_indicators is None: + raise ValueError("inner_order_indicators is necessary in unatomic executor") + + self.update_bar_count() + self.update_current(trade_start_time, trade_end_time, trade_exchange) + if generate_report: + self.update_report(trade_start_time, trade_end_time) + + self.indicator.clear() + + if atomic: + self.indicator.update_order_indicators(trade_start_time, trade_end_time, trade_info, trade_exchange) + else: + self.indicator.agg_order_indicators(inner_order_indicators, indicator_config) + + self.indicator.cal_trade_indicators(trade_start_time, self.freq, indicator_config) + self.indicator.record(trade_start_time) diff --git a/qlib/backtest/backtest.py b/qlib/backtest/backtest.py index e9d864c92..3892fde41 100644 --- a/qlib/backtest/backtest.py +++ b/qlib/backtest/backtest.py @@ -1,10 +1,25 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +from ..utils.resam import parse_freq def backtest_loop(start_time, end_time, trade_strategy, trade_executor): """backtest funciton for the interaction of the outermost strategy and executor in the nested decison execution + Returns + ------- + report: Report + it records the trading report information + """ + return_value = {} + for _decison in collect_data_loop(start_time, end_time, trade_strategy, trade_executor, return_value): + pass + return return_value.get("report"), return_value.get("indicator") + + +def collect_data_loop(start_time, end_time, trade_strategy, trade_executor, return_value: dict = None): + """Generator for collecting the trade decision data for rl training + Parameters ---------- start_time : pd.Timestamp|str @@ -15,26 +30,8 @@ def backtest_loop(start_time, end_time, trade_strategy, trade_executor): the outermost portfolio strategy trade_executor : BaseExecutor the outermost executor - - Returns - ------- - report: Report - it records the trading report information - """ - trade_executor.reset(start_time=start_time, end_time=end_time) - level_infra = trade_executor.get_level_infra() - trade_strategy.reset(level_infra=level_infra) - - _execute_result = None - while not trade_executor.finished(): - _trade_decision = trade_strategy.generate_trade_decision(_execute_result) - _execute_result = trade_executor.execute(_trade_decision) - - return trade_executor.get_report() - - -def collect_data_loop(start_time, end_time, trade_strategy, trade_executor): - """Generator for collecting the trade decision data for rl training + return_value : dict + used for backtest_loop Yields ------- @@ -49,3 +46,19 @@ def collect_data_loop(start_time, end_time, trade_strategy, trade_executor): while not trade_executor.finished(): _trade_decision = trade_strategy.generate_trade_decision(_execute_result) _execute_result = yield from trade_executor.collect_data(_trade_decision) + + if return_value is not None: + all_executors = trade_executor.get_all_executors() + + all_reports = { + "{}{}".format(*parse_freq(_executor.time_per_step)): _executor.get_report() + for _executor in all_executors + if _executor.generate_report + } + all_indicators = { + "{}{}".format( + *parse_freq(_executor.time_per_step) + ): _executor.get_trade_indicator().generate_trade_indicators_dataframe() + for _executor in all_executors + } + return_value.update({"report": all_reports, "indicator": all_indicators}) diff --git a/qlib/backtest/exchange.py b/qlib/backtest/exchange.py index 6accb5e05..b80663245 100644 --- a/qlib/backtest/exchange.py +++ b/qlib/backtest/exchange.py @@ -48,14 +48,17 @@ class Exchange: :param trade_unit: trade unit, 100 for China A market :param min_cost: min cost, default 5 :param extra_quote: pandas, dataframe consists of - columns: like ['$vwap', '$close', '$factor', 'limit']. + columns: like ['$vwap', '$close', '$volume', '$factor', 'limit_sell', 'limit_buy']. The limit indicates that the etf is tradable on a specific day. Necessary fields: $close is for calculating the total value at end of each day. Optional fields: + $volume is only necessary when we limit the trade amount or caculate PA(vwap) indicator $vwap is only necessary when we use the $vwap price as the deal price $factor is for rounding to the trading unit - limit will be set to False by default(False indicates we can buy this + limit_sell will be set to False by default(False indicates we can sell this + target on this day). + limit_buy will be set to False by default(False indicates we can buy this target on this day). index: MultipleIndex(instrument, pd.Datetime) """ diff --git a/qlib/backtest/executor.py b/qlib/backtest/executor.py index d68ff3ab1..c216a461c 100644 --- a/qlib/backtest/executor.py +++ b/qlib/backtest/executor.py @@ -20,7 +20,7 @@ class BaseExecutor: time_per_step: str, start_time: Union[str, pd.Timestamp] = None, end_time: Union[str, pd.Timestamp] = None, - show_indicator: bool = False, + indicator_config: dict = {}, generate_report: bool = False, verbose: bool = False, track_data: bool = False, @@ -33,7 +33,40 @@ class BaseExecutor: time_per_step : str trade time per trading step, used for genreate the trade calendar show_indicator: bool, optional - whether to show indicators, such as FFR/PA/POS, .etc + whether to show indicators, : + - 'pa', the price advantage + - 'pos', the positive rate + - 'ffr', the fulfill rate + indicator_config: dict, optional + config for calculating trade indicator, including the following fields: + - 'show_indicator': whether to show indicators, optional, default by False. The indicators includes + - 'pa', the price advantage + - 'pos', the positive rate + - 'ffr', the fulfill rate + - 'pa_config': config for calculating price advantage(pa), optional + - 'base_price': the based price than which the trading price is advanced, Optional, default by 'twap' + - If 'base_price' is 'twap', the based price is the time weighted average price + - If 'base_price' is 'vwap', the based price is the volume weighted average price + - 'weight_method': weighted method when calculating total trading pa by different orders' pa in each step, optional, default by 'mean' + - If 'weight_method' is 'mean', calculating mean value of different orders' pa + - If 'weight_method' is 'amount_weighted', calculating amount weighted average value of different orders' pa + - If 'weight_method' is 'value_weighted', calculating value weighted average value of different orders' pa + - 'ffr_config': config for calculating fulfill rate(ffr), optional + - 'weight_method': weighted method when calculating total trading ffr by different orders' ffr in each step, optional, default by 'mean' + - If 'weight_method' is 'mean', calculating mean value of different orders' ffr + - If 'weight_method' is 'amount_weighted', calculating amount weighted average value of different orders' ffr + - If 'weight_method' is 'value_weighted', calculating value weighted average value of different orders' ffr + Example: + { + 'show_indicator': True, + 'pa_config': { + 'base_value': 'twap', + 'weight_method': 'value_weighted', + }, + 'ffr_config':{ + 'weight_method': 'value_weighted', + } + } generate_report : bool, optional whether to generate report, by default False verbose : bool, optional @@ -51,7 +84,7 @@ class BaseExecutor: """ self.time_per_step = time_per_step - self.show_indicator = show_indicator + self.indicator_config = indicator_config self.generate_report = generate_report self.verbose = verbose self.track_data = track_data @@ -132,18 +165,20 @@ class BaseExecutor: yield trade_decision return self.execute(trade_decision) - def get_trade_account(self): - raise NotImplementedError("get_trade_account is not implemented!") - def get_report(self): - raise NotImplementedError("get_report is not implemented!") + if self.generate_report: + _report = self.trade_account.report.generate_report_dataframe() + _positions = self.trade_account.get_positions() + return _report, _positions + else: + raise ValueError("generate_report should be True if you want to generate report") def get_all_executors(self): """Return all executors""" return [self] def get_trade_indicator(self): - return self.trade_account.indicator.trade_indicator + return self.trade_account.indicator class NestedExecutor(BaseExecutor): @@ -159,7 +194,7 @@ class NestedExecutor(BaseExecutor): inner_strategy: Union[BaseStrategy, dict], start_time: Union[str, pd.Timestamp] = None, end_time: Union[str, pd.Timestamp] = None, - show_indicator: bool = False, + indicator_config: dict = {}, generate_report: bool = False, verbose: bool = False, track_data: bool = False, @@ -190,7 +225,7 @@ class NestedExecutor(BaseExecutor): time_per_step=time_per_step, start_time=start_time, end_time=end_time, - show_indicator=show_indicator, + indicator_config=indicator_config, generate_report=generate_report, verbose=verbose, track_data=track_data, @@ -198,7 +233,7 @@ class NestedExecutor(BaseExecutor): **kwargs, ) - if generate_report and trade_exchange is not None: + if trade_exchange is not None: self.trade_exchange = trade_exchange def reset_common_infra(self, common_infra): @@ -209,7 +244,7 @@ class NestedExecutor(BaseExecutor): """ super(NestedExecutor, self).reset_common_infra(common_infra) - if self.generate_report and common_infra.has("trade_exchange"): + if common_infra.has("trade_exchange"): self.trade_exchange = common_infra.get("trade_exchange") self.inner_executor.reset_common_infra(common_infra) @@ -222,66 +257,43 @@ class NestedExecutor(BaseExecutor): sub_level_infra = self.inner_executor.get_level_infra() self.inner_strategy.reset(level_infra=sub_level_infra, outer_trade_decision=trade_decision) - def _update_trade_account(self, inner_indicators): - trade_step = self.trade_calendar.get_trade_step() - trade_start_time, trade_end_time = self.trade_calendar.get_step_time(trade_step) - self.trade_account.update_bar_count() - if self.generate_report: - self.trade_account.update_bar_report( - trade_start_time=trade_start_time, - trade_end_time=trade_end_time, - trade_exchange=self.trade_exchange, - ) - - self.trade_account.indicator.clear() - self.trade_account.indicator.agg_report_info(inner_indicators=inner_indicators) - self.trade_account.indicator.agg_FFR() - self.trade_account.indicator.agg_PA(inner_indicators=inner_indicators) - - if self.show_indicator: - FFR_value = self.trade_account.indicator.get_statistics_FFR(method="value_weighted") - PA_value = self.trade_account.indicator.get_statistics_PA(method="value_weighted") - POS_values = self.trade_account.indicator.get_statistics_POS() - print( - "[Indicator({}) {:%Y-%m-%d}]: FFR: {}, PA: {}, POS: {}".format( - self.time_per_step, trade_start_time, FFR_value, PA_value, POS_values - ) - ) - def execute(self, trade_decision): - for _data in self.collect_data(trade_decision): + return_value = {} + for _decison in self.collect_data(trade_decision, return_value): pass - return self._execute_result + return return_value.get("execute_result") - def collect_data(self, trade_decision): + def collect_data(self, trade_decision, return_value=None): if self.track_data: yield trade_decision self._init_sub_trading(trade_decision) execute_result = [] - inner_indicators = [] + inner_order_indicators = [] _inner_execute_result = None while not self.inner_executor.finished(): _inner_trade_decision = self.inner_strategy.generate_trade_decision(_inner_execute_result) _inner_execute_result = yield from self.inner_executor.collect_data(trade_decision=_inner_trade_decision) execute_result.extend(_inner_execute_result) - inner_indicators.append(self.inner_executor.get_trade_indicator()) + inner_order_indicators.append(self.inner_executor.get_trade_indicator().get_order_indicator) if hasattr(self, "trade_account"): - self._update_trade_account(inner_indicators=inner_indicators) + trade_step = self.trade_calendar.get_trade_step() + trade_start_time, trade_end_time = self.trade_calendar.get_step_time(trade_step) + self.trade_account.update_bar_end( + trade_start_time, + trade_end_time, + self.trade_exchange, + atomic=False, + generate_report=self.generate_report, + inner_order_indicators=inner_order_indicators, + indicator_config=self.indicator_config, + ) self.trade_calendar.step() - self._execute_result = execute_result + if return_value is not None: + return_value.update({"execute_result": execute_result}) return execute_result - def get_report(self): - sub_env_report_dict = self.inner_executor.get_report() - if self.generate_report: - _report = self.trade_account.report.generate_report_dataframe() - _positions = self.trade_account.get_positions() - _count, _freq = parse_freq(self.time_per_step) - sub_env_report_dict.update({f"{_count}{_freq}": (_report, _positions)}) - return sub_env_report_dict - def get_all_executors(self): """Return all executors, including self and inner_executor.get_all_executors()""" return [self, *self.inner_executor.get_all_executors()] @@ -295,7 +307,7 @@ class SimulatorExecutor(BaseExecutor): time_per_step: str, start_time: Union[str, pd.Timestamp] = None, end_time: Union[str, pd.Timestamp] = None, - show_indicator: bool = False, + indicator_config: dict = {}, generate_report: bool = False, verbose: bool = False, track_data: bool = False, @@ -314,7 +326,7 @@ class SimulatorExecutor(BaseExecutor): time_per_step=time_per_step, start_time=start_time, end_time=end_time, - show_indicator=show_indicator, + indicator_config=indicator_config, generate_report=generate_report, verbose=verbose, track_data=track_data, @@ -377,41 +389,14 @@ class SimulatorExecutor(BaseExecutor): # do nothing pass - self.trade_account.update_bar_count() - - if self.generate_report: - self.trade_account.update_bar_report( - trade_start_time=trade_start_time, - trade_end_time=trade_end_time, - trade_exchange=self.trade_exchange, - ) - - self.trade_account.indicator.clear() - self.trade_account.indicator.update_trade_info(trade_info=execute_result) - self.trade_account.indicator.update_FFR() - self.trade_account.indicator.update_PA( - freq=self.time_per_step, trade_start_time=trade_start_time, trade_end_time=trade_end_time + self.trade_account.update_bar_end( + trade_start_time, + trade_end_time, + self.trade_exchange, + atomic=True, + generate_report=self.generate_report, + trade_info=execute_result, + indicator_config=self.indicator_config, ) - self.trade_account.indicator.record(trade_start_time=trade_start_time) - - if self.show_indicator: - FFR_value = self.trade_account.indicator.get_statistics_FFR(method="value_weighted") - PA_value = self.trade_account.indicator.get_statistics_PA(method="value_weighted") - POS_values = self.trade_account.indicator.get_statistics_POS() - print( - "[Indicator({}) {:%Y-%m-%d %H:%M:%S}]: FFR: {}, PA: {}, POS: {}".format( - self.time_per_step, trade_start_time, FFR_value, PA_value, POS_values - ) - ) - self.trade_calendar.step() return execute_result - - def get_report(self): - if self.generate_report: - _report = self.trade_account.report.generate_report_dataframe() - _positions = self.trade_account.get_positions() - _count, _freq = parse_freq(self.time_per_step) - return {f"{_count}{_freq}": (_report, _positions)} - else: - return {} diff --git a/qlib/backtest/report.py b/qlib/backtest/report.py index d12595db5..5052a1e88 100644 --- a/qlib/backtest/report.py +++ b/qlib/backtest/report.py @@ -52,11 +52,13 @@ class Report: self.init_bench(freq=freq, benchmark_config=benchmark_config) def init_vars(self): - self.accounts = OrderedDict() # account postion value for each trade date - self.returns = OrderedDict() # daily return rate for each trade date - self.turnovers = OrderedDict() # turnover for each trade date - self.costs = OrderedDict() # trade cost for each trade date - self.values = OrderedDict() # value for each trade date + self.accounts = OrderedDict() # account postion value for each trade time + self.returns = OrderedDict() # daily return rate for each trade time + self.total_turnovers = OrderedDict() # total turnover for each trade time + self.turnovers = OrderedDict() # turnover for each trade time + self.total_costs = OrderedDict() # total trade cost for each trade time + self.costs = OrderedDict() # trade cost rate for each trade time + self.values = OrderedDict() # value for each trade time self.cashes = OrderedDict() self.benches = OrderedDict() self.latest_report_time = None # pd.TimeStamp @@ -87,10 +89,10 @@ class Report: def _sample_benchmark(self, bench, trade_start_time, trade_end_time): def cal_change(x): - return (x + 1).prod() - 1 + return (x + 1).prod() _ret = resam_ts_data(bench, trade_start_time, trade_end_time, method=cal_change) - return 0.0 if _ret is None else _ret + return 0.0 if _ret is None else _ret - 1 def is_empty(self): return len(self.accounts) == 0 @@ -101,6 +103,12 @@ class Report: def get_latest_account_value(self): return self.accounts[self.latest_report_time] + def get_latest_total_cost(self): + return self.total_costs[self.latest_report_time] + + def get_latest_total_turnover(self): + return self.total_turnovers[self.latest_report_time] + def update_report_record( self, trade_start_time=None, @@ -108,7 +116,9 @@ class Report: account_value=None, cash=None, return_rate=None, + total_turnover=None, turnover_rate=None, + total_cost=None, cost_rate=None, stock_value=None, bench_value=None, @@ -119,12 +129,14 @@ class Report: account_value, cash, return_rate, + total_turnover, turnover_rate, + total_cost, cost_rate, stock_value, ]: raise ValueError( - "None in [trade_start_time, account_value, cash, return_rate, turnover_rate, cost_rate, stock_value]" + "None in [trade_start_time, account_value, cash, return_rate, total_turnover, turnover_rate, total_cost, cost_rate, stock_value]" ) if trade_end_time is None and bench_value is None: @@ -135,20 +147,24 @@ class Report: # update report data self.accounts[trade_start_time] = account_value self.returns[trade_start_time] = return_rate + self.total_turnovers[trade_start_time] = total_turnover self.turnovers[trade_start_time] = turnover_rate + self.total_costs[trade_start_time] = total_cost self.costs[trade_start_time] = cost_rate self.values[trade_start_time] = stock_value self.cashes[trade_start_time] = cash self.benches[trade_start_time] = bench_value # update latest_report_date self.latest_report_time = trade_start_time - # finish daily report update + # finish report update in each step def generate_report_dataframe(self): report = pd.DataFrame() report["account"] = pd.Series(self.accounts) report["return"] = pd.Series(self.returns) + report["total_turnover"] = pd.Series(self.total_turnovers) report["turnover"] = pd.Series(self.turnovers) + report["total_cost"] = pd.Series(self.total_costs) report["cost"] = pd.Series(self.costs) report["value"] = pd.Series(self.values) report["cash"] = pd.Series(self.cashes) @@ -163,7 +179,7 @@ class Report: def load_report(self, path): """load report from a file should have format like - columns = ['account', 'return', 'turnover', 'cost', 'value', 'cash', 'bench'] + columns = ['account', 'return', 'total_turnover', 'turnover', 'cost', 'total_cost', 'value', 'cash', 'bench'] :param path: str/ pathlib.Path() """ @@ -179,7 +195,9 @@ class Report: account_value=r.loc[trade_start_time]["account"], cash=r.loc[trade_start_time]["cash"], return_rate=r.loc[trade_start_time]["return"], + total_turnover=r.loc[trade_start_time]["total_turnover"], turnover_rate=r.loc[trade_start_time]["turnover"], + total_cost=r.loc[trade_start_time]["total_cost"], cost_rate=r.loc[trade_start_time]["cost"], stock_value=r.loc[trade_start_time]["value"], bench_value=r.loc[trade_start_time]["bench"], @@ -188,147 +206,184 @@ class Report: class Indicator: def __init__(self): - self.indicator_his = dict() - self.trade_indicator = dict() - - def __getitem__(self, key): - return self.trade_indicator[key] - - def __setitem__(self, key, value): - self.trade_indicator[key] = value - - def __contains__(self, key): - return key in self.trade_indicator + self.order_indicator_his = OrderedDict() + self.order_indicator = OrderedDict() + self.trade_indicator_his = OrderedDict() + self.trade_indicator = OrderedDict() def clear(self): - self.trade_indicator = dict() + self.order_indicator = OrderedDict() + self.trade_indicator = OrderedDict() def record(self, trade_start_time): - self.indicator_his[trade_start_time] = pd.DataFrame(self.trade_indicator) + self.order_indicator_his[trade_start_time] = self.order_indicator + self.trade_indicator_his[trade_start_time] = self.trade_indicator - def update_trade_info(self, trade_info: list): + def _update_order_trade_info(self, trade_info: list): amount = dict() deal_amount = dict() trade_price = dict() + trade_value = dict() trade_cost = dict() for order, _trade_val, _trade_cost, _trade_price in trade_info: amount[order.stock_id] = order.amount * (order.direction * 2 - 1) deal_amount[order.stock_id] = order.deal_amount * (order.direction * 2 - 1) trade_price[order.stock_id] = _trade_price + trade_value[order.stock_id] = _trade_val * (order.direction * 2 - 1) trade_cost[order.stock_id] = _trade_cost - self["amount"] = pd.Series(amount) - self["deal_amount"] = pd.Series(deal_amount) - self["trade_price"] = pd.Series(trade_price) - self["trade_cost"] = pd.Series(trade_cost) + self.order_indicator["amount"] = pd.Series(amount) + self.order_indicator["deal_amount"] = pd.Series(deal_amount) + self.order_indicator["trade_price"] = pd.Series(trade_price) + self.order_indicator["trade_value"] = pd.Series(trade_value) + self.order_indicator["trade_cost"] = pd.Series(trade_cost) - def update_FFR(self): - self["fulfill_rate"] = self["deal_amount"] / self["amount"] + def _update_order_fulfill_rate(self): + self.order_indicator["ffr"] = self.order_indicator["deal_amount"] / self.order_indicator["amount"] - def update_PA(self, freq, trade_start_time, trade_end_time, base_price="twap"): - base_price = base_price.lower() + def _update_order_price_advantage(self, trade_exchange, trade_start_time, trade_end_time): + self.order_indicator["base_price"] = self.order_indicator["trade_price"] + instruments = list(self.order_indicator["base_price"].index) + self.order_indicator["volume"] = pd.Series( + [ + trade_exchange.get_volume(stock_id=inst, start_time=trade_start_time, end_time=trade_end_time) + for inst in instruments + ], + index=instruments, + ) + self.order_indicator["pa"] = ( + self.order_indicator["trade_price"] - self.order_indicator["base_price"] + ) / self.order_indicator["base_price"] - instruments = list(self["amount"].index) - if base_price == "twap": - # too slow - # price_info, _ = get_higher_freq_feature(instruments, fields=["$close"], start_time=trade_start_time, end_time=trade_end_time, freq=freq) - # price_info = price_info.astype(float) - - # self["base_price"] = price_info["$close"].groupby(level="instrument").mean() - self["base_price"] = self["trade_price"] - - elif base_price == "vwap": - # too slow - price_info, _ = get_higher_freq_feature( - instruments, - fields=["$close", "$volume"], - start_time=trade_start_time, - end_time=trade_end_time, - freq=freq, - ) - price_info = price_info.astype(float) - self["base_price"] = price_info.groupby(level="instrument").apply( - lambda x: (x["$close"] * x["$volume"]).sum() / x["$volume"].sum() - ) - self["volume"] = price_info["$volume"].groupby(level="instrument").sum() - else: - raise ValueError(f"base_price {base_price} is not supported!") - - self["pa"] = (self["trade_price"] - self["base_price"]) / self["base_price"] - - def agg_report_info(self, inner_indicators): + def _agg_order_trade_info(self, inner_order_indicators): amount = pd.Series() deal_amount = pd.Series() trade_price = pd.Series() + trade_value = pd.Series() trade_cost = pd.Series() - for inner_indicator in inner_indicators: - amount = amount.add(inner_indicator["amount"], fill_value=0) - deal_amount = deal_amount.add(inner_indicator["deal_amount"], fill_value=0) - trade_price = trade_price.add(inner_indicator["trade_price"] * inner_indicator["deal_amount"], fill_value=0) - trade_cost = trade_cost.add(inner_indicator["trade_cost"], fill_value=0) + for _order_indicator in inner_order_indicators: + amount = amount.add(_order_indicator["amount"], fill_value=0) + deal_amount = deal_amount.add(_order_indicator["deal_amount"], fill_value=0) + trade_price = trade_price.add( + _order_indicator["trade_price"] * _order_indicator["deal_amount"], fill_value=0 + ) + trade_value = trade_value.add(_order_indicator["trade_value"], fill_value=0) + trade_cost = trade_cost.add(_order_indicator["trade_cost"], fill_value=0) - self["amount"] = amount - self["deal_amount"] = deal_amount - trade_price /= self["deal_amount"] - self["trade_price"] = trade_price - self["trade_cost"] = trade_cost + self.order_indicator["amount"] = amount + self.order_indicator["deal_amount"] = deal_amount + trade_price /= self.order_indicator["deal_amount"] + self.order_indicator["trade_price"] = trade_price + self.order_indicator["trade_value"] = trade_value + self.order_indicator["trade_cost"] = trade_cost - def agg_FFR(self): - self["fulfill_rate"] = self["deal_amount"] / self["amount"] + def _agg_order_fulfill_rate(self): + self.order_indicator["ffr"] = self.order_indicator["deal_amount"] / self.order_indicator["amount"] - def agg_PA(self, inner_indicators, base_price="twap"): + def _agg_order_price_advantage(self, inner_order_indicators, base_price="twap"): base_price = base_price.lower() + volume = pd.Series() + for _order_indicator in inner_order_indicators: + volume = volume.add(_order_indicator["volume"], fill_value=0) + self.order_indicator["volume"] = volume if base_price == "twap": base_price = pd.Series() price_count = pd.Series() - for inner_indicator in inner_indicators: - base_price = base_price.add(inner_indicator["base_price"], fill_value=0) - price_count = price_count.add(pd.Series(1, index=inner_indicator["base_price"].index), fill_value=0) + for _order_indicator in inner_order_indicators: + base_price = base_price.add(_order_indicator["base_price"], fill_value=0) + price_count = price_count.add(pd.Series(1, index=_order_indicator["base_price"].index), fill_value=0) base_price /= price_count - self["base_price"] = base_price + self.order_indicator["base_price"] = base_price elif base_price == "vwap": base_price = pd.Series() - volume = pd.Series() - for inner_indicator in inner_indicators: - base_price = base_price.add(inner_indicator["base_price"] * inner_indicator["volume"], fill_value=0) - volume = volume.add(inner_indicator["volume"], fill_value=0) - base_price /= volume - self["base_price"] = base_price - self["volume"] = volume + for _order_indicator in inner_order_indicators: + base_price = base_price.add(_order_indicator["base_price"] * _order_indicator["volume"], fill_value=0) + base_price /= self.order_indicator["volume"] + self.order_indicator["base_price"] = base_price + else: raise ValueError(f"base_price {base_price} is not supported!") - self["pa"] = (self["trade_price"] - self["base_price"]) / self["base_price"] + self.order_indicator["pa"] = self.order_indicator["trade_price"] / self.order_indicator["base_price"] - 1 - def get_statistics_FFR(self, method="mean"): + def _cal_trade_fulfill_rate(self, method="mean"): if method == "mean": - return self["fulfill_rate"].mean() + return self.order_indicator["ffr"].mean() elif method == "amount_weighted": - weights = self["deal_amount"].abs() - return (self["fulfill_rate"] * weights).sum() / weights.sum() + weights = self.order_indicator["deal_amount"].abs() + return (self.order_indicator["ffr"] * weights).sum() / weights.sum() elif method == "value_weighted": - weights = (self["deal_amount"] * self["trade_price"]).abs() - return (self["fulfill_rate"] * weights).sum() / weights.sum() + weights = self.order_indicator["trade_value"].abs() + return (self.order_indicator["ffr"] * weights).sum() / weights.sum() else: raise ValueError(f"method {method} is not supported!") - def get_statistics_PA(self, method="mean"): - pa_order = self["pa"] * (self["amount"] < 0).astype(int) + def _cal_trade_price_advantage(self, method="mean"): + pa_order = self.order_indicator["pa"] * (self.order_indicator["amount"] < 0).astype(int) if method == "mean": return pa_order.mean() elif method == "amount_weighted": - weights = self["deal_amount"].abs() + weights = self.order_indicator["deal_amount"].abs() return (pa_order * weights).sum() / weights.sum() elif method == "value_weighted": - weights = (self["deal_amount"] * self["trade_price"]).abs() + weights = self.order_indicator["trade_value"].abs() return (pa_order * weights).sum() / weights.sum() else: raise ValueError(f"method {method} is not supported!") - def get_statistics_POS(self): - pa_order = self["pa"] * (self["amount"] < 0).astype(int) - return (pa_order > 1e-8).astype(int).sum() / len(pa_order) + def _cal_trade_positive_rate(self): + pa_order = self.order_indicator["pa"] * (self.order_indicator["amount"] < 0).astype(int) + return (pa_order > 0).astype(int).sum() / len(pa_order) + + def _cal_trade_amount(self): + return self.order_indicator["deal_amount"].abs().sum() + + def _cal_trade_value(self): + return self.order_indicator["trade_value"].abs().sum() + + def update_order_indicators(self, trade_start_time, trade_end_time, trade_info, trade_exchange): + self._update_order_trade_info(trade_info=trade_info) + self._update_order_fulfill_rate() + self._update_order_price_advantage(trade_exchange, trade_start_time, trade_end_time) + + def agg_order_indicators(self, inner_order_indicators, indicator_config={}): + self._agg_order_trade_info(inner_order_indicators) + self._agg_order_fulfill_rate() + pa_config = indicator_config.get("pa_config", {}) + self._agg_order_price_advantage(inner_order_indicators, base_price=pa_config.get("base_price", "twap")) + + def cal_trade_indicators(self, trade_start_time, freq, indicator_config={}): + show_indicator = indicator_config.get("show_indicator", False) + ffr_config = indicator_config.get("ffr_config", {}) + pa_config = indicator_config.get("pa_config", {}) + fulfill_rate = self._cal_trade_fulfill_rate(method=ffr_config.get("weight_method", "mean")) + price_advantage = self._cal_trade_price_advantage(method=pa_config.get("weight_method", "mean")) + positive_rate = self._cal_trade_positive_rate() + trade_amount = self._cal_trade_amount() + trade_value = self._cal_trade_value() + self.trade_indicator["ffr"] = fulfill_rate + self.trade_indicator["pa"] = price_advantage + self.trade_indicator["pos"] = positive_rate + self.trade_indicator["amount"] = trade_amount + self.trade_indicator["value"] = trade_value + if show_indicator: + print( + "[Indicator({}) {:%Y-%m-%d %H:%M:%S}]: FFR: {}, PA: {}, POS: {}".format( + freq, trade_start_time, fulfill_rate, price_advantage, positive_rate + ) + ) + + @property + def get_order_indicator(self): + return self.order_indicator + + @property + def get_trade_indicator(self): + return self.trade_indicator + + def generate_trade_indicators_dataframe(self): + return pd.DataFrame.from_dict(self.trade_indicator_his, orient="index") diff --git a/qlib/contrib/evaluate.py b/qlib/contrib/evaluate.py index 0ef8f95a5..a048ead30 100644 --- a/qlib/contrib/evaluate.py +++ b/qlib/contrib/evaluate.py @@ -11,7 +11,7 @@ import warnings from ..log import get_module_logger from ..backtest import get_exchange, backtest as backtest_func from ..utils import get_date_range -from ..utils.resam import parse_freq +from ..utils.resam import parse_freq, NORM_FREQ_MONTH, NORM_FREQ_WEEK, NORM_FREQ_DAY, NORM_FREQ_MINUTE from ..data import D from ..config import C @@ -37,12 +37,12 @@ def risk_analysis(r, N: int = None, freq: str = "day"): def cal_risk_analysis_scaler(freq): _count, _freq = parse_freq(freq) _freq_scaler = { - "minute": 240 * 252, - "day": 252, - "week": 50, - "month": 12, + NORM_FREQ_MINUTE: 240 * 252, + NORM_FREQ_DAY: 252, + NORM_FREQ_WEEK: 50, + NORM_FREQ_MONTH: 12, } - return _count * _freq_scaler[_freq] + return _freq_scaler[_freq] / _count if N is None and freq is None: raise ValueError("at least one of `N` and `freq` should exist") @@ -63,7 +63,51 @@ def risk_analysis(r, N: int = None, freq: str = "day"): "information_ratio": information_ratio, "max_drawdown": max_drawdown, } - res = pd.Series(data, index=data.keys()).to_frame("risk") + res = pd.Series(data).to_frame("risk") + return res + + +def indicator_analysis(df, method="mean"): + """analyze statistical time-series indicators of trading + + Parameters + ---------- + df : pandas.DataFrame + columns: like ['pa', 'pos', 'ffr', 'amount', 'value']. + Necessary fields: + - 'pa' is the price advantage in trade indicators + - 'pos' is the positive rate in trade indicators + - 'ffr' is the fulfill rate in trade indicators + Optional fields: + - 'amount' is the total deal amount, only necessary when method is 'amount_weighted' + - 'value' is the total trade value, only necessary when method is 'value_weighted' + + index: Index(datetime) + method : str, optional + statistics method, by default "mean" + - if method is 'mean', count the mean statistical value of each trade indicator + - if method is 'amount_weighted', count the amount weighted mean statistical value of each trade indicator + - if method is 'value_weighted', count the value weighted mean statistical value of each trade indicator + + Returns + ------- + pd.DataFrame + statistical value of each trade indicator + """ + indicators_df = df[["pa", "pos", "ffr"]] + + if method == "mean": + res = indicators_df.mean() + elif method == "amount_weighted": + weights = df["amount"].abs() + res = indicators_df.mul(weights, axis=0).sum() / weights.sum() + elif method == "value_weighted": + weights = df["value"].abs() + res = indicators_df.mul(weights, axis=0).sum() / weights.sum() + else: + raise ValueError(f"indicator_analysis method {method} is not supported!") + + res = res.to_frame("value") return res diff --git a/qlib/workflow/record_temp.py b/qlib/workflow/record_temp.py index 9516d363a..4ecd5ccdf 100644 --- a/qlib/workflow/record_temp.py +++ b/qlib/workflow/record_temp.py @@ -7,7 +7,7 @@ import warnings import pandas as pd from pathlib import Path from pprint import pprint -from ..contrib.evaluate import risk_analysis +from ..contrib.evaluate import indicator_analysis, risk_analysis, indicator_analysis from ..data.dataset import DatasetH from ..data.dataset.handler import DataHandlerLP @@ -294,7 +294,9 @@ class PortAnaRecord(RecordTemp): artifact_path = "portfolio_analysis" - def __init__(self, recorder, config, risk_analysis_freq, **kwargs): + def __init__( + self, recorder, config, risk_analysis_freq, indicator_analysis_freq, indicator_analysis_method=None, **kwargs + ): """ config["strategy"] : dict define the strategy class as well as the kwargs. @@ -304,6 +306,10 @@ class PortAnaRecord(RecordTemp): define the backtest kwargs. risk_analysis_freq : str|List[str] risk analysis freq of report + indicator_analysis_freq : str|List[str] + indicator analysis freq of report + indicator_analysis_method : str, optional, default by None + the candidated values include 'mean', 'amount_weighted', 'value_weighted' """ super().__init__(recorder=recorder, **kwargs) @@ -312,10 +318,17 @@ class PortAnaRecord(RecordTemp): self.backtest_config = config["backtest"] if isinstance(risk_analysis_freq, str): risk_analysis_freq = [risk_analysis_freq] + if isinstance(indicator_analysis_freq, str): + indicator_analysis_freq = [indicator_analysis_freq] + self.risk_analysis_freq = [ "{0}{1}".format(*parse_freq(_analysis_freq)) for _analysis_freq in risk_analysis_freq ] - self.report_freq = self._get_report_freq(self.executor_config) + self.indicator_analysis_freq = [ + "{0}{1}".format(*parse_freq(_analysis_freq)) for _analysis_freq in indicator_analysis_freq + ] + self.indicator_analysis_method = indicator_analysis_method + self.all_freq = self._get_report_freq(self.executor_config) def _get_report_freq(self, executor_config): ret_freq = [] @@ -328,21 +341,26 @@ class PortAnaRecord(RecordTemp): def generate(self, **kwargs): # custom strategy and get backtest - report_dict = normal_backtest( + report_dict, indicator_dict = normal_backtest( executor=self.executor_config, strategy=self.strategy_config, **self.backtest_config ) - for report_freq, (report_normal, positions_normal) in report_dict.items(): + for _freq, (report_normal, positions_normal) in report_dict.items(): self.recorder.save_objects( - **{f"report_normal_{report_freq}.pkl": report_normal}, artifact_path=PortAnaRecord.get_path() + **{f"report_normal_{_freq}.pkl": report_normal}, artifact_path=PortAnaRecord.get_path() ) self.recorder.save_objects( - **{f"positions_normal_{report_freq}.pkl": positions_normal}, artifact_path=PortAnaRecord.get_path() + **{f"positions_normal_{_freq}.pkl": positions_normal}, artifact_path=PortAnaRecord.get_path() + ) + + for _freq, indicators_normal in indicator_dict.items(): + self.recorder.save_objects( + **{f"indicators_normal_{_freq}.pkl": indicators_normal}, artifact_path=PortAnaRecord.get_path() ) for _analysis_freq in self.risk_analysis_freq: if _analysis_freq not in report_dict: warnings.warn( - f"the freq {_analysis_freq} report is not found, please set the corresponding env with `generate_report==True`" + f"the freq {_analysis_freq} report is not found, please set the corresponding env with `generate_report=True`" ) else: report_normal, _ = report_dict.get(_analysis_freq) @@ -353,25 +371,46 @@ class PortAnaRecord(RecordTemp): analysis["excess_return_with_cost"] = risk_analysis( report_normal["return"] - report_normal["bench"] - report_normal["cost"], freq=_analysis_freq ) + analysis_df = pd.concat(analysis) # type: pd.DataFrame # log metrics - self.recorder.log_metrics(**flatten_dict(analysis_df["risk"].unstack().T.to_dict())) + analysis_dict = flatten_dict(analysis_df["risk"].unstack().T.to_dict()) + self.recorder.log_metrics(**{f"{_analysis_freq}.{k}": v for k, v in analysis_dict.items()}) # save results self.recorder.save_objects( - **{f"port_analysis_{report_freq}.pkl": analysis_df}, artifact_path=PortAnaRecord.get_path() + **{f"port_analysis_{_analysis_freq}.pkl": analysis_df}, artifact_path=PortAnaRecord.get_path() ) logger.info( - f"Portfolio analysis record 'port_analysis_{report_freq}.pkl' has been saved as the artifact of the Experiment {self.recorder.experiment_id}" + f"Portfolio analysis record 'port_analysis_{_analysis_freq}.pkl' has been saved as the artifact of the Experiment {self.recorder.experiment_id}" ) # print out results - pprint("The following are analysis results of the excess return without cost.") + pprint(f"The following are analysis results of benchmark return({_analysis_freq}).") + pprint(risk_analysis(report_normal["bench"], freq=_analysis_freq)) + pprint(f"The following are analysis results of the excess return without cost({_analysis_freq}).") pprint(analysis["excess_return_without_cost"]) - pprint("The following are analysis results of the excess return with cost.") + pprint(f"The following are analysis results of the excess return with cost({_analysis_freq}).") pprint(analysis["excess_return_with_cost"]) + for _analysis_freq in self.indicator_analysis_freq: + indicators_normal = indicator_dict.get(_analysis_freq) + if self.indicator_analysis_method is None: + analysis_df = indicator_analysis(indicators_normal) + else: + analysis_df = indicator_analysis(indicators_normal, method=self.indicator_analysis_method) + + # log metrics + analysis_dict = analysis_df["value"].to_dict() + self.recorder.log_metrics(**{f"{_analysis_freq}.{k}": v for k, v in analysis_dict.items()}) + # save results + self.recorder.save_objects( + **{f"indicator_analysis_{_analysis_freq}.pkl": analysis_df}, artifact_path=PortAnaRecord.get_path() + ) + pprint(f"The following are analysis results of indicators({_analysis_freq}).") + pprint(analysis_df) + def list(self): list_path = [] - for _freq in self.report_freq: + for _freq in self.all_freq: list_path.extend( [ PortAnaRecord.get_path(f"report_normal_{_freq}.pkl"), @@ -380,7 +419,7 @@ class PortAnaRecord(RecordTemp): ) for _analysis_freq in self.risk_analysis_freq: - if _analysis_freq in self.report_freq: + if _analysis_freq in self.all_freq: list_path.append(PortAnaRecord.get_path(f"port_analysis_{_analysis_freq}.pkl")) else: warnings.warn(f"{_analysis_freq} is not found")