From 45f73361e3bbcf6625600c52dccff9b3cdca723c Mon Sep 17 00:00:00 2001 From: lwwang1995 Date: Thu, 18 Mar 2021 11:17:42 +0800 Subject: [PATCH 01/24] add tcts baseline --- .../TCTS/workflow_config_tcts_Alpha360.yaml | 93 +++++ qlib/contrib/model/pytorch_tcts.py | 393 ++++++++++++++++++ 2 files changed, 486 insertions(+) create mode 100644 examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml create mode 100644 qlib/contrib/model/pytorch_tcts.py diff --git a/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml b/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml new file mode 100644 index 000000000..589f4b43e --- /dev/null +++ b/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml @@ -0,0 +1,93 @@ +qlib_init: + provider_uri: "~/.qlib/qlib_data/cn_data" + region: cn +market: &market csi300 +benchmark: &benchmark SH000300 +data_handler_config: &data_handler_config + start_time: 2008-01-01 + end_time: 2020-08-01 + fit_start_time: 2008-01-01 + fit_end_time: 2014-12-31 + instruments: *market + infer_processors: + - class: RobustZScoreNorm + kwargs: + fields_group: feature + clip_outlier: true + - class: Fillna + kwargs: + fields_group: feature + learn_processors: + - class: DropnaLabel + - class: CSRankNorm + kwargs: + fields_group: label + label: ["Ref($close, -2) / Ref($close, -1) - 1", + "Ref($close, -3) / Ref($close, -1) - 1", + "Ref($close, -4) / Ref($close, -1) - 1", + "Ref($close, -5) / Ref($close, -1) - 1", + "Ref($close, -6) / Ref($close, -1) - 1"] +port_analysis_config: &port_analysis_config + strategy: + class: TopkDropoutStrategy + module_path: qlib.contrib.strategy.strategy + kwargs: + topk: 50 + n_drop: 5 + backtest: + verbose: False + limit_threshold: 0.095 + account: 100000000 + benchmark: *benchmark + deal_price: close + open_cost: 0.0005 + close_cost: 0.0015 + min_cost: 5 +task: + model: + class: TCTS + module_path: qlib.contrib.model.pytorch_tcts + kwargs: + d_feat: 6 + hidden_size: 64 + num_layers: 2 + dropout: 0.0 + n_epochs: 200 + lr: 1e-3 + early_stop: 20 + batch_size: 800 + metric: loss + loss: mse + GPU: 0 + fore_optimizer: adam + weight_optimizer: adam + output_dim: 5 + fore_lr: 5e-7 + weight_lr: 5e-7 + steps: 3 + target_label: 0 + dataset: + class: DatasetH + module_path: qlib.data.dataset + kwargs: + handler: + class: Alpha360 + module_path: qlib.contrib.data.handler + kwargs: *data_handler_config + segments: + train: [2008-01-01, 2014-12-31] + valid: [2015-01-01, 2016-12-31] + test: [2017-01-01, 2020-08-01] + record: + - class: SignalRecord + module_path: qlib.workflow.record_temp + kwargs: {} + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 + - class: PortAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + config: *port_analysis_config \ No newline at end of file diff --git a/qlib/contrib/model/pytorch_tcts.py b/qlib/contrib/model/pytorch_tcts.py new file mode 100644 index 000000000..9f44ba31c --- /dev/null +++ b/qlib/contrib/model/pytorch_tcts.py @@ -0,0 +1,393 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + + +from __future__ import division +from __future__ import print_function + +import os +import numpy as np +import pandas as pd +import copy +from sklearn.metrics import roc_auc_score, mean_squared_error +import logging +from ...utils import ( + unpack_archive_with_buffer, + save_multiple_parts_file, + create_save_path, + drop_nan_by_y_index, +) +from ...log import get_module_logger, TimeInspector + +import torch +import torch.nn as nn +import torch.optim as optim + +from ...model.base import Model +from ...data.dataset import DatasetH +from ...data.dataset.handler import DataHandlerLP + + +class TCTS(Model): + """TCTS Model + + Parameters + ---------- + d_feat : int + input dimension for each time step + metric: str + the evaluate metric used in early stop + optimizer : str + optimizer name + GPU : str + the GPU ID(s) used for training + """ + + def __init__( + self, + d_feat=6, + hidden_size=64, + num_layers=2, + dropout=0.0, + n_epochs=200, + batch_size=2000, + early_stop=20, + loss="mse", + fore_optimizer="adam", + weight_optimizer="adam", + output_dim=5, + fore_lr=5e-7, + weight_lr=5e-7, + steps=3, + GPU=0, + seed=None, + target_label=0, + **kwargs + ): + # Set logger. + self.logger = get_module_logger("TCTS") + self.logger.info("TCTS pytorch version...") + + # set hyper-parameters. + self.d_feat = d_feat + self.hidden_size = hidden_size + self.num_layers = num_layers + self.dropout = dropout + self.n_epochs = n_epochs + self.batch_size = batch_size + self.early_stop = early_stop + self.loss = loss + self.device = torch.device("cuda:%d" % (GPU) if torch.cuda.is_available() else "cpu") + self.use_gpu = torch.cuda.is_available() + self.seed = seed + self.output_dim = output_dim + self.fore_lr = fore_lr + self.weight_lr = weight_lr + self.steps = steps + self.target_label = target_label + + self.logger.info( + "TCTS parameters setting:" + "\nd_feat : {}" + "\nhidden_size : {}" + "\nnum_layers : {}" + "\ndropout : {}" + "\nn_epochs : {}" + "\nbatch_size : {}" + "\nearly_stop : {}" + "\nloss_type : {}" + "\nvisible_GPU : {}" + "\nuse_GPU : {}" + "\nseed : {}".format( + d_feat, + hidden_size, + num_layers, + dropout, + n_epochs, + batch_size, + early_stop, + loss, + GPU, + self.use_gpu, + seed, + ) + ) + + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) + + self.fore_model = GRUModel( + d_feat=self.d_feat, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + dropout=self.dropout, + ) + self.weight_model = MLPModel( + d_feat=360 + 2 * self.output_dim + 1, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + dropout=self.dropout, + output_dim=self.output_dim, + ) + if fore_optimizer.lower() == "adam": + self.fore_optimizer = optim.Adam(self.fore_model.parameters(), lr=self.fore_lr) + elif fore_optimizer.lower() == "gd": + self.fore_optimizer = optim.SGD(self.fore_model.parameters(), lr=self.fore_lr) + else: + raise NotImplementedError("optimizer {} is not supported!".format(fore_optimizer)) + if weight_optimizer.lower() == "adam": + self.weight_optimizer = optim.Adam(self.weight_model.parameters(), lr=self.weight_lr) + elif weight_optimizer.lower() == "gd": + self.weight_optimizer = optim.SGD(self.weight_model.parameters(), lr=self.weight_lr) + else: + raise NotImplementedError("optimizer {} is not supported!".format(weight_optimizer)) + + self.fitted = False + self.fore_model.to(self.device) + self.weight_model.to(self.device) + + def loss_fn(self, pred, label, weight): + + loc = torch.argmax(weight, 1) + loss = (pred - label[np.arange(weight.shape[0]), loc]) ** 2 + return torch.mean(loss) + + def train_epoch(self, x_train, y_train, x_valid, y_valid): + + x_train_values = x_train.values + y_train_values = np.squeeze(y_train.values) + + indices = np.arange(len(x_train_values)) + np.random.shuffle(indices) + + init_fore_model = copy.deepcopy(self.fore_model) + for p in init_fore_model.parameters(): + p.init_fore_model = False + + self.fore_model.train() + self.weight_model.train() + + for p in self.weight_model.parameters(): + p.requires_grad = False + for p in self.fore_model.parameters(): + p.requires_grad = True + + for i in range(self.steps): + for i in range(len(indices))[:: self.batch_size]: + + if len(indices) - i < self.batch_size: + break + + feature = torch.from_numpy(x_train_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_train_values[indices[i : i + self.batch_size]]).float().to(self.device) + + init_pred = init_fore_model(feature) + pred = self.fore_model(feature) + + dis = init_pred - label.transpose(0, 1) + weight_feature = torch.cat((feature, dis.transpose(0, 1), label, init_pred.view(-1, 1)), 1) + weight = self.weight_model(weight_feature) + + loss = self.loss_fn(pred, label, weight) # hard + + self.fore_optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_value_(self.fore_model.parameters(), 3.0) + self.fore_optimizer.step() + + x_valid_values = x_valid.values + y_valid_values = np.squeeze(y_valid.values) + + indices = np.arange(len(x_valid_values)) + np.random.shuffle(indices) + for p in self.weight_model.parameters(): + p.requires_grad = True + for p in self.fore_model.parameters(): + p.requires_grad = False + + # fix forecasting model and valid weight model + for i in range(len(indices))[:: self.batch_size]: + + if len(indices) - i < self.batch_size: + break + + feature = torch.from_numpy(x_valid_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_valid_values[indices[i : i + self.batch_size]]).float().to(self.device) + + pred = self.fore_model(feature) + dis = pred - label.transpose(0, 1) + weight_feature = torch.cat((feature, dis.transpose(0, 1), label, pred.view(-1, 1)), 1) + weight = self.weight_model(weight_feature) + loc = torch.argmax(weight, 1) + valid_loss = torch.mean((pred - label[:, 0]) ** 2) + loss = torch.mean(-valid_loss * torch.log(weight[np.arange(weight.shape[0]), loc])) + + self.weight_optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_value_(self.weight_model.parameters(), 3.0) + self.weight_optimizer.step() + + def test_epoch(self, data_x, data_y): + + # prepare training data + x_values = data_x.values + y_values = np.squeeze(data_y.values) + + self.fore_model.eval() + + scores = [] + losses = [] + + indices = np.arange(len(x_values)) + + for i in range(len(indices))[:: self.batch_size]: + + if len(indices) - i < self.batch_size: + break + + feature = torch.from_numpy(x_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_values[indices[i : i + self.batch_size]]).float().to(self.device) + + pred = self.fore_model(feature) + loss = torch.mean((pred - label[:, abs(self.target_label)]) ** 2) + losses.append(loss.item()) + + return np.mean(losses) + + def fit( + self, + dataset: DatasetH, + evals_result=dict(), + verbose=True, + save_path=None, + ): + + df_train, df_valid, df_test = dataset.prepare( + ["train", "valid", "test"], + col_set=["feature", "label"], + data_key=DataHandlerLP.DK_L, + ) + + x_train, y_train = df_train["feature"], df_train["label"] + x_valid, y_valid = df_valid["feature"], df_valid["label"] + x_test, y_test = df_test["feature"], df_test["label"] + + if save_path == None: + save_path = create_save_path(save_path) + + best_loss = np.inf + best_epoch = 0 + stop_round = 0 + fore_best_param = copy.deepcopy(self.fore_optimizer.state_dict()) + weight_best_param = copy.deepcopy(self.weight_optimizer.state_dict()) + + for epoch in range(self.n_epochs): + print("Epoch:", epoch) + + print("training...") + self.train_epoch(x_train, y_train, x_valid, y_valid) + print("evaluating...") + val_loss = self.test_epoch(x_valid, y_valid) + test_loss = self.test_epoch(x_test, y_test) + + print("valid %.6f, test %.6f" % (val_loss, test_loss)) + + if val_loss < best_loss: + best_loss = val_loss + stop_round = 0 + best_epoch = epoch + torch.save(copy.deepcopy(self.fore_model.state_dict()), save_path + "_fore_model.bin") + torch.save(copy.deepcopy(self.weight_model.state_dict()), save_path + "_weight_model.bin") + + else: + stop_round += 1 + if stop_round >= self.early_stop: + print("early stop") + break + + print("best loss:", best_loss, "@", best_epoch) + best_param = torch.load(save_path + "_fore_model.bin") + self.fore_model.load_state_dict(best_param) + best_param = torch.load(save_path + "_weight_model.bin") + self.weight_model.load_state_dict(best_param) + self.fitted = True + + if self.use_gpu: + torch.cuda.empty_cache() + + def predict(self, dataset): + if not self.fitted: + raise ValueError("model is not fitted yet!") + + x_test = dataset.prepare("test", col_set="feature") + index = x_test.index + self.fore_model.eval() + x_values = x_test.values + sample_num = x_values.shape[0] + preds = [] + + for begin in range(sample_num)[:: self.batch_size]: + + if sample_num - begin < self.batch_size: + end = sample_num + else: + end = begin + self.batch_size + + x_batch = torch.from_numpy(x_values[begin:end]).float().to(self.device) + + with torch.no_grad(): + if self.use_gpu: + pred = self.fore_model(x_batch).detach().cpu().numpy() + else: + pred = self.fore_model(x_batch).detach().numpy() + + preds.append(pred) + + return pd.Series(np.concatenate(preds), index=index) + + +class MLPModel(nn.Module): + def __init__(self, d_feat, hidden_size=256, num_layers=3, dropout=0.0, output_dim=1): + super().__init__() + + self.mlp = nn.Sequential() + self.softmax = nn.Softmax(dim=1) + + for i in range(num_layers): + if i > 0: + self.mlp.add_module("drop_%d" % i, nn.Dropout(dropout)) + self.mlp.add_module("fc_%d" % i, nn.Linear(d_feat if i == 0 else hidden_size, hidden_size)) + self.mlp.add_module("relu_%d" % i, nn.ReLU()) + + self.mlp.add_module("fc_out", nn.Linear(hidden_size, output_dim)) + + def forward(self, x): + # feature + # [N, F] + out = self.mlp(x).squeeze() + out = self.softmax(out) + return out + + +class GRUModel(nn.Module): + def __init__(self, d_feat=6, hidden_size=64, num_layers=2, dropout=0.0): + super().__init__() + + self.rnn = nn.GRU( + input_size=d_feat, + hidden_size=hidden_size, + num_layers=num_layers, + batch_first=True, + dropout=dropout, + ) + self.fc_out = nn.Linear(hidden_size, 1) + + self.d_feat = d_feat + + def forward(self, x): + # x: [N, F*T] + x = x.reshape(len(x), self.d_feat, -1) # [N, F, T] + x = x.permute(0, 2, 1) # [N, T, F] + out, _ = self.rnn(x) + return self.fc_out(out[:, -1, :]).squeeze() From b24af7fff6311a2ff0e8e5456b359febf5d6099c Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Mon, 24 May 2021 05:07:38 +0000 Subject: [PATCH 02/24] multiprocessing support --- .../model_rolling/task_manager_rolling.py | 10 ++- .../online_srv/online_management_simulate.py | 15 +++- .../online_srv/rolling_online_management.py | 22 ++++- qlib/model/trainer.py | 88 ++++++++++++++++++- qlib/workflow/online/manager.py | 29 +++--- qlib/workflow/task/manage.py | 7 +- 6 files changed, 145 insertions(+), 26 deletions(-) diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index 4f3ac04b1..89233b37b 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -4,6 +4,7 @@ """ This example shows how a TrainerRM works based on TaskManager with rolling tasks. After training, how to collect the rolling results will be shown in task_collecting. +Based on the ability of TaskManager, `worker` method offer a simple way for multiprocessing. """ from pprint import pprint @@ -13,10 +14,10 @@ import qlib from qlib.config import REG_CN from qlib.workflow import R from qlib.workflow.task.gen import RollingGen, task_generator -from qlib.workflow.task.manage import TaskManager +from qlib.workflow.task.manage import TaskManager, run_task from qlib.workflow.task.collect import RecorderCollector from qlib.model.ens.group import RollingGroup -from qlib.model.trainer import TrainerRM +from qlib.model.trainer import TrainerRM, task_train data_handler_config = { @@ -122,6 +123,11 @@ class RollingTaskExample: trainer = TrainerRM(self.experiment_name, self.task_pool) trainer.train(tasks) + def worker(self): + # train tasks by other progress or machines for multiprocessing. It is same as TrainerRM.worker. + print("========== worker ==========") + run_task(task_train, self.task_pool, experiment_name=self.experiment_name) + def task_collecting(self): print("========== task_collecting ==========") diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 4bb5022ee..de6dbcb21 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -78,8 +78,8 @@ class OnlineSimulationExample: provider_uri="~/.qlib/qlib_data/cn_data", region="cn", exp_name="rolling_exp", - task_url="mongodb://10.0.0.4:27017/", - task_db_name="rolling_db", + task_url="mongodb://10.0.0.4:27017/", # not necessary when using TrainerR or DelayTrainerR + task_db_name="rolling_db", # not necessary when using TrainerR or DelayTrainerR task_pool="rolling_task", rolling_step=80, start_time="2018-09-10", @@ -113,7 +113,7 @@ class OnlineSimulationExample: self.rolling_gen = RollingGen( step=rolling_step, rtype=RollingGen.ROLL_SD, ds_extra_mod_func=None ) # The rolling tasks generator, ds_extra_mod_func is None because we just need to simulate to 2018-10-31 and needn't change the handler end time. - self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) # Also can be TrainerR, TrainerRM, DelayTrainerR + self.trainer = TrainerRM(self.exp_name, self.task_pool) # Also can be TrainerR, TrainerRM, DelayTrainerR self.rolling_online_manager = OnlineManager( RollingStrategy(exp_name, task_template=tasks, rolling_gen=self.rolling_gen), trainer=self.trainer, @@ -139,6 +139,15 @@ class OnlineSimulationExample: print("========== signals ==========") print(self.rolling_online_manager.get_signals()) + def worker(self): + # train tasks by other progress or machines for multiprocessing + # FIXME: only can call after finishing simulation when using DelayTrainerRM, or there will be some exception. + print("========== worker ==========") + if isinstance(self.trainer, TrainerRM): + self.trainer.worker() + else: + print(f"{type(self.trainer)} is not supported for worker.") + if __name__ == "__main__": ## to run all workflow automatically with your own parameters, use the command below diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index 25b8b2a0c..40da30db7 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -13,10 +13,12 @@ Finally, the OnlineManager will finish second routine and update all strategies. import os import fire import qlib +from qlib.model.trainer import DelayTrainerR, DelayTrainerRM, TrainerR, TrainerRM, end_task_train, task_train from qlib.workflow import R from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.online.manager import OnlineManager +from qlib.workflow.task.manage import TaskManager, run_task data_handler_config = { "start_time": "2013-01-01", @@ -80,8 +82,9 @@ class RollingOnlineExample: self, provider_uri="~/.qlib/qlib_data/cn_data", region="cn", - task_url="mongodb://10.0.0.4:27017/", - task_db_name="rolling_db", + trainer=DelayTrainerRM(), # you can choose from TrainerR, TrainerRM, DelayTrainerR, DelayTrainerRM + task_url="mongodb://10.0.0.4:27017/", # not necessary when using TrainerR or DelayTrainerR + task_db_name="rolling_db", # not necessary when using TrainerR or DelayTrainerR rolling_step=550, tasks=[task_xgboost_config], add_tasks=[task_lgb_config], @@ -104,17 +107,28 @@ class RollingOnlineExample: RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD), ) ) - - self.rolling_online_manager = OnlineManager(strategies) + self.trainer = trainer + self.rolling_online_manager = OnlineManager(strategies, trainer=self.trainer) _ROLLING_MANAGER_PATH = ( ".RollingOnlineExample" # the OnlineManager will dump to this file, for it can be loaded when calling routine. ) + def worker(self): + # train tasks by other progress or machines for multiprocessing + print("========== worker ==========") + if isinstance(self.trainer, TrainerRM): + for task in self.tasks + self.add_tasks: + name_id = task["model"]["class"] + self.trainer.worker(experiment_name=name_id) + else: + print(f"{type(self.trainer)} is not supported for worker.") + # Reset all things to the first status, be careful to save important data def reset(self): for task in self.tasks + self.add_tasks: name_id = task["model"]["class"] + TaskManager(task_pool=name_id).remove() exp = R.get_exp(experiment_name=name_id) for rid in exp.list_recorders(): exp.delete_recorder(rid) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index fd76e6728..07bb839a2 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -12,9 +12,11 @@ In ``DelayTrainer``, the first step is only to save some necessary info to model """ import socket +import time from typing import Callable, List from qlib.data.dataset import Dataset +from qlib.log import get_module_logger from qlib.model.base import Model from qlib.utils import flatten_dict, get_cls_kwargs, init_instance_by_config from qlib.workflow import R @@ -190,6 +192,8 @@ class TrainerR(Trainer): Returns: List[Recorder]: a list of Recorders """ + if isinstance(tasks, dict): + tasks = [tasks] if len(tasks) == 0: return [] if train_func is None: @@ -213,6 +217,8 @@ class TrainerR(Trainer): Returns: List[Recorder]: the same list as the param. """ + if isinstance(recs, Recorder): + recs = [recs] for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs @@ -250,6 +256,8 @@ class DelayTrainerR(TrainerR): Returns: List[Recorder]: a list of Recorders """ + if isinstance(recs, Recorder): + recs = [recs] if end_train_func is None: end_train_func = self.end_train_func if experiment_name is None: @@ -315,6 +323,8 @@ class TrainerRM(Trainer): Returns: List[Recorder]: a list of Recorders """ + if isinstance(tasks, dict): + tasks = [tasks] if len(tasks) == 0: return [] if train_func is None: @@ -329,12 +339,24 @@ class TrainerRM(Trainer): run_task( train_func, task_pool, + query={"filter": {"$in": tasks}}, # only train these tasks experiment_name=experiment_name, before_status=before_status, after_status=after_status, **kwargs, ) + # FIXME: reset to waiting automatically + for _id in _id_list: + is_prn = False + while tm.re_query(_id)["status"] == "running": + if not is_prn: + get_module_logger("TrainerRM").warn( + f"A task (_id: {_id}) is not being trained by this Trainer. Ignore this message if it is being trained by others." + ) + is_prn = True + time.sleep(10) + recs = [] for _id in _id_list: rec = tm.re_query(_id)["res"] @@ -352,10 +374,33 @@ class TrainerRM(Trainer): Returns: List[Recorder]: the same list as the param. """ + if isinstance(recs, Recorder): + recs = [recs] for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs + def worker( + self, + train_func: Callable = None, + experiment_name: str = None, + ): + """ + The multiprocessing method for `train`. It can share a same task_pool with `train` and can run in other progress or other machines. + + Args: + train_func (Callable): the training method which needs at least `task`s and `experiment_name`. None for the default training method. + experiment_name (str): the experiment name, None for use default name. + """ + if train_func is None: + train_func = self.train_func + if experiment_name is None: + experiment_name = self.experiment_name + task_pool = self.task_pool + if task_pool is None: + task_pool = experiment_name + run_task(train_func, task_pool=task_pool, experiment_name=experiment_name) + class DelayTrainerRM(TrainerRM): """ @@ -395,6 +440,8 @@ class DelayTrainerRM(TrainerRM): Returns: List[Recorder]: a list of Recorders """ + if isinstance(tasks, dict): + tasks = [tasks] if len(tasks) == 0: return [] return super().train( @@ -410,8 +457,6 @@ class DelayTrainerRM(TrainerRM): Given a list of Recorder and return a list of trained Recorder. This class will finish real data loading and model fitting. - NOTE: This method will train all STATUS_PART_DONE tasks in the task pool, not only the ``recs``. - Args: recs (list): a list of Recorder, the tasks have been saved to them. end_train_func (Callable, optional): the end_train method which need at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. @@ -421,7 +466,8 @@ class DelayTrainerRM(TrainerRM): Returns: List[Recorder]: a list of Recorders """ - + if isinstance(recs, Recorder): + recs = [recs] if end_train_func is None: end_train_func = self.end_train_func if experiment_name is None: @@ -441,6 +487,42 @@ class DelayTrainerRM(TrainerRM): before_status=TaskManager.STATUS_PART_DONE, **kwargs, ) + + # FIXME: reset to waiting automatically + tm = TaskManager(task_pool=task_pool) + for query_task in tm.query({"filter": {"$in": tasks}}): + _id = query_task["_id"] + is_prn = False + while tm.re_query(_id)["status"] == "running": + if not is_prn: + get_module_logger("DelayTrainerRM").warn( + f"A task (_id: {_id}) is not being trained by this Trainer. Ignore this message if it is being trained by others." + ) + is_prn = True + time.sleep(10) + for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs + + def worker(self, end_train_func=None, experiment_name: str = None): + """ + The multiprocessing method for `end_train`. It can share a same task_pool with `end_train` and can run in other progress or other machines. + + Args: + end_train_func (Callable, optional): the end_train method which need at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. + experiment_name (str): the experiment name, None for use default name. + """ + if end_train_func is None: + end_train_func = self.end_train_func + if experiment_name is None: + experiment_name = self.experiment_name + task_pool = self.task_pool + if task_pool is None: + task_pool = experiment_name + run_task( + end_train_func, + task_pool=task_pool, + experiment_name=experiment_name, + before_status=TaskManager.STATUS_PART_DONE, + ) diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index 443cd61ad..ef6cb8dfa 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -18,10 +18,12 @@ There are 4 total situations for using different trainers in different situation ========================= =================================================================================== Situations Description ========================= =================================================================================== -Online + Trainer When you REAL want to do a routine, the Trainer will help you train the models. +Online + Trainer When you want to do a REAL routine, the Trainer will help you train the models. It + will train models task by task and strategy by strategy. -Online + DelayTrainer In normal online routine, whether Trainer or DelayTrainer will REAL train models - in this routine. So it is not necessary to use DelayTrainer when do a REAL routine. +Online + DelayTrainer When your models don't have any temporal dependence, the DelayTrainer will train + nothing until all tasks have been prepared. It makes user can train all tasks in + the end of `routine` or `first_train`. Simulation + Trainer When your models have some temporal dependence on the previous models, then you need to consider using Trainer. This means it will REAL train your models in @@ -103,17 +105,21 @@ class OnlineManager(Serializable): """ if strategies is None: strategies = self.strategies - for strategy in strategies: + models_list = [] + for strategy in strategies: self.logger.info(f"Strategy `{strategy.name_id}` begins first training...") tasks = strategy.first_tasks() models = self.trainer.train(tasks, experiment_name=strategy.name_id) - models = self.trainer.end_train(models, experiment_name=strategy.name_id) + models_list.append(models) self.logger.info(f"Finished training {len(models)} models.") - online_models = strategy.prepare_online_models(models, **model_kwargs) self.history.setdefault(self.cur_time, {})[strategy] = online_models + if not self.status == self.STATUS_SIMULATING or not self.trainer.is_delay(): + for strategy, models in zip(strategies, models_list): + models = self.trainer.end_train(models, experiment_name=strategy.name_id) + def routine( self, cur_time: Union[str, pd.Timestamp] = None, @@ -139,20 +145,22 @@ class OnlineManager(Serializable): cur_time = D.calendar(freq=self.freq).max() self.cur_time = pd.Timestamp(cur_time) # None for latest date + models_list = [] for strategy in self.strategies: self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") if self.status == self.STATUS_NORMAL: strategy.tool.update_online_pred() tasks = strategy.prepare_tasks(self.cur_time, **task_kwargs) - models = self.trainer.train(tasks) - if self.status == self.STATUS_NORMAL or not self.trainer.is_delay(): - models = self.trainer.end_train(models, experiment_name=strategy.name_id) + models = self.trainer.train(tasks, experiment_name=strategy.name_id) + models_list.append(models) self.logger.info(f"Finished training {len(models)} models.") online_models = strategy.prepare_online_models(models, **model_kwargs) self.history.setdefault(self.cur_time, {})[strategy] = online_models - if not self.trainer.is_delay(): + if not self.status == self.STATUS_SIMULATING or not self.trainer.is_delay(): + for strategy, models in zip(self.strategies, models_list): + models = self.trainer.end_train(models, experiment_name=strategy.name_id) self.prepare_signals(**signal_kwargs) def get_collector(self) -> MergeCollector: @@ -297,6 +305,7 @@ class OnlineManager(Serializable): # NOTE: Assumption: the predictions of online models need less than next cur_time, or this method will work in a wrong way. self.prepare_signals(**signal_kwargs) if signals_time > cur_time: + # FIXME: if use DelayTrainer and worker (and worker is faster than main progress), there are some possibilities of showing this warning. self.logger.warn( f"The signals have already parpred to {signals_time} by last preparation, but current time is only {cur_time}. This may be because the online models predict more than they should, which can cause signals to be contaminated by the offline models." ) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 658eec4d6..0e495bb0f 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -69,7 +69,7 @@ class TaskManager: ENCODE_FIELDS_PREFIX = ["def", "res"] - def __init__(self, task_pool: str = None): + def __init__(self, task_pool: str): """ Init Task Manager, remember to make the statement of MongoDB url and database name firstly. @@ -79,8 +79,7 @@ class TaskManager: the name of Collection in MongoDB """ self.mdb = get_mongodb() - if task_pool is not None: - self.task_pool = getattr(self.mdb, task_pool) + self.task_pool = getattr(self.mdb, task_pool) self.logger = get_module_logger(self.__class__.__name__) def list(self) -> list: @@ -288,7 +287,7 @@ class TaskManager: for t in self.task_pool.find(query): yield self._decode_task(t) - def re_query(self, _id): + def re_query(self, _id) -> dict: """ Use _id to query task. From ca0363ded804ad97d21d2d151ef823df9336a7c5 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 27 May 2021 06:04:46 +0000 Subject: [PATCH 03/24] update trainer and manage --- qlib/model/trainer.py | 38 ++++++++++++------------------------ qlib/workflow/task/manage.py | 34 ++++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 07bb839a2..ace3031ed 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -283,6 +283,9 @@ class TrainerRM(Trainer): STATUS_BEGIN = "begin_task_train" STATUS_END = "end_task_train" + # This tag is the _id in TaskManager to distinguish tasks. + TM_ID = "_id in TaskManager" + def __init__(self, experiment_name: str = None, task_pool: str = None, train_func=task_train): """ Init TrainerR. @@ -336,31 +339,24 @@ class TrainerRM(Trainer): task_pool = experiment_name tm = TaskManager(task_pool=task_pool) _id_list = tm.create_task(tasks) # all tasks will be saved to MongoDB + query = {"_id": {"$in": _id_list}} run_task( train_func, task_pool, - query={"filter": {"$in": tasks}}, # only train these tasks + query=query, # only train these tasks experiment_name=experiment_name, before_status=before_status, after_status=after_status, **kwargs, ) - # FIXME: reset to waiting automatically - for _id in _id_list: - is_prn = False - while tm.re_query(_id)["status"] == "running": - if not is_prn: - get_module_logger("TrainerRM").warn( - f"A task (_id: {_id}) is not being trained by this Trainer. Ignore this message if it is being trained by others." - ) - is_prn = True - time.sleep(10) + tm.wait(query=query) recs = [] for _id in _id_list: rec = tm.re_query(_id)["res"] rec.set_tags(**{self.STATUS_KEY: self.STATUS_BEGIN}) + rec.set_tags(**{self.TM_ID: _id}) recs.append(rec) return recs @@ -475,31 +471,21 @@ class DelayTrainerRM(TrainerRM): task_pool = self.task_pool if task_pool is None: task_pool = experiment_name - tasks = [] + _id_list = [] for rec in recs: - tasks.append(rec.load_object("task")) + _id_list.append(rec.list_tags()[self.TM_ID]) + query = {"_id": {"$in": _id_list}} run_task( end_train_func, task_pool, - query={"filter": {"$in": tasks}}, # only train these tasks + query=query, # only train these tasks experiment_name=experiment_name, before_status=TaskManager.STATUS_PART_DONE, **kwargs, ) - # FIXME: reset to waiting automatically - tm = TaskManager(task_pool=task_pool) - for query_task in tm.query({"filter": {"$in": tasks}}): - _id = query_task["_id"] - is_prn = False - while tm.re_query(_id)["status"] == "running": - if not is_prn: - get_module_logger("DelayTrainerRM").warn( - f"A task (_id: {_id}) is not being trained by this Trainer. Ignore this message if it is being trained by others." - ) - is_prn = True - time.sleep(10) + TaskManager(task_pool=task_pool).wait(query=query) for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 0e495bb0f..167087260 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -108,6 +108,15 @@ class TaskManager: def _dict_to_str(self, flt): return {k: str(v) for k, v in flt.items()} + def _decode_query(self, query): + if "_id" in query: + if isinstance(query["_id"], dict): + for key in query["_id"]: + query["_id"][key] = [ObjectId(i) for i in query["_id"][key]] + else: + query["_id"] = ObjectId(query["_id"]) + return query + def replace_task(self, task, new_task): """ Use a new task to replace a old one @@ -223,8 +232,7 @@ class TaskManager: dict: a task(document in collection) after decoding """ query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) query.update({"status": status}) task = self.task_pool.find_one_and_update( query, {"$set": {"status": self.STATUS_RUNNING}}, sort=[("priority", pymongo.DESCENDING)] @@ -282,8 +290,7 @@ class TaskManager: dict: a task(document in collection) after decoding """ query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) for t in self.task_pool.find(query): yield self._decode_task(t) @@ -338,8 +345,7 @@ class TaskManager: """ query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) self.task_pool.delete_many(query) def task_stat(self, query={}) -> dict: @@ -353,8 +359,7 @@ class TaskManager: dict """ query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) tasks = self.query(query=query, decode=False) status_stat = {} for t in tasks: @@ -376,8 +381,7 @@ class TaskManager: def reset_status(self, query, status): query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) print(self.task_pool.update_many(query, {"$set": {"status": status}})) def prioritize(self, task, priority: int): @@ -401,9 +405,19 @@ class TaskManager: return sum(task_stat.values()) def wait(self, query={}): + """ + When multiprocessing, the main progress may fetch nothing from TaskManager because there are still some running tasks. + So main progress should wait until all tasks are trained well by other progress or machines. + + Args: + query (dict, optional): the query dict. Defaults to {}. + """ task_stat = self.task_stat(query) total = self._get_total(task_stat) last_undone_n = self._get_undone_n(task_stat) + if last_undone_n == 0: + return + self.logger.warn(f"Waiting for {last_undone_n} undone tasks. Please make sure they are running.") with tqdm(total=total, initial=total - last_undone_n) as pbar: while True: time.sleep(10) From 94ab4bbf3feb5496720c6359dc85cfb1766ed5dd Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 1 Jun 2021 07:45:39 +0000 Subject: [PATCH 04/24] add docs --- qlib/workflow/task/manage.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 167087260..dd42caf65 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -24,7 +24,9 @@ from bson.binary import Binary from bson.objectid import ObjectId from pymongo.errors import InvalidDocument from qlib import auto_init, get_module_logger +import qlib from tqdm.cli import tqdm +import yaml from .utils import get_mongodb @@ -72,24 +74,26 @@ class TaskManager: def __init__(self, task_pool: str): """ Init Task Manager, remember to make the statement of MongoDB url and database name firstly. + A TaskManager instance serves a specific task pool. + The static method of this module serves the whole MongoDB. Parameters ---------- task_pool: str the name of Collection in MongoDB """ - self.mdb = get_mongodb() - self.task_pool = getattr(self.mdb, task_pool) + self.task_pool = getattr(get_mongodb(), task_pool) self.logger = get_module_logger(self.__class__.__name__) - def list(self) -> list: + @staticmethod + def list() -> list: """ - List the all collection(task_pool) of the db + List the all collection(task_pool) of the db. Returns: list """ - return self.mdb.list_collection_names() + return get_mongodb().list_collection_names() def _encode_task(self, task): for prefix in self.ENCODE_FIELDS_PREFIX: @@ -109,6 +113,16 @@ class TaskManager: return {k: str(v) for k, v in flt.items()} def _decode_query(self, query): + """ + If the query includes any `_id`, then it needs `ObjectId` to decode. + For example, when using TrainerRM, it needs query `{"_id": {"$in": _id_list}}`. Then we need to `ObjectId` every `_id` in `_id_list`. + + Args: + query (dict): query dict. Defaults to {}. + + Returns: + dict: the query after decoding. + """ if "_id" in query: if isinstance(query["_id"], dict): for key in query["_id"]: From ab6b88ce14814fe5679e4f5b5f9a016a2397c1a6 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 1 Jun 2021 07:48:14 +0000 Subject: [PATCH 05/24] delete useless import --- qlib/workflow/task/manage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index dd42caf65..7a85036da 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -24,9 +24,7 @@ from bson.binary import Binary from bson.objectid import ObjectId from pymongo.errors import InvalidDocument from qlib import auto_init, get_module_logger -import qlib from tqdm.cli import tqdm -import yaml from .utils import get_mongodb From 8d05cd2dafcb8f8dbf2bcdb453b5d9236d3bd766 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 1 Jun 2021 09:40:53 +0000 Subject: [PATCH 06/24] modify tests.config.py --- .../online_srv/online_management_simulate.py | 64 ++++++++++++- .../online_srv/rolling_online_management.py | 6 +- qlib/tests/config.py | 94 ++++++++++++++----- 3 files changed, 138 insertions(+), 26 deletions(-) diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 5f024192f..8650859ff 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -5,6 +5,7 @@ This example is about how can simulate the OnlineManager based on rolling tasks. """ +from pprint import pprint import fire import qlib from qlib.model.trainer import DelayTrainerR, DelayTrainerRM, TrainerR, TrainerRM @@ -13,7 +14,63 @@ from qlib.workflow.online.manager import OnlineManager from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.task.manage import TaskManager -from qlib.tests.config import CSI100_RECORD_LGB_TASK_CONFIG, CSI100_RECORD_XGBOOST_TASK_CONFIG +from qlib.tests.config import CSI100_RECORD_LGB_TASK_CONFIG_ONLINE, CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE + +data_handler_config = { + "start_time": "2018-01-01", + "end_time": "2018-10-31", + "fit_start_time": "2018-01-01", + "fit_end_time": "2018-03-31", + "instruments": "csi100", +} + +dataset_config = { + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": { + "class": "Alpha158", + "module_path": "qlib.contrib.data.handler", + "kwargs": data_handler_config, + }, + "segments": { + "train": ("2018-01-01", "2018-03-31"), + "valid": ("2018-04-01", "2018-05-31"), + "test": ("2018-06-01", "2018-09-10"), + }, + }, +} + +record_config = [ + { + "class": "SignalRecord", + "module_path": "qlib.workflow.record_temp", + }, + { + "class": "SigAnaRecord", + "module_path": "qlib.workflow.record_temp", + }, +] + +# use lgb model +task_lgb_config = { + "model": { + "class": "LGBModel", + "module_path": "qlib.contrib.model.gbdt", + }, + "dataset": dataset_config, + "record": record_config, +} + +# use xgboost model +task_xgboost_config = { + "model": { + "class": "XGBModel", + "module_path": "qlib.contrib.model.xgboost", + }, + "dataset": dataset_config, + "record": record_config, +} class OnlineSimulationExample: @@ -46,7 +103,10 @@ class OnlineSimulationExample: tasks (dict or list[dict]): a set of the task config waiting for rolling and training """ if tasks is None: - tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG, CSI100_RECORD_LGB_TASK_CONFIG] + #tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE, CSI100_RECORD_LGB_TASK_CONFIG_ONLINE] + tasks = [task_xgboost_config, task_lgb_config] + #pprint(CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE) + #pprint(task_xgboost_config) self.exp_name = exp_name self.task_pool = task_pool self.start_time = start_time diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index b4f7245b7..99a91e027 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -18,7 +18,7 @@ from qlib.workflow import R from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.online.manager import OnlineManager -from qlib.tests.config import CSI100_RECORD_XGBOOST_TASK_CONFIG, CSI100_RECORD_LGB_TASK_CONFIG +from qlib.tests.config import CSI100_RECORD_XGBOOST_TASK_CONFIG_ROLLING, CSI100_RECORD_LGB_TASK_CONFIG_ROLLING class RollingOnlineExample: @@ -34,9 +34,9 @@ class RollingOnlineExample: add_tasks=None, ): if add_tasks is None: - add_tasks = [CSI100_RECORD_LGB_TASK_CONFIG] + add_tasks = [CSI100_RECORD_LGB_TASK_CONFIG_ROLLING] if tasks is None: - tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG] + tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG_ROLLING] mongo_conf = { "task_url": task_url, # your MongoDB url "task_db_name": task_db_name, # database name diff --git a/qlib/tests/config.py b/qlib/tests/config.py index 80461f6f9..c61b5651e 100644 --- a/qlib/tests/config.py +++ b/qlib/tests/config.py @@ -43,17 +43,29 @@ RECORD_CONFIG = [ ] -def get_data_handler_config(market=CSI300_MARKET): +def get_data_handler_config( + start_time="2008-01-01", + end_time="2020-08-01", + fit_start_time="2008-01-01", + fit_end_time="2014-12-31", + instruments=CSI300_MARKET, +): return { - "start_time": "2008-01-01", - "end_time": "2020-08-01", - "fit_start_time": "2008-01-01", - "fit_end_time": "2014-12-31", - "instruments": market, + "start_time": start_time, + "end_time": end_time, + "fit_start_time": fit_start_time, + "fit_end_time": fit_end_time, + "instruments": instruments, } -def get_dataset_config(market=CSI300_MARKET, dataset_class=DATASET_ALPHA158_CLASS): +def get_dataset_config( + dataset_class=DATASET_ALPHA158_CLASS, + train=("2008-01-01", "2014-12-31"), + valid=("2015-01-01", "2016-12-31"), + test=("2017-01-01", "2020-08-01"), + handler_kwargs={"instruments": CSI300_MARKET}, +): return { "class": "DatasetH", "module_path": "qlib.data.dataset", @@ -61,48 +73,88 @@ def get_dataset_config(market=CSI300_MARKET, dataset_class=DATASET_ALPHA158_CLAS "handler": { "class": dataset_class, "module_path": "qlib.contrib.data.handler", - "kwargs": get_data_handler_config(market), + "kwargs": get_data_handler_config(**handler_kwargs), }, "segments": { - "train": ("2008-01-01", "2014-12-31"), - "valid": ("2015-01-01", "2016-12-31"), - "test": ("2017-01-01", "2020-08-01"), + "train": train, + "valid": valid, + "test": test, }, }, } -def get_gbdt_task(market=CSI300_MARKET): +def get_gbdt_task(dataset_kwargs={}, handler_kwargs={"instruments": CSI300_MARKET}): return { "model": GBDT_MODEL, - "dataset": get_dataset_config(market), + "dataset": get_dataset_config(**dataset_kwargs, handler_kwargs=handler_kwargs), } -def get_record_lgb_config(market=CSI300_MARKET): +def get_record_lgb_config(dataset_kwargs={}, handler_kwargs={"instruments": CSI300_MARKET}): return { "model": { "class": "LGBModel", "module_path": "qlib.contrib.model.gbdt", }, - "dataset": get_dataset_config(market), + "dataset": get_dataset_config(**dataset_kwargs, handler_kwargs=handler_kwargs), "record": RECORD_CONFIG, } -def get_record_xgboost_config(market=CSI300_MARKET): +def get_record_xgboost_config(dataset_kwargs={}, handler_kwargs={"instruments": CSI300_MARKET}): return { "model": { "class": "XGBModel", "module_path": "qlib.contrib.model.xgboost", }, - "dataset": get_dataset_config(market), + "dataset": get_dataset_config(**dataset_kwargs, handler_kwargs=handler_kwargs), "record": RECORD_CONFIG, } -CSI300_DATASET_CONFIG = get_dataset_config(market=CSI300_MARKET) -CSI300_GBDT_TASK = get_gbdt_task(market=CSI300_MARKET) +CSI300_DATASET_CONFIG = get_dataset_config(handler_kwargs={"instruments": CSI300_MARKET}) +CSI300_GBDT_TASK = get_gbdt_task(handler_kwargs={"instruments": CSI300_MARKET}) -CSI100_RECORD_XGBOOST_TASK_CONFIG = get_record_xgboost_config(market=CSI100_MARKET) -CSI100_RECORD_LGB_TASK_CONFIG = get_record_lgb_config(market=CSI100_MARKET) +CSI100_RECORD_XGBOOST_TASK_CONFIG = get_record_xgboost_config(handler_kwargs={"instruments": CSI100_MARKET}) +CSI100_RECORD_LGB_TASK_CONFIG = get_record_lgb_config(handler_kwargs={"instruments": CSI100_MARKET}) + +# use for rolling_online_managment.py +ROLLING_HANDLER_CONFIG = { + "start_time": "2013-01-01", + "end_time": "2020-09-25", + "fit_start_time": "2013-01-01", + "fit_end_time": "2014-12-31", + "instruments": CSI100_MARKET, +} +ROLLING_DATASET_CONFIG = { + "train": ("2013-01-01", "2014-12-31"), + "valid": ("2015-01-01", "2015-12-31"), + "test": ("2016-01-01", "2020-07-10"), +} +CSI100_RECORD_XGBOOST_TASK_CONFIG_ROLLING = get_record_xgboost_config( + dataset_kwargs=ROLLING_DATASET_CONFIG, handler_kwargs=ROLLING_HANDLER_CONFIG +) +CSI100_RECORD_LGB_TASK_CONFIG_ROLLING = get_record_lgb_config( + dataset_kwargs=ROLLING_DATASET_CONFIG, handler_kwargs=ROLLING_HANDLER_CONFIG +) + +# use for online_management_simulate.py +ONLINE_HANDLER_CONFIG = { + "start_time": "2018-01-01", + "end_time": "2018-10-31", + "fit_start_time": "2018-01-01", + "fit_end_time": "2018-03-31", + "instruments": CSI100_MARKET, +} +ONLINE_DATASET_CONFIG = { + "train": ("2018-01-01", "2018-03-31"), + "valid": ("2018-04-01", "2018-05-31"), + "test": ("2018-06-01", "2018-09-10"), +} +CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE = get_record_xgboost_config( + dataset_kwargs=ONLINE_DATASET_CONFIG, handler_kwargs=ONLINE_HANDLER_CONFIG +) +CSI100_RECORD_LGB_TASK_CONFIG_ONLINE = get_record_lgb_config( + dataset_kwargs=ONLINE_DATASET_CONFIG, handler_kwargs=ONLINE_HANDLER_CONFIG +) From b2fe2385d561e0169abed9a3d859e3e0af789ac2 Mon Sep 17 00:00:00 2001 From: zhupr Date: Tue, 1 Jun 2021 21:02:32 +0800 Subject: [PATCH 07/24] fix XGBoost predict error --- qlib/contrib/model/xgboost.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qlib/contrib/model/xgboost.py b/qlib/contrib/model/xgboost.py index 2a38f4fe1..300326143 100755 --- a/qlib/contrib/model/xgboost.py +++ b/qlib/contrib/model/xgboost.py @@ -62,7 +62,7 @@ class XGBModel(Model, FeatureInt): if self.model is None: raise ValueError("model is not fitted yet!") x_test = dataset.prepare(segment, col_set="feature", data_key=DataHandlerLP.DK_I) - return pd.Series(self.model.predict(xgb.DMatrix(x_test.values)), index=x_test.index) + return pd.Series(self.model.predict(xgb.DMatrix(x_test)), index=x_test.index) def get_feature_importance(self, *args, **kwargs) -> pd.Series: """get feature importance From 811d2c975e4c277651e3e87220ac4c36eb63d8d4 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 2 Jun 2021 08:56:15 +0000 Subject: [PATCH 08/24] update & fix --- .../online_srv/online_management_simulate.py | 61 +------------------ .../online_srv/rolling_online_management.py | 1 + qlib/workflow/online/manager.py | 7 ++- 3 files changed, 7 insertions(+), 62 deletions(-) diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 8650859ff..bd7c4675d 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -16,62 +16,6 @@ from qlib.workflow.task.gen import RollingGen from qlib.workflow.task.manage import TaskManager from qlib.tests.config import CSI100_RECORD_LGB_TASK_CONFIG_ONLINE, CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE -data_handler_config = { - "start_time": "2018-01-01", - "end_time": "2018-10-31", - "fit_start_time": "2018-01-01", - "fit_end_time": "2018-03-31", - "instruments": "csi100", -} - -dataset_config = { - "class": "DatasetH", - "module_path": "qlib.data.dataset", - "kwargs": { - "handler": { - "class": "Alpha158", - "module_path": "qlib.contrib.data.handler", - "kwargs": data_handler_config, - }, - "segments": { - "train": ("2018-01-01", "2018-03-31"), - "valid": ("2018-04-01", "2018-05-31"), - "test": ("2018-06-01", "2018-09-10"), - }, - }, -} - -record_config = [ - { - "class": "SignalRecord", - "module_path": "qlib.workflow.record_temp", - }, - { - "class": "SigAnaRecord", - "module_path": "qlib.workflow.record_temp", - }, -] - -# use lgb model -task_lgb_config = { - "model": { - "class": "LGBModel", - "module_path": "qlib.contrib.model.gbdt", - }, - "dataset": dataset_config, - "record": record_config, -} - -# use xgboost model -task_xgboost_config = { - "model": { - "class": "XGBModel", - "module_path": "qlib.contrib.model.xgboost", - }, - "dataset": dataset_config, - "record": record_config, -} - class OnlineSimulationExample: def __init__( @@ -103,10 +47,7 @@ class OnlineSimulationExample: tasks (dict or list[dict]): a set of the task config waiting for rolling and training """ if tasks is None: - #tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE, CSI100_RECORD_LGB_TASK_CONFIG_ONLINE] - tasks = [task_xgboost_config, task_lgb_config] - #pprint(CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE) - #pprint(task_xgboost_config) + tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE, CSI100_RECORD_LGB_TASK_CONFIG_ONLINE] self.exp_name = exp_name self.task_pool = task_pool self.start_time = start_time diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index 99a91e027..6abbbfb0e 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -19,6 +19,7 @@ from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.online.manager import OnlineManager from qlib.tests.config import CSI100_RECORD_XGBOOST_TASK_CONFIG_ROLLING, CSI100_RECORD_LGB_TASK_CONFIG_ROLLING +from qlib.workflow.task.manage import TaskManager class RollingOnlineExample: diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index ef6cb8dfa..dc1186038 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -163,17 +163,20 @@ class OnlineManager(Serializable): models = self.trainer.end_train(models, experiment_name=strategy.name_id) self.prepare_signals(**signal_kwargs) - def get_collector(self) -> MergeCollector: + def get_collector(self, **kwargs) -> MergeCollector: """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. This collector can be a basis as the signals preparation. + + Args: + **kwargs: the params for get_collector. Returns: MergeCollector: the collector to merge other collectors. """ collector_dict = {} for strategy in self.strategies: - collector_dict[strategy.name_id] = strategy.get_collector() + collector_dict[strategy.name_id] = strategy.get_collector(**kwargs) return MergeCollector(collector_dict, process_list=[]) def add_strategy(self, strategies: Union[OnlineStrategy, List[OnlineStrategy]]): From 8222795ac4ae51f504aa1ace432fe0ff69b3bbf1 Mon Sep 17 00:00:00 2001 From: Young Date: Wed, 2 Jun 2021 09:16:46 +0000 Subject: [PATCH 09/24] fix format with black --- qlib/workflow/online/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index dc1186038..d3cc0cbf8 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -167,7 +167,7 @@ class OnlineManager(Serializable): """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. This collector can be a basis as the signals preparation. - + Args: **kwargs: the params for get_collector. From 1320e53f8172a7abc54dc8a6753f417edc47430a Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 3 Jun 2021 03:23:48 +0000 Subject: [PATCH 10/24] fix DelayTrainerRM --- qlib/model/trainer.py | 3 ++- qlib/workflow/online/manager.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index ace3031ed..28d854477 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -350,7 +350,8 @@ class TrainerRM(Trainer): **kwargs, ) - tm.wait(query=query) + if not self.is_delay(): + tm.wait(query=query) recs = [] for _id in _id_list: diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index dc1186038..d3cc0cbf8 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -167,7 +167,7 @@ class OnlineManager(Serializable): """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. This collector can be a basis as the signals preparation. - + Args: **kwargs: the params for get_collector. From ed54f1213c717950b38ca9da7be95f64b20c9ddc Mon Sep 17 00:00:00 2001 From: Jactus Date: Mon, 7 Jun 2021 17:13:36 +0800 Subject: [PATCH 11/24] Fix exception hook bug --- qlib/workflow/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/qlib/workflow/utils.py b/qlib/workflow/utils.py index 596ff0927..5a93eacca 100644 --- a/qlib/workflow/utils.py +++ b/qlib/workflow/utils.py @@ -17,7 +17,6 @@ def experiment_exit_handler(): Thus, if any exception or user interuption occurs beforehead, we should handle them first. Once `R` is ended, another call of `R.end_exp` will not take effect. """ - signal.signal(signal.SIGINT, experiment_kill_signal_handler) # handle user keyboard interupt sys.excepthook = experiment_exception_hook # handle uncaught exception atexit.register(R.end_exp, recorder_status=Recorder.STATUS_FI) # will not take effect if experiment ends @@ -39,10 +38,3 @@ def experiment_exception_hook(type, value, tb): print(f"{type.__name__}: {value}") R.end_exp(recorder_status=Recorder.STATUS_FA) - - -def experiment_kill_signal_handler(signum, frame): - """ - End an experiment when user kill the program through keyboard (CTRL+C, etc.). - """ - R.end_exp(recorder_status=Recorder.STATUS_FA) From 7d9544fb91fc350f29b2dba4bec5d6424f75e8ef Mon Sep 17 00:00:00 2001 From: al Date: Tue, 8 Jun 2021 09:35:36 +0800 Subject: [PATCH 12/24] Remove non-existing parameter from doc Remove non-existing TradeExchange parameter from generate_target_weight_position doc --- qlib/contrib/strategy/strategy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qlib/contrib/strategy/strategy.py b/qlib/contrib/strategy/strategy.py index 4f8eb0ab1..74290b7d6 100644 --- a/qlib/contrib/strategy/strategy.py +++ b/qlib/contrib/strategy/strategy.py @@ -148,7 +148,6 @@ class WeightStrategyBase(BaseStrategy, AdjustTimer): pred score for this trade date, index is stock_id, contain 'score' column. current : Position() current position. - trade_exchange : Exchange() trade_date : pd.Timestamp trade date. """ From 492a62a569e2a64dfb9e79e61e61cffcce8ae61f Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 17:32:24 +0800 Subject: [PATCH 13/24] tcts demo page --- examples/benchmarks/TCTS/TCTS.md | 50 ++++++++++++++++++ examples/benchmarks/TCTS/task_description.png | Bin 0 -> 25240 bytes examples/benchmarks/TCTS/workflow.png | Bin 0 -> 29877 bytes 3 files changed, 50 insertions(+) create mode 100644 examples/benchmarks/TCTS/TCTS.md create mode 100644 examples/benchmarks/TCTS/task_description.png create mode 100644 examples/benchmarks/TCTS/workflow.png diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md new file mode 100644 index 000000000..e8cbc8005 --- /dev/null +++ b/examples/benchmarks/TCTS/TCTS.md @@ -0,0 +1,50 @@ +# Temporally Correlated Task Scheduling for Sequence Learning +We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://www.overleaf.com/project/5eb8efb42dcf710001d781d6). + +### Background +Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. + +![Temporally Correlated Tasks.](task_description.png) + + +### Method +Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. +![The optimization workflow of one episode.](workflow.png) + + +At step $$s$$, with training data $$(x_s,y_s)$$, the scheduler $$\varphi(\cdots;\omega)$$ chooses a suitable task $$T_{i_s}$$ (green solid lines) to update the model $$f(\cdots;\theta)$$ (blue solid lines). After $$S$$ steps, we evaluate the model $$f$$ on the validation set $$\Ddev$$ and update the scheduler $$\varphi$$ (green dashed lines). + +### DataSet +* We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. +* We split the data into training (01/01/2008-12/31/2013), validation (01/01/2014-12/31/2015), and test sets (01/01/2016-08/01/2020) based on the transaction time. + +### Experiments +#### Task Description +* The main tasks $$T_k$$ ($$task_k$$ in Figure1) refers to forecasting return of stock $$i$$ as following, + +```math +$$ +r_{i}^k = \frac{\price_i^{t+k}}{\price_i^{t+k-1}} - 1 +$$ +``` +* Temporally correlated task sets $$\domT_k = {T_1, T_2, ... , T_k}$$, in this paper, $$\domT_3$$, $$\domT_5$$ and $$\domT_10$$ are used. +#### Baselines +* GRU/MLP/LightGBM (LGB)/Graph Attention Networks (GAT) +* Multi-task learning (MTL): In multi-task learning, multiple tasks are jointly trained and mutually boosted. Each task is treated equally, while in our setting, we focus on the main task. +* Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task T_1, then T_2, and gradually move to the last one. +#### Result +| Methods | $$T_1$$ | $$T_2$$ | $$T_3$$ | +| :----: | :----: | :----: | :----: | +| GRU | 0.049 / 1.903 | 0.018 / 1.972 | 0.014 / 1.989 | +| MLP | 0.023 / 1.961 | 0.022 / 1.962 | 0.015 / 1.978 | +| LGB | 0.038 / 1.883 | 0.023 / 1.952 | 0.007 / 1.987 | +| GAT | 0.052 / 1.898 | 0.024 / 1.954 | 0.015 / 1.973 | +| MTL($$\domT_3$$) | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | +| CL($$\domT_3$$) | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | +| Ours($$\domT_3$$) | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | +| MTL($$\domT_5$$) | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | +| CL($$\domT_5$$) | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | +| Ours($$\domT_5$$) | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | +| MTL($$\domT_{10}$$) | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | +| CL($$\domT_{10}$$) | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | +| Ours($$\domT_{10}$$) | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file diff --git a/examples/benchmarks/TCTS/task_description.png b/examples/benchmarks/TCTS/task_description.png new file mode 100644 index 0000000000000000000000000000000000000000..7a9005bf24869c6c88bec34513a67704743492ee GIT binary patch literal 25240 zcmdpd*I!f57cHI8n+ix5Lk(RafOLc}M7q*@N4oS9x*$po5J0+!fItYnccch}CN)&) zz4vzG??1Q?_vJq1BzfXKEME9+&t#>GaC++Lz0;c#_fP<^MNrEB-yN3! zp1hg5kEHB0aL<5ka924m`C9BfFS=3l=?G2jR0Q8LJ@)~GCf2NzGE#G@zqIh#>pi;-;mjy#ILrTJKT=}l1 zsGTp9VHISHWzQqrl&s-UFTYS&Kyneo--@kt` zHSNVmN)(Yh@p28~?&ZqeaDH%CT(f_4?%TaDtS@(ajZ%G0jbu>gSoxijSkKldM*=ti0?^50IG` zCTSe+d8ntlxl|kGowS8c)rSL557QwSXUxPQj(yyaaV6a*Cc32SHb7ZP_POC9Z#7F5 z!Si|UG2H_sTC!2JgX%pm#6spRY-uTy!L%S^U#g;;dWxF)<_bCj#{~1b z>vX&uZZB!je%P@6oO;-UtL^3Q247OJ{;FQ)X$Cs)+P z&kS^{D7^oqLa58yWP_4K}8N7B?*1Dj{B7WI3AOq`+5R09L0J zm&u@zbw0>u$$21&tFguY+i%b}f!0^>+>CwH_d1MS(K5Dw#! zyZcgZI^WludWm3N<~ivA&Mv3wmcywA0}>8P%1q}!t?4e_!6!;6JEN1HU6iOTPC9#| z^-+zkDgtPIe`F@cJcIBaE2clJSH`-_KS$fH)(zBSk35HA8>TH1{_t^LxW29$_Hk@q z$hoVjzpP{aopj50kwAE>bVawqf6AtZvx{n>zuqm`Anu@@HEko_7d%@9GID=XQ+>6A ztTH5%F9%n-Bix9ew>QpuwtxT6h$+!*<>5Cq1?!%T0P4;9wty<#Y7Z=rb-)K<$)qeZ zCVyD0`(MJAotbp?cgIS+cSNNw=F_Mgonv<((`r0jZ)Mnl&a;HON?17$5ymq+FN+yb>bYn2)CNCuc&J{y^x>m= z)7KeOqfkj1w5$rHQKshUuv?7B!@Oz~ zb3J5CkE4KDEe*f>;>Nh0i$0k!T&HtTuw=Q@d=V<$o6BSqNn?p5XN%C0|M9wKEkpHB zce7Q$lcyNI{yxXs44a1KpUc;aO^WTS7fTBTiX4tiL`av<%DVMc#}J5gQ+GbD(%shz zPv_zt6=3;xEYdJkbF-Yx24VeU62EBnfFIaXB`Jb$^?9-BP%475(<@Tz3EgFE)hISG z#~jnflWRmF(GJEH!C|;!k9`N*c>~xPGkmxb8}374$)7Q^U~T@<`Et}Qw1Wwv1f_E7 z|Le;PH!P)8$MsUU5ICxm*zc*vBWT>{s9P2UIR-qLgrqq65B>*OmZiotJXK)k1I7$Q zeU6>zD9YFhJ3unMG|Yf^cB)r9SNr_#{7XpCx5FO(PMjm}`iT_OJddDVk$C)<#H=S*n(qo%|z7@%&fq)MvV+ zRkCtge1vrHqkz~j?T$Z}9uh&p!%UtOe14zAI-dpk9Da$rH(8|uW-z~f8dv!?$M9Kr z0ai9Ze?mhVX+20BNvWQIr*FYB7|AEz*A1Y&Hc+yVnsuB9SG`DDOFy%(Q#+G$Xq%_9 zPn>*B@2~>?zuqzR@YUq;!B zF6i0D6Xl0U+}wc5BDM&TXV|*`_VZ4nBrM*4Pr4kT67p}4re&+mh&#zT!jxv`ei=XZ-9cy7 z)&|Vps&S|^{pCWyF9Q}(YWfj+bp$)$x!Rs6L3Xp3-_7^vs50vxmjNF(V=2AauqS7Z z8G&uu$HXOi7;{j(@mvi)Dq*Ea8X_d>$=<^pP}l%-9l@f}a_++r&vR7|Gvm7w_y<2Q z=G)x+e7W`PcTBOy!FVBC1d<&h_xLIu)&yVWK>qIzsdU_u9A3Y0STgE=O&CArs;W&0 zz~_b-{s)PceQ&k<=i~3m{&&Wf8BjUN$S;}wd(H>@?sdNm;~hh1!_qHE_Ri)a0|wL_ z#QpD1l-OroDLT!TUjhE?j3hEl$5p^K;|~~6WaWMXTNKF4VOfsF3~!-lonYFw`b)PQ zA=9SY8Jabf^SseM(4_w%IdEiqVy%5xe!gK9RC&Fqe>@BVs1CZafly=9q2Ez0s zB|jPW2a3X2P?Xtus@G9~sfIwu1$opFrT+b3M!*irCEueEO01Va^?hv|ANf+9(IRk~WH+FG=F1ayn7j9?!&`u5|%Pc>Wnuh{mZViH(Tm9?Twi&M1uS;etfkuPE`lW-NmHyPn>Yr=BxFZBgKV-ouc&|EPgz&*4xnml(;CGe*dkE*n@7Ak) zs^A+9qZWyux0sB-PE{#VDA_m9$TG<~|GJe?tqZxZ5l`3;=5IZz;c=zLx_A>mK(k z)XRc>3?6mFzX9~3g`d?QH?+%%DnyC_~Vi7Y--i`*K9RDT5Re6sXAz*y*d01h?~ zCOoYyc`*F2Gf~1`UD4kq`Ol~10VIJRh&E2)C3<4YCh);tjpS;PFq=*p{))FEc1_ax z5x7ZS4tScCr0~F}K77BQ>7y!LLji<#273jn{Lh4W5hjt=L{nS-__^gPvDgN*2gBGq zg{i*5rHM4MfV6dQZGa2fw3B-9PCt5W>Wgpc`krca;hP<59CkVBEc)`{%AC^Tn8m>P z10(s3MccD&CA|0WM+{IK?U8{mIL#LVR}-+RB0W+}I1EJ&K5r5tkY*3Vo=FlXIqU?6 zv5VZMi}7gL0VQp=T<57jm&QVko~$52!^VGPb1QbYXD5F0Eq}$AHEZ$rV&9Gc(NI6c z1H$m>!-i3(d~BOFp1{7bb>_df^@0e6Int|w6_h)FyDrF@PQ-5-&Tby1zd zA6eEt)^1Z<8%z+!8#KiFzO}V`h^{gjGxeQ@VCDhsZ4LO+_dtZ8iTlB)0_o7src%@S z%$t;=jH5dZzC^7wE%0DyKXKrG&BJ41*OeL7?1RlkkB>1?vm%fPX3YjX!aa$N0tra4 zuuwcG9T#Vy;(ZMs_4bVEcWWYunXpJacSY89emTfMUNetuv3YeFh+{;-U1RKLI+DHk z))EJakm3UlAW>d6pT}!{^>{hm<~EN}3n zCQ7vOMNzF<*%Y;_;Ws6f#JV*L1OCv#i`3E*uf~PGRqo$DBEPhkIAy;Z!HbnQd9o&- zxV+MjBAuv90c09qFn)&yMML#}B)>~BseT345FM%UfzwE3E12nj^E~;~%JtDs@N#(D!Lp)>F1KK9?rfC>zNeCX&w?dZ5I| znm((UD65cP%Z|ehEnm^e_0sWmbhx0PunXbVt|us@6BR(>0|@-XHcf0gH>vr4!x7?{ok% z1mEUet_u#bb-onDZ&>p3DB;3j@`0!DXt2pk&Ak`a>39a23520R=KbZV00PiQ_x1PcuNb~0C2?0caZzp$ zekzT_g48;aHiJHjpZgj$H_{&%M7o2pr?knw|hX|1pI66mF-8R zC3A80#jMomEc1nz2L3t}Yom|$s~L9=HW5JSc=`&^hJl4}0en6C?YBu)vOK%@a!R{Y zK&gh@q-zMJ7(poqS*6v=5MWjxX;VDHR8d(Dr?C4oNaU@?{D%uA=Y_#N5OTsEIrmPu z4(KuJ3`jfzjg53cFqxOS9oemZ_G{P9%B+$LH}P@=XlYHypf?Zt>aLrp{&s3V1_BSz>? z5_Ra|*s&AFLSC#w3(;YEfuZV_4H5b{lvBq8{L(fGBum41IT*BY@bY}??oCOm{FV37)>bGcFybi|+IFl65rtI3pY|OM#<~zZ1rbrlA;rG=ehV81_8Ax{UCs zzfnJYJ`N@vj%`}Cw&}UfJCWb%l8}<$LHj~y?Nns;y3Wt*RAl&QuLJK(hwV&Qw*!RR zt`*2qi6RH}Fwf_>%Bzj-2{N%;1E^SZD9RC91tUh2UQT}I{$MMO5vi)|cw|f1qDRYgh z!pEJ!9PQ41I^{aPnaHEdC}5k(hXB`15$Wdcn!pK6Yjfet79flEF+K3X_}CFx$4#`X z`i$h*6=+t0!Tc8O50x0kah-JNUGJMipz1whwH(rewF87?8y;_!=ta5r!t5{%pMJ9! zQ5QcArFy&`w|K0Oe0fm8>V1ixa-Zd$LO(;PsY!5l;i^sCz0_ko9AW7;1IrIcNQhS* z?18`H?jT?#(|v;)>U}U#QnjoF8nUWzwqk$_^-L;n=)t(?M? z=D*wTz>lSiB1U~rFbT|;x0O~Idl5F-n!<$oy|2yB)PMp=f3HA(h{=-QYwrt5h`Lq7 znmPvfAN!Z+ZG>Yo$<+K{jTBeDnV0R<$x!?Ls1&#QaxzVeBhRIrYn2bJ`U?K>^F5rw7sPSRcnj*>2mZ z>3hdz*R@19nh!03PGcfvrdcc?pZ?>AH~m)4R4$$Ks{Ck0h~`}f@WJpnnK~#oQ;EX> z4Grx7Qfitd-9!IScB`lovh#Obz=&mkNCjViENm=M;O((8uubzDvubPyOY)1S_Zbjo z6HEr!rgz*5)Bs(L!WxRS%Q`iZi&$Vm)fye>x@5q}QzmXe)t|G%1+M!7MO6a{xX$Pm z=SOc3#Bhs$4byMiVjP06Yj0jxK=xeG+4ulzXGsab)O&`5l85Q)7Z*;z>j?V;FL0gL z)H7di(2d(%q=oMQHFV__)TK#{e6pd*Wn(RPN*v{DTsS4oGOE5%7u@>{<)~z2PlMhQ zX+MiTk*{mGH`z#QIq*RD&kZS!x$m9mV%*5N1iF4Q%4z!*+6t-n-mqmr^VgKu9;V;L zqa~wyD!w1d)F|F~OYT`N?Ip!c2dShs4qZQObY4Gdj_??bY`K%}NT}UBuVU%-k(%v* zT`E9a^uDn;jYH?AYva}0;5}C3==WmI6>4?613}94eZT?8{rxySW14cjrD zH=fB$mO4s87reiF`7X3Bn!WEe@jHOI3M4w^;>uun{j^L^wRF+jPmbS)_7<<=8#k^` zTP|vxcLv!oS!aBTJxi#rS?YDe$%~oB!?`>~PyR~-dCnl8=$gB!v!0ipgTe4VKoRW0qS^tBhsuaO*Bjy6Yny(H_U(46P*_3>QW3@iKD1bc*CQA5ICN#|XY z>vgdg^-@n?>UfF)E^iAVq&>Z<)3qYuF>r$f&r+PX9i^Sa1*9Pa-c)>Ol>H;6}}f=j?wXVSTK*DTl80=saICAkQ+x5uMffr zIM(rFtkMA!Fae%X{;VC};) zwf_$dJ`MfGt6G@&p~YO=kBF1=h8_1gZ(`Efn-p+sP^Ga8?C1d4vMOh{kk@_DTDfWm z*+ejq_nZ_km}ws$!d$toPN*g3eF7!K#FB5VqYWx{Msqkor|a&(7_QB1=^wZerkmx? zap9&Ct?qkYAKQ6QTB%K#Kz4UhRT=e+{O^Q-JDChI>?j!}@1Apqs*R90tl?HnShX}aS1-N}098?g#{;rBvui)GzjV}u(k#x);3YRcRl z7Z)B%scGgoQ<254hJQ1eHUHia4=PW;Jy&jW(1`ES3Rn+dN2RPdXH^JDRF&oCkRcb9 z;b5P}<(dlDxd#3fovbp+Uo+BHULX>oQJ)~E5(b~sl8W&aSxGt)r__oYT_Akdn)>SL zfrEhDSD;j{8^4QXy4F|t%#t!z>jILh$@LkJ{F6t% z{f@1EZ^wu4F;9CNxZ5iqv4cnWW!`oA3Azvna`uAhu+BS#`l?kbgr8-y;r80AUT6n- zG0c|RImNIQ-dq6pJdY4S9PYZ`UH3!w)oXO-4!y$y?m<=|1;Nq{5uxKUBXKd6tc|H3 zhVJPk3BG!mMcr|$uk|qwyfmtONAg_O>Xxp7MPQV}D#R;?8eM}*dweI`4^E>uiYvEU zWlRr&sXP5_TXvF0bc+ii8S`o3#%5Fg>%X^>q*}e^+Z7#D zs1dW>xZC~7+~DDTylCqwMu>zrEOav9?%#XPH~mR-OY4bu@~IPmPQY;x&VWsPoY>Y1 zft4a53wPOKPI%qf%1ACh&23%IKk@LxR< zYX88vGTDm$L5#xJ=utJRJJ0jLX_=(oQ>$aj=X{poX57nyPK~|zdY{o8Xb~;>Zsxn` z&=Ti#(vgsqtFa}HJ@wl3>?k|ev{fUAFU{_@ne;Xb#5mr4AZLUMvbfvQzqILdF2|Sa z=$|uDGX85?SX#}uC%2z&bOTZ1H5Q;7y%}wOL%BPCq<&%x2pRd{P7NX-V>`2$&d<*R^j=_F{N75=opdS;9EjIiHefV|H zdosXG2(Mj(9nN(^E4;2r@D|73nW1?lm) zXp8=IvMr4T^~uiEWDgPc-p!xkZ2y3GPrF6R#F#gZ(9o3Fx68pApw3q5p?gz7HruX0 zLm&m8cZez^Q#-E3*YANzKN=j~G(!)3n$B@%ifWAmgk%=>tBVlmBNV(n7BiqRT2S^R z(eox7#f?fLG`O|&4;>yLuBCkO20CMUKqPz(UZG|)+QH1bC5dqrX#5iqp;W-xL`l9% zL<~BUKEbcIe-K zE@}m}p1f`A-=4T1Ve?Lpm2rmSfP!NQ?;{VMCK_0GO(*)59)!h+OEPe%^I+xWl1i_M zg{LhmlF9abj-lH{DM&zj-k7`{64DIh9uflETZ;@HTZ?j0uc#%G<*K=nqZ}8#DJcz6 zZY^kJ8Mfn-SEiD^oM4Mj>))&#oYm>zZ^bo4gnASy`+e*5)u|Zinh_>4_W1|A^g@ys zfLN1+bYvNpMZ~hlE4g9!CE2ss+GXpSVTnP+pG$WFVMau()y8wSqNTT6{nPxyX0soq z<03!w_)%VqoEJL!IMIt>Gi^LCs)~5-ySJi$ZDY*n=pAM|bd3*BW>SXB63BX&xZmOM zL=Jn&KY=ME;%`}9<*>crZ}k37l@{+6nI%Lgl5xYzy1Gdlu=j9*cM_A=KgSnZYI)$J;mN>9_}cMed~vSo=(WYnZpT ze)H7&W!@g+0L*!ZPbesc%Idy;gz1_bJKRys;+-yAeTI;dGrRrvAX$uZQk4tt>4h3d zsp-4xbfOR)`tgjdXt)0RrdGTr^ADEk2P5uH=7Q3Xaq^A;_QnHZqqhNVAhptCU|M057P%o7S@@bjySu zkSGkf6IbHB!l6-W&idpOx>L*es`d*g&AUOlUv#57j{Gm4#OWKs!O+-VwlI#~4?$>q zT+uoUyZ1N7_?2*(KhRTm>#f1uJ%#n97gYN8MiR(JSVjvpf7}pJ=508#=~Nwfg_dxc zojD0>c~Hq$-&@OQ*)QSqlT%r-#$y(8f-S=NcI==+_n#-yEt7g4OGE?qo?fWa-7rTx zimkK)Q+oP=Z4OEZ6iLGPbE)@i(4YXC5WCa=_fvF6?Ph!Q!f!!?R)*eB{X{Jj(uHyY z7yM@>OIjf|!fsD$Tf>cL>{=a&ax&`saExldk4M=Kz(*c*znm_P(XBL9K+gTT&(E)l zCQ8mmq%^RQPEZ?gF%~2wAKQimE>Ui$?%@s5Y^O_C8A>ms+PQu2Z7ogrcF|!ls0Gd@ zJbhQw((&rYZ-y$DGlR+QAyF|KG4+gWGS$U{Fu8BiijX3I!%mW=G3-TWG3rx;Bhr1OmwyRixKTE^A+A?isZlC zYqH$uC0;(Gqq09WX}Ym=A}+En16vJvpewo9dm47d<`X`e(>|ISc<=lz{Fx8eFhI$; z>fZZqUe8KxSQL+@r)pMuqgtSqSM!m+qtR5CH1l+1Ug82{(=4TS*HYBc= zym20@q~M_T9BdF+_VqMC2c(^`K@w=;NyNxqxjKVzq@@F{!Mm zvxaW5jmVHseW2@gqBeiRbtBGxIJ?IGb+VyA^!o)|G%rYgU7lawgej=z{0SKtWm`4s&;gPTh#xt7=2PU(${ z^_ZM1e!0tTnls^DwksISZ5>dNEWWy$^<`-pOoLF^0v&9+X4);vT4$C7taWY1dQ_7- zPguU;U?jX*vrE>e8yC4IaDh(~uJPfN@|O!TzloI8DN@ouYT8{kz^k+S{R)fx*V2UV z=QNvwYh6c&0KvE2!XaF`*8Am6TXcfIe_8xi6-=|0W&9*JTbH{1ALRMieEzp{rpT3m z&t_f{`QOz$g{_A10vEL{39X`7bG6v5h?X_!?5DVn4^DSEK#$%0i8ZCX^T=8~s65^8 z5rvL3cu&gRP7l;ZveDNjwsK1w>a2G&Xpf)!(-o#|aY@;EzJKHw8D)E|S?F122AhYCkT@B=Aia;Ma&Gmb zTz-oOoL||LBjP~pb1`+E3%Liw&h^b6qQlnulaVU`vL{NE;b7N$YN?2JoCWSSbFj}I zH%~WXBkTv*cX{D`$z&Czh}EaQJ*D7$ufhs=?2Cj1YF`i<$;T_PtOw`pG$n?Meu$&pMJYsV$cfM@qjg4Fbb4C9*>5*ZpQ2Zov zj4Ro**MbnRZ)!F8N4>_w_7UBSP|B{q9!v)=3Es6!OL#gl+xlE6P4{k;a_E{b;VdAk zXgbh=$eM;!1pfmcdh!%MB=eK^mZTXgT@Btyomja05)wpIoyfG+|Ij{tb>XXlc~Gp?|;Rwp<2i|NVfcs{%ub2JFE6;P6c$mH|n=2+;`4l+=qs* z!(e(+oUa4xaxT}&8UZGhm;}>Kji+smV=or)WDX!>17o%|i{qK9f3F znZ?wvP(lkRs=q|}ikgU2?Yn@zKA^@$P2LIP%y89~`2JqiL$19az$ zlEMPfp0~0xdXtXp&Sy1qGsuWhAGWD&+tA3ZqP> zR0T2%NiU|$PY1-)72`I8d$U>OEWfQ7HuZS1yiu}l-8cEbt zI|GoMIM-wtlbI2`ejnDs5*u$VXnHofY2;JD;-PR!A|Vs^g(^p2ez2B&#&{1%(8tco z&l*LbpUKb8Tf|7XN6lo;o#staLkJ-j#81u)U^kM#qSvwX%L~4@>lRKnIuqBZ>Nsl9 zC*yp!GlE^ln#}&VUVKB{faDpbt}lB>D*|krYfXyb-7`g=G6h(*;XI{}_c{|m4VS72 zl|l`EQi9?;C^k4*Pa8_LSRj)2v=FS!kGm7Ns_Wi$PCpJausLFG?U9qC*l=&buE&Af zEJ}W)h^LK~@SabSdXQ&U9NmiY<`d5Ezagy|Ppt?AW_978MbZ-Sk*}O2gL`|m>;nkY zSZ<7ara=ZHPNLgVEBy@RRHfZ;QVAGL`S({Cw& zOO(kwymI)3>aKhT7 z{;Ysn7VMp(rZ9(&ypZ9I*}eu6EWDTn_EU$T-S(K;$<|$%(84vpZw_av2$Ok(`oQN| z#q+@((|aNEnnX~mhpjryU<x$&&+fY7afihp$(1shkCrKzo5es^tg5YgB6^pRQl}xUM1RSN z5KrlOz26thD4mlbH~BI+(-vQnFruh1-f=U64R7bjl#yts?*8bewS~1qxxs>ZB3C+i zs-Hi|PqcZuV`H(1wbNO}cSHS;vmfuOM3Ao4J8?T5d_lgWa09s>PIuL(H=w62wT%i=uWg3-@V zasImz`7>E7J_RsK{f=fEpf)v|f~M0?8LCA)#ao23KKh%7QfF(FFbI7dVC{D}(xg00 zvj6?vMX^#+_VuE$_u^e}alh;&yMp=Je|}=U?WmQE zcvRmA(Kay!&|>>cHjMtV;(Sz5a`Cpkj4n$(>I<&SC`Fs-mDzPc$d|~8yn>9_Os-xu5*6K8*U|RzfYhH4n)g9#;)& zTdmyI0vHK(CA_}A{wFM8UvfW>|{y)MI#ng9nR#<#JP|9!L1AjnTH=zhsd6 z{nAI6k&~UNd|1sQGzMs*YiB|6HBH7J)`8Gc%sA%;C2V1$$GkePREL!Q> zRTGQ^^{%zXdI}4o$#EFiOPyZsrh}*TfYF*t^irdkSq^Iby&C&Y@67wph>_BDs|OI% z4`)23f;wPs@GONug~ouVfR9>sopq$U-pR^|=|*rXtl*{hO~5@Ym9L`Z^qM4fP}8*P z_y9BU%aL|GTL+ZIw{Dem%`Dm<{+0PJ2*IT*tW#N+C>N?%LBDY)EyjQ%0Wk0V=a+TV za{DP8##0U#;)|R&y{@JdZ|IW00+679C>l&X4zt!XiH8m9 zt$8zns~Xr8j{{eFb=7b|z+DaEUJbbY!gH*TPgS}=as4$k1-h+R)fz8Ae^pCO50ZOK zcO8|Q_Kw<%niC)YNp-Q<48EMT^7(cb-g5yACHscm_fg=}PsDc4l1~Z(7Vnz{_?jNI zixRsypL_Iq&5>ZnNRS zSSRJi9M|xoqPpy23;4%=d#b0#&(%cNFvr16q7JbzA!MzU84eYQoua>~xwA{PAJAw# z|GMW8^H57KrE8Z(e`cxERw=W~Fgy8Fa^?K)-W}e$dvcG_vO1OTKlBfq(K!3UHiKJH zs$;InMMoa@`Q@W+*9$kEPPKJ6xp!?1iotgZDrU67r7WF97P7>M$pWC)&jJN>pgPg7 z+AOTyHdW6NC(`dD+@EtxnL@Ae59U42wgEX(KNLqqC46z;xvvtYg78J6zODaf&Sq{P z=G(y<42i~4q;ycbRkD16U4YzIXNUr%P~J_pq&~p z@B~Dr3w71~NoITz0LPRl<{+Zw_Wj$cx;LzHhs>wXyHw}W&p!fZ*Y<(2JJ22K4X(o= z>*!~ecWuSNbgkw5S6^T(+==*)1MNNhuRVaIy8xy;1U0w2^AD-u;_u_N{`GKnB3egh%~c7dK^DxD?TFeG~jZeSbrfjD#i)oY+_BBJBG z2M89wI}rn-lo$P5h5*<18gP5gMo#wKQHy3m*#TbbZs^5U>`9)_xy*J9QHMr~y~WH&V6Wmn+l9Nl=C) zRAFze6?)Py(7k^V9;h3AWNDN%9*~T5rf50O-aQ3m>`AKP%(b@%(cA4GGFXJ45oTG8 zSMz1jbD|-el6TgD;yxu*ri+D&O%i%HS_c?o%Qnok#^%Z*S{t?2JAk!QKC_Q!M~_Rf zmRKOuu|8bOk}+E6QNh&JM4toO&{|E*I7Y?$sv*BbiRg?(W8)P@dUjv7j7wi{wF1im zoFeZIs~kRrJixyei6;@%*Z&V+W5w7PYK5-9@ogw!|70+=rxH zy`w`jj|m(Dhj$lS#Z^@0(@FgaZeHw5hStB|56{Ng=KG6HjKlY z#0jkgN^PuqCQXWR)lq>4F2vgZWV8$_xBswv{E{+)nIqG*FG%TNM82f zYY&60I!S0#5bdkj+W3}$x6$si62)zA4yOfc&na8KN0pfh(ZhOSDesYgZ9&Y(bjSi~i! zncmFR5lfJm^kaw2FllRZcMzQnSfH0CBF-%)W|_{sW8UPS_L222lgWk!B1pDT3kjJJ zTyGurrN6W$ttC4YZx?2;x$Rw3k$7GdR?560PAgoeUSCpe=L$)i`TdP3!M_3Py0Zb` z54Hwe4n->Gzow;w_Wpg_9_`Bq-j`LvS_Iy4WhU>i-btPMh1%+Vg-K_ank# z=*ab;!^NH2ZN%FdlE!|Ll~ho1P+`Id&b;OTfv6#pIQ4 z>Oz7%#Vx$W#Z-X)(*CyBR;}1JrWW`9m2zL)R*6tM11`w2xlO5G+8)vo-tI(NWSu{S zuOAy7u`-|^FL5FbToB81%yduSO&!g1`$Tb^AWq;EP~r6TuQ$r_{UDAnzAeYbWFP) zo^05vp~CaCsXPt736rckf$z1p7(}Sad(3dVh2ev^?5`AvOxq$(1VvsnCGwjPlTKNf zYriFTVyOM-k#`!)`r9dt?JFS@?qC0Q`V0pHbqiKL+TWAaeYg5w6Ms*N3M*vt;>vUa zo-Q?++iuQ+9%2FJX9c+%;8VW6@8up$I5u(#?VVLA)&!tvue_q#l~VDv3X|qZeqn{U zsyJy15vMUhI?wpl5nl37R+ce?m)GD*CDgBq%?E(V)c#@mihkrOh`z1oCJdskfJHd4 z5~dp06u%biuUNO?o!^@uFrFjzgW&Zc?34*?I-f1-?b@MaNh$NRThU%c}Fd zZ*hXdfr6eS-J+xRJ}6&z1@K%^&Wh*tZBzb@b5qMKH-F9<+eTw#`25ck9W8@bVG$Dtn3Du`S*w4 z18fJi@iq47=sY$Yyd`SCV7+eA$C-O^C^kwyCPZh8jgV%DS73}98FsG5&Oncd3vb`G z77+C;XkYk(;!qWTT(dY34I=JU%QySv{?pfa2MO=Yl65=*klo8@fmLir$3;Wy?XZjE#HNzABtz- zfY22yE_9L(m!jzp1=10v?2WfpdYc?9AOj@c1OKJ=#YH1<`;CnQen4tAhNvHey{?3_ zz+=eQI_HSK8Eyqs%@oiB!P+9PnFM63PG9ohiSx6&64>TS5?1-W9%pPP?ak%Vh^4XO z$>pOKAv0U2dP2;NvR?3F?}E@w?>T(n=I&(z4d2uuWZifTv&yz?7S=+SFLsa zQoY3s`g66t4}f_ATi@6f`;`K#J)C((0ZAA6rFruE1S=-OxFnr#%8->$`G;4vF{2VO zM)cw}mK^v&4(1LYFl-i<7atClm`9(i$D5VoW9qDeM6>3qyxK|PEc8JAM4jT) zzy92On^dkfBVhAZK^MFw`G+i~}7~1#^QMbJ^7)r(2ANt?Bt_@py0?tB-Cy$;oY7f=2 z5H<|P;au}K+dMDWbRPc|x3a6jgC6(n4c)&j$kcE=EYIHw1`)Y*vGaiR0HSwHqj1|( zM3UC|l<;Prrwq!creo=aCW3bC`udkwI=+WtX3kE59%^DEN2_x6MZ5=Lu_1nMrBl~` zC0YqW71mPhbZd|5MzN#)tF;6y2+&>vq)+ZNd4*j!?i3v5yg!-}6HL5_s|hEQ5}fpR zus1s`*{wGBGoGD$Ov82OqVjHtHtT!I%6MBU<*TcOu^|DI;rupnd3r_k*N?;M{k7M& z%q+FEm!q`57IuWUAk&4iqtbss=^;Pwj=rvBote|dP(e2X@eHHBsV)78y?O<@^6ZIQ zkjQ?sFei#rKJJhCBTHoU4C&%`rpxpKymkln`Vhl0TPKD4{|;64{KueU0MHe^!JCeS znASYIuy*K@9iBeiFhdA|{AO*?iVNvA$E6ruBk7%{?3K$84PSmvr|$4zy7TyP2ALsB zxJy&NY+t2OzZcTG<5?VpQs_R=HK@AX2dHcR7?ifj`-;edMZfni?s3v?T$~R$Z-_Y2 zqI9N*QO)A3{`9tH;S{TTY4!fc_N1GSq>I9N59y!dtT6A;g7)9-o?^_nMJ(1UbX4hb zd#`G9JeELp_>7fAwoKQfOBYOX0$s(j^WyPf0t-zagZ(+e;?+^s&4F@0hxuc@vPUgd zr;bVmCYe=|;H`0frJC}*Bbi>FkX()PJePPO_nW#wlB^FPlwnh8#;@2-7Jm*dK?1x1 z&ko_H()Uw!+(lxdAbPHZ0*|;gf4rGqq3*ITnDVC#8nD$=xA>4;cF-gWntH%?I`4f- zj4IA~uzfI%KNA-DTyWELXpfx)IujQuJ3zd<`RNLa&P+YRBSzGHIolF1bSnM>E0zB6 zrqTA6H(LMqC;ewSyUu1kKlgk(&a+<~xUwT*a!fn8VnTK49=aEHL{J3PT-1Fl`by1Z zv#fT+_U@8g--6ONgX?lDnVG!@xpuFBd+e;Qz9v%&2ec|TZx}bl{4KSIzRa`hZIm7x zs}FgrTkANWjDN%ghs0opGP7dYf1s*d6 z+sft}*(NkR4c>`4+TBpoq6_V$CXmT@dLqR$CyJ*_%9~%;PZT07E~BKUJIrb&%iUC( zEL)t&B-C#02p_9|Pv5xEAwBf8y1414=fn={Cap3erbYR=+ivU5>?lU4eo6hHh= z-wQ9>Ft^GG<~Y=QBK3K%F)@*{eHJm*w8uWISfL3Z9wu+by<&<$Go>FE)`1AOWQz=; z{QuM1x&Jfy`2RmbXbDN=oY#=^Ea&q%%^@8fYs}f?d-<%vgI0;`sIpUNX4=Ow3&DD06h zhxKD^Eq_w_$)`u8uX^jg-;4#E^R<;U}{gYZpBXNg=owuE_ii)_Du1+ay)-XU%k2I*GO%q zPhlSExDLgc(s?%tM0k^`a@z#IiXoi=;-`RoEd>+jYo+v52ndCc?{BBvh9;dz!Rihg zEgZi;>54QTOEeoZKaC{+AUuk(l=6o=lFuKndAuZbTiYiy5*rNmPWqfwD6_jCM4D2d zR?>w#zzQ^?U=2MFYkvLi_~}Xe`1S?ISSM#=(q$Px>l`FOF721#i~rocpySG$oIhEl zVdg6TAqP4;`|5Im=i8$dH}n(*6B2@604VE&k;(0wZE|DrS=^n?r9UonIpe(43?LzK z?qTi^)*@6S_+oX`&Bo;kOOj92AHDzfs-CoRy-8b%)dtZ0JUCNfMOnTSban1-QbO>%(do87|v}P^Tk$TE^$i zwsa)CB$1k=e&zPnws}Erut#_+q06x`93;RPDDIA^WHHdg>U4C%#hG5a^Ew4`YpPfI z)+%c|toBHECMbgL^NaLUj3O(QX2k8Q9UIlg#2jRu*|*asWU5mzS?ENcixQS7pSvU) z+YCMX zQ|aU{1#Qq^ji|L1#Zg<&SrO}G4CwFJ!>|a(2o%{3noS2FPyF*;?9J=m7b2b*L2;vO zTIO3kr~Zx(o{Q@c1CLI8coxc;GhMqo6CpX5^burB7QYq69)x_j)}M?o4$K@dqv#h4 znJ1j!i+o{BFUOmSe|6eu9FT}( zpXfs@2)r*xgfZ3@2ZB^qWT+txe*}MhK;Z7V?{Bk+D}i5W>bjh#=db5GBBr-fN;KXZQ4Lp}P3NvO+TTT!cD2E~s zy~CZ(*_9myoRArvGGDK0#n}&_J~2gpX+dbAw570BE^94JIhJDo=A(0%Q)6CD#id5o zo{RDo_AUQIMS6$CNeaGF#&F7={joo%io;1)-qZ*{+Zt^Zl8NROgB5>co)XY zb#ni9oY;c#wtA%|riaOyY%w#F822vY9AKZOnMhLTHqidPh_>RL1I|628fnhZlYpv{ zlULYBvIQ5GJ7FA^v1`HDWne)pUw%#gI&+lz-J8tS47aPWEPsm%1I6dvB?f2C)lDc6tSnONFE*|p~mFThUUFzKQUoy7k!6yqfO#P$z-W<~Hb&&kD& zofp93$A3J5-E3TA;$e}&8h4m_IphqlqS$m`YF6u+_g+o=Fm(%dBdZj zQ?lY)dwePk)W2=iwNSPq&)r>jvFcgeZ{u5Y`|G^W&sM3aNgK`IpZ1xobwRN3z2ESO z%8+xkJ?&d$ylKlutL$NCW-CK1f+#wkTcq~XWHY)z2!sE>OZozNv_xK{a>7oufwA}h z{99mGU(SP^RJ(E2iNk1R=BJw%%;~4Mtjop)lfv8f)eE?oQR51rN7iq06rM&730P8w zTV*xVNE^M>?7^f0wMB>j4Kvq(U3Py4Gf%g1nk4&MEKCgbQ0TEs7i)xAQ)DQMZrQ^t znu(z^yoP0G_M7l2%thaFZz@OxWbawVg}xBTIK30Z->5QxHj9Z)6}0)JX(MCLS%*rj z7+93od|9HkK2AmJw5;4<7PWUX#8DZum2SaKdUog0@9&fbo4D_tZbSYInuu+o(Zmru z^FNf|WGblY40U=z!+q2C&w1dCWH;|WhDU`(4?w16t)$&zQP*!~@HwyIB|MTDUJc4_ zFYw~^rwjjETb3ufFe^>nvRQ1<)7`$NuJ8tbp{L0Mg1+1Kdh~8_k zN_CHb4`k?h4pqIj`}#I6SU0|fK9ZU8`t}2ee@pWS-Y)qmbSNEtd*$f0PWTWKDSB?# z8JI5fbc=3f3D*>e58-)iM+7whqa$uOzOR(2g?v{L+Y<*i`ccoh75cYTjdVl^Rn*w zgvleY_foffV_Emd%yDDbJwKSd!}}fQ88;mt-(&4^=fyS?w%!%|-y0b7RRH>UxQKEt zY^M|mJ41g4R8a+EsLl`#g#p|2zpOx5@wkR(mSdzVT(EV~>F4a+n)|J*#Gr24yj@w% zqFvbqGV?K_f_J7p`w;eWPw?mf1`L6FU*Ay#egtDq&q7-ED1=n_}!yy z-kC}TeQdv7=*9)&YUf@G>HWOx=|GICl#?WJknNkh`k|K+37@7P&>3^u1B{-MsH4K! zv@K#cH3ls_b2lIKPTRRQyHy?2?%O1!nd@ciPsVoL`(q1C!5A=Dm-&h}Mv?d#Q=486 z36Y^{@WScs-nxGQsVI>PJiycKOlgK~q(2P(Q6T0kR9_OX{V@CA)1mTP>$~rO!u_5( zNSVpXZa+&I-m{Be-WL3^=vlrEUF%``m?#-kh#HR*>MYPt-1;1=`1FYPN25uU#$(jf z8S>WEAIjUT;>_pF3^|V%Au%m(H@eww^ufdu>{0o~&0C=ENht)tEIP@Sew?YU$kW*w zIi&w6Y8)D>lQ?;>cX7a@1DNRIIA+61sMjkqf&RkC)owZcygDDcDm+TNYM4WX-Bvhc zoNUj3ev9V^$e!JZ?MdnYQV5mtm<1{xI2ON5Vo@Oci%i}9JDyq3m2{$Vt=Q@3Tw#`8 zbD_bN1d*c#o0-8OJEOh1`4H~)^I^_aEp8d)y)85{8D$;NPah8pDO%tNDV;}Z3pzHZ zpvin}M+GRtv|jf+c9h6NakpZ+@1>A@RDelCUxB&l;_=HkAm8%u=C2pGe2!M*;5#oIi7 zK%>&Rv4yXI;?I;igX58&A7MfVa|7xeG5%aPX5101_d?8Wo3kKr5$C>w7p|=6RP^sS z?{2*wToIC90FP-Hk#5JHIHVp}Ik4o$Mr<_4eOs|ISZ*!}UGamEQA-w4Z&vDDnDgpu zzxyMsEU3u!v>5lZ>dXJu56U6Ji*E_Oi0j*GlzK6y&eZoDl{3sC8t3R+NAtC~^qBi~ z{{QBRu&7uDjTDQ~rzU zUd7)nuCEQr6GK1KVu|cOK#?R0l(j6YRGc8z=pe9{R___FM3<$zK3~;6>qfQ#;!IXT zx=ca?Abw62lY(n&`qOBEd!mNo0mK_w3s+vetKM1wLT0qC83bZIV8XJV1|vu(aMgtM z_ipZzAZYe|p`lgf34lu9-#K6!Yaj{C9so@*+^6wx$a}HrOw;Pw6Yq8$G+vDbd7ov=;t^ZECqf;7 z;I_s(eNxX@Y)cOh7t z{KE~%fQ5zs|0LES^~Bo^Fw0E%q-JeeI>%r{ifk9{+jci+`e@X(As6y71 zRG})}xr^4sm;n$2USazt)AI$iBcC38dt6Ai5xq##YbKQ*Q z9Z_BALo##JZ*6kL2L`?34ELsbmW#2(722wC@Rg*sI1%fear=sU0gTxu_yXD@9`t2= zB6jUa3Mds`l8zhsQOj&_JeQ@i)r{f9Xa%nykDah{v%&&SR9rB3#bepmkG&+VFI{5t z>wlDfVxbJYzO?nco1iN6lK79n8=c0nF}t~(RBvhEGAfT&hOB)UF*1YoR0jmaGT7O> zGMc75!pPjA^s~7x9v`}*7-E=H^ngH(arvJtBXm#qjl05u^9hAq;ytg1RzS4I1_v%Y zKjtOL7bMf9ZR`^;s6$xOIieBnywW!QbyVKCR9zS>gjrRE$fxBAx-Om>o~(x{D5Q1s zE($}3?kICISBITR$i_G;8&~QxFED!iFnMs)S{oO^Ao9AiUfE}-90dD~y%h21AVEem zyckv>D*&%`kMT(SlG}#k%6Oyw+U_UiLq>OIdE{cI9%}zv%D%!N+%29)%{7fHvGu57 z04gffW|0(C|KVl!PBO{Pn2}w8+Zo=$^A|jaCOSHv2uaDSfah1C((T7}d{BzNp_7t! znt-<6CBTHL;WKPmj^uh8wn3Q40B`1Z53yV+R^;_uV8sd6;#Uh|2t7W-c}zlFQeR`s zy0OJ^GpD3nPt5P84N9BrWhw3+OGiNF8;o%y8fIF)O2qoKP$vF-B}Yu`YTRA#1Z>u` zB8mkDX>Bv*&grMGuhMg<-2#|?&hcSmVB`=()@Ia8g<0%XJ&bjx{o@%yFfF`4!h=et zW7B(}LSZ6}4^e>+!kpwKN)rosI4iln?8s+5UlAI6&U>?O%ue=OJUZR%OE#wJLz+En zN;i|rHMYF&WC=~Ba5aG@$p-G4mo1BUAaXj}r?wY=E&gamZ@b(5%>Yb-*y{;IRLIWQ+<%}?q zvzMNo^?Z&aajz}ep)Z*^t^L?Ue%>u6W}M9wR54B|A-;v_pn&HQB)j0ax~Z-O?GU6; zj!Jle?0|}qJ|#ii+j(?ejm+-|efl-17ziIu@3;n&jFREL4F7DNI%Lths5P=ckc2dV z8(M_3b_OK^8|Xd@RWbO7XbVrvdu}DPalJ?`zmNa*vh*gUCQeC(AZ8Gwk>WL5NA&m7 zW}|v*1DM?YvG(qI&V2-%VM9wrP5CD0Kk|m+a2L^Tau3s(>7EZ_av3N87OoyUa%lKG zV2YmjoirpWQV80?V?&ZZsw_9s7MXS|#KkMd;n#JA0!O12oMSKeuQQ1a<8&Bi!`0mF z!G)brTqhNLm@B=NlbHpmdy5j32CoaU^;v!&!2u#o0-7+1Zf7(U8UX|u*63_#@*M$# z7tgyo`$tmx-dyHr5Jbni#S|zpuEC)>_6HWOP%t^;!fZ0tgXUZ{(koxcq?5|mB!~+8 zjl|o#w)h;zE6Oa~e>hMd`W{O(jZ$nrUmhkSQqZ~^AaCYR(Qvy$`jLm#yrfLB^C-i8 zo2+-+v%Yk!RQ_{oJsT=t4rWdfpS`iE=@$?46bUyxMibNqp!Q$yT=dlU?C@+q&Upd= zaap<2-!$pFS%g2!y79yB=>K%jjE{S9e0oK*s&};WQYOlcywI5W1j@l{{Gz6y=3HM zZT$PLK=v;01(S{h;JX-nZAmp)AkBAT$b;>gdvUNHLV6I{g#GFNb+fZ#Q9-ESi(ds_ zRxnyiF^!RGF0q2b?=i1F~-g;L0}-M}k3;)O+n9|GLQDiXzt z(ZusHGFTKbt3k;e3h`r>RhyE{QjrMrbIj{~w)U1;o~w$gjpqCG`qA-M_Aht;4L@%J zIravn%P0S0fut39NXqhC2D0C&@$gOHdG#kVi0%}*2q%P3OBtk97o2xpu-9X z&zPjxGoQ}=a!Gta5Sf{`#(X>)iz*1Z@*|&MEb?t~gsA3r3DJt`n8axt3*Y6peIF|# zU2!*!j@mS5>J;LaFQk!l1+GC-(05p7VbfN{U%f;lt>^CJGWxPMvKa=c7J!~&t48G; zuEu|U%0+VzYmaX*>UZ z{d1c~qX`B1f*lT0&DtHmi!D?Br08*ZN=Lknf_)_F1%$05k-78=V@kev!Ju9ic$C+L{q3H z71|b<3ay?vl}rg_Y(}_rhp>;#B2t2`=N5kZ`8jE6tQIG}U!)y&HStFjn^4H4mP|t4 zje%BbPznHbRT}?SjYKfrWWx(&q%NJdyxkg192sf97G66)^t0XVj(jQ&?7|{gcuif7 zJ1*eooZ|!2xgYRZ-sfB3xp}D~&@Wwxf7`PbAI zOI0RAU8xMU?WES~tW?5PwrIHLv7DX1`O2mVo5{3c@r1FeW5WIz`^WN^KUmw!i*mis zPYn>?Yu>m1s>;6qotJ!;X*R;VkR!zp{hJ5Ril-YxS1Xwjrsi~ETGbLUEJ1|sXDI*% zpj%SZxL3$k1a1xuoP$y^aXL`&9-{fWIFH@z^hgvyhSH^$e}3=#jK<3Df#qujTb-&?1;TD!A0TgY>sw%A4pP?1{0_yYlQ{cF%#=r|e-BDZiNcpt@zU*A4pYobUVAojF zeL#ofhV}&ae|Dm-a5M8+{Te6{*B!2xaM4~R(RWk)tJG_M!mGZ8sxhN&ukMl7NF@e~ zu!~;JLAZ7cJZ3T{y@{G|d=sdDue@ji|l7=-S`J@cE(<;ySRN<`}`gwPD26c*GDpyzrQxm=pE(7J+} zU3Vz<0Y`3Z;_(t=WF;-g8ci@?lBn)c%Gyrr))bUxG#U0s0j?o)v$wZ?D%OB9;Oj(< z2C+3?Be3-Xi+n<0@%bd-9BcYw6y0k1sHOK2$x*veEeS0&oVi66_&AdAlXc^iTD4*#@)(cCOu(~H`0_dG)e#`r9aC#~m!@UfbU+n3&AJN0Oq=VO){ekb{hFJ7^ciuk=*IgP7;8g+9YVA3H>8 z_><(F84@a9H~vKwE`%=u@yaISWCWg{M~HK@xk*a=K;v9 z&^lG@%*;E4NX_26oi>DQ9Y0H2ZnpqLiZvnPr!YE$!(%0CBy^(~@hO>Stt@SI{oVe| zMe2k)eeTn@%11&qA^~utIAH$L6Wx7Wn~#2E#rOm)bnKd7t8NjCAg!a+m-d!;-;v_m zLa>`!g&doBHbqenK7b(;DWIsC2}xMy1p${C8UY#D6R-UcmRl%tbU#2xw67W8+EzK% zEC*43gxhG{BbBmSvdT3$`EcaGDVR@T@o%wf{^rqA5mV~_R1rlR6z}d6A^(JX8{T_w z5qK@~8!R}c_J-E*(nK7D;`n0tW*|E(G6U}+kBIFTUz1m@8^7K^iA0x!hs$nABJ}b- zVgOuc<@0gu!d1%AB5cxUZ(!kkcN6{} zD_%vpNst#nKHES4vVUfYGff{#F+KPMU`W7c?UuPCLIQr(x=(BRhU2%v)|c}wX?$@6E>PGDkVUZtfeo< zyzIIx#oip?Bh$g|?EM_L8VZ@d!Wv_fis}t(9`M0WjlVnl=wU*j_owxDgFtzg_pQGj zSX#s7_U~5z2p212@!@vsH`%@xx@R~&_5Gqf3gG!N?(4j+N80a~Q-17qQ_SxbIF_%K z@EPgWXj^vtNH}>fqIKPo-}*|p)caresw%lVO+oFhM{StxYs)G0%gu5-Jj(6-&9gqE zin@M?X{Z0D33*oP?Ej~; hQm=PV--S~#V@Ssl9}NGJ@i&Px7N*uF7$a!H{{yKCe^dYf literal 0 HcmV?d00001 diff --git a/examples/benchmarks/TCTS/workflow.png b/examples/benchmarks/TCTS/workflow.png new file mode 100644 index 0000000000000000000000000000000000000000..403a17de3923713e395c076b611f7e154f370d6b GIT binary patch literal 29877 zcmc$_WpEt9)+}i8h?y)LF*7qu7E6{z7Be$5Gue{G%*@OT7Be$5Gwt}^Z})BN#{2z# zbWBWl&x!6cRj2Y~W`!%tOCrL3h5Pd53*t8^G375`z~VqhF8~^Jrsp|}3G@Nxs4OY+ zrE&uQ81x0wR9H^<%a@v1_&0qh(05onDGkRjUy%C#9biMY#YSJgI7xpK6IOB4IbVm; zLpS||xw2h5p{Iw6XlvtT)$0vnc6HT%k>6l`$YOySkqPBJwOP`1PklAo?dMwuyo!gs}+Y(V##tNsyMut?Kze z&-Z1V)gt;xNQ$(;KHD^;-E&Vy(NL^ux)ml+t}c6F_iph}pZ`kO(@PU$8|pAautfW# z0j{a3{S#8l-A+Ps=+Kg7Re3Ytyl{0|tTD(@1SJlIK#84$CkG`IcrOAAN~lzaA~-Mz zMVV1YfjFQdYW+q{9^k^;CI?Di;uL2od~SUMle==iHUP;j~Duj)bn(1xMe8WXE*K`SMv5Gr6EOxQ`ioraUg<| znBFVI(?^}HL1$ocJ?N65p@LM8A2m&Ji&P^(2 z_oq*GG8S*X`o&HN_X~^UCM$g-C_d^wVD+Y7f8)iAw{X6R^R)}b4 ze3?Sc=#KiM(V_ZtZdfH!etNFetNZzmC=xDho%va--M2V@rH(2+5FWJaw%JlO;^OoM zwqpf3`!?#?ZH`fnR}`f%sa2%8SsWt=Z1I7a^29L)>P>Hm;!WXkWg1baXGQvN`#KoD zY+nn?eC&tDN|=0{AtFo#kS3gb3`z)keuK}xy(+$aC?-@sg^Pm;?Aj)@bE1h~$_3OV z9$xQXzh{f_lz*e96OpoGDJt43GBk<8Tg3AS<{iA|ix{O-q}A|);yFYr z9`Z%jZbG6vIam@z59>m$jdxUR0~CyE63ZK7qk4>a()aZLWj29qCqDgUWMwGeuu&?Nr2shGm~8F zBl%FMgh&5u+~5~9c24cFmtP>+Xw-WbPNrw!bM9lpD#?;7TiE_=Ok@816CU=xO^{U8 zkucN46IL_21>v(L_Yds>y}$C`{#qBTry2YOI3^l;Cqts(HIxx>=fi2p-oq-fs|HwbSDSRH{yc|v1x1~A$~VHBl|^>&8DKK+jYwXbUJcBhISV`Z{fF=83`s& z+z69T>{gghe-2?%AMnZSy)@J8j&0v5GB)R`(N&;&M%?lJg$p}AUS`O(5Na<`3UW&# zl%=pVaig~kaXiOge?M1|c^f8#_zfelNm{nUty{TCQEIhXz;z=_9RdBl+SWBKUM*># z;lUZy(`^(4s>gkouCMSJXf~a>z16HTm1q-m;@_iC!R9+HU~oCgSF7s%YW%Ss`v&Q5 z>b>#=X*3&%zmot^WnC9@} zm}=DxMSlmc44_nVG|=KfY``*$)R7_nmbN=)v*hv~QyqFx7j{^HML6`9@fkWK>C;Wp zzxuJ{gri!9rl2DjA%2nvh*i*rG_Lw^d$~#cBs>)8FvZ?jHW+Z7trDkQ6!7<}H6V?! z!dLlZHX@1=KccMuKpxQ%OLzUK`(ziPXG|%} zaP`%KWHv;EaGH(@E6}tv0h=_57NsCIZy$}G#y$UOy5c=Xp*Kuw0)u74OEjuU67B+X+IG94q-r5AZFEU^iOK;@U2lYZQ*{Z|7 z|5<$Z+>!5}RIplb z50*4%4kP>Q)Ki~35hA$tAb#Z&&67w=zP8!F%AH6ObzSq!KVO*p29c`%e)!9SESIu$ z&IxN_B5Y{fpj2pE<$35ht*gOxA(_{W3<(bcUNH7h`e{#^^f(iZ(NrB{gh)=PJZ<)p zku!{x;?)_DP8`Q6uQJ%~%$6(akm2mRlP@&tWToiQ)3*!6x>oMq%on`vN8gaCvdfTi ztUxD7``w8&cEyIvagrbQEtJ_OkHk&{nsrj^cBYL1vuvS--r$iaXz8tY=h2gdxB;+z zJi95pLpzCJ_Qeh+!Mplpc48VYHZr%#@mt%=kR-pGTA*45n5*g1UN4W~Ql#PCn4@u%Cx`*)}QZcDE3CWE^B?>4Eg#USHp(DHZ5M5W=b*@b+LBZ45o3z8A?BccDyM}^FVo}gG9Dh{_<4smmnMkOlNN11QIdZ8(T7AxXK zVcu(Wsupdg_ji7`nzEm?;4qy}Z0VFPeocja_AfY3zi;|}&=gC&!vGB_RHcyum@x;5 zi2R0Th^!gu%h_4~q)fb+aMaGpbfkkismz%ESaWwpgXGH+A}duqD)J(Ez*4EUdMI)x+Vhym6)uiu*cjKO3C{WaEdmo!ku4l|Q~d$)6}5 zm^PaUbEEW^O2|=kX+;bPSD3F%Z<7UcuYUrQJ$k;(4T~^#RaL;6h6lP%`unUQ#+@=& zpXS8`)GoJVm04|w%krJ&5zgR+@MeaT{&=5ALqRKd+E#MAdlv!L%lvND|t-1 zeI0Txp< z5(Y4rrSgX)z?B`{40S%N+KOb=*L6*XzpRl?Sq*y^xPmJb%p2hvmRrUDNxePkbG@uW>hJ6;O zOx#;)G!hy$Gm1=3+j*UIS;p@CZEcNy-ZLcbxl+x{)pD4!^>D_dvu@u@LTs{z>OIA1 z52Bo1kLGo!vRTsj^>JF3{Of zp|ZKkcf|5ZSb`)>?OYU|z4XAk(w)&SpRv7w;DV_Sv<>#@Rbf6^eyp8Jh{nas3|iI+ zxwsCr-`wJ)_3jLFrQ7tur*)<9yzgjS-XH#pEL`@aTy^nNx-T@V!yE_Apid)T|Bc6g zxPD6hF;)6K9DIbnC|-~NAJI)Zk3%0*0Z5{B}w|Gp5 z99D;R>K?21DB^!tWRb5~gpW)#>(Qi}!)VRB;oX3hh7*6p2qKM%?4)42IqXTA5uC5{ z8|(~TgS)v{hcu;+uVlkJok9WBC&VaCLH`iABnVJEyj(8EiU zWMg;br6%Jid~60@?P-WHQlfks3DzPtu1BJH3?2H{)I6}nxaJQ3K3vgTgi6YUyjY?r z9^$9*fos2YWvtmVoS!DLchs$+{5eV(BX2AeMiJ0pF=@`(qqX98aIz5GFI;dx*!r1BKl8<8N-}b~vO+pBpT& zKWPuB$Po*36RXyOFEf2YG2CFH{&=Z}`uU&E#Y@U0EQo~!D`s!}kA%jElXXs;0pHVi z!9E{~KjQEI>2a=s8d9#q^?d2L8gwLscjGy2$E3rzAO7~&d;kXsm5f1=b!WPSam`j~ z@e-bxkk(Pa@O?wF!idbjmNP7Nxc+)7szrAVg%?blEurF7nDx#lXzhc+h`THA?r^ai z*ZPYs)FgZ8c8H{Zu9sik7v>~Uf+GFznorQ64ofOS%2DFmVw&4cZlfM~Sa^5m3vPm! zJN8*g2}%@{9icAlEOKm5{fjB!;U=C0zd63ES7!<1Z=uC;GqwsnLiO3QOQl5%^1t>B z^$8z;l?ky;tt0ULRG@cb-qvuk^mvJK9m}vyO}_8g=#vl2o7!9jZmEF!>tE9-(tFDB z9|>8??$pC5y!aB0BH|oNu2n%MTh@y8_z)ToBzDFP>9+rQ&E6}hw1GWlL>E!){w6%X z-h5=cohe~9b3rt@R115D-88E0Vk7Qrg2QcNp;M))2SS+B3vId+-UK0lzO-@^u|c;f z4$m8Oe}fs6XSm;0CUUlTnZMb_xqg2?88JuCt+6hkguch?OQR7t#+!tl34G)W5-2&z zaSx38YwQ=1I2A3SN?VHA8f^@fjarrBO_shwR$*n|!Y0)}+i?qIKJ@4}b0I|-{3J`L zai0&PZ|rz(ep%BQSIF3|rtI(BaMHz`#Go%`@#JzlSv*=hjwoH7V3}WdbedYRrjg>U7rCmLloZqn~aiHX7Fq5SD1XPFX&EDCvX}Bhhin++@3^HB7oF-t5K4}t8_&D zbHbjbkysI2u3$6RYdC4*{D}5mGBfrH)WU^Uh#1!4hA-FK`?rN@aoQH4B<18@%v97X zQyAZ01R}8q$0e!``f`-bmiRN3Qe6Q7Iq;esB_1Dr>2-v(8$>Z|d6%JzXgih(LQl^Eee zU7^4So+yij5dPentzRvVGC0npQ;k0y@z>g*CvIaRG~nWMVlE&n0x}3t)2aA60A+%B z^qqniD4_1(Uu=yxjJ3^IhcmthC@hN4Xs~zPA=bRY9IAcVxAsrGVuPZs&d#RpsUrb? z2e#8!ItkfKN(RSi-zm46q`nJ-n@Jl9x6yA@m+4EakKHAw)Rl*6!14Yj3^`tr6ylF} z#_8O!(=RES$i|U`^BcNF1@4?e%iWpvYz5~D)ZZa94}*pDWHf0Ld_I$j?c}jwZMX|e zgC7_8$PzT+G|22K_+H|#UO!UGe{h;hCc(;>AQ{kKetYq5t~Q)R&7nH?>#nInxTM15 z@M2aOrgCh;v$$Ui7rD7;Dc|>XfUD@Wwmrue%C83WRjK>@|%9<($V{> zw56&t@{Tnh^$(&jY)~AJ8T*NhY_liBN`sWymRf9Fx8%RGG=jc5@>}jwv$U{;tw_Of zLjUfC-a6yvYPhrva>WS^Sl7+XpBuga=3j&6k`x5hoc$4#8O@8%GlxfvJN~wQbk|0k ze4gJt!+on%bspplKq^#T_CXoPnjEZyD;<$Gs@G9oUCte>>nhzB8P?@b4Yyiy6eM#B zTa;^G2%>tm9+(jrQ+dCS5rI`3Ht;THazLS$u`=+1(R~nMZ26`16sKK9ed;Gy4{UIG z>r6oA?%w%P5);zmgPWhY1>$s$dTUdnB%sZt1gIaRj3JpqnrIvX73j#G75@XsV)0Lq5$*bPvBY4CD8JS9iLDL zmaO|4Je1{UO<8-(BS5|R@jHSIKVd~KPhGOBS%T(~in9WE`jTf&>W#*zr%iHV#HNlV zZ17s!CT28)vTO+S!VP1Bkk`;A8bDi0ih-u$DCp`f3q9Dj@m=jL9dcp7FwZlzlV5YE>h?|o_?Qw<1S1>AJoX9z zpc7ig+;Gpb9iM9!t$#U|C!t)QX5Kd`$NGq~Fq`YZWlfuz@HZAqRLdX6ilIO)>~T#S z@dGV&e2RrqX-!=ZEpGE>hfeA;Wsx$emp6&y-E5`IG=TI8{eGhH+>+&Q6OII(co83) zpp?1U-@o$}$Gd|hP0xmsChx_)+(xB0vD=<>_VHLX%4iC-_EUvxNa{Ke1`<;h-YkE3 zES{DW&uJcgz0rz4**Fw&L@GyOwfy1y@k;kgC-N?})Uf8-&E5nWt6ku2GybUkt_hWh zw8M4&*G3tRC~Gmsq5PZlsdVfw!k+}ePwz3zQod=9n~rU>5#WxEr?$Hvo}A8z?Zt>Z z;TP@+Lc`+UM1!!2l6R#kSnLB=Wo5~l?mR1J2KC%$`0yuICNzIIN1O+qdhi{2w2Q5{ zy|q5F=3{0#HB3@^qrud-A{g?l?IDHbTWb3(JrhdzN1Hk4PhLJTxaLaM<J(iI8UzC8hj z+QikSaNCxR(9}B_oWM~YIeq#jMh*o`bgg@}c^SM*!*B1`^Sh6qX+rr%PxFn7mR?x} zV%~;TLsJ}uypd9&lHad(FOUZ^ZSF_3%VxG?J{Vg}vgusuOpz1uUk2|}=+FgW)Lgre z1ApocTy!u=6K8_z#au*v598*oT7>86p#tg8>yHzu5s4M}DyfrgE_X`H9|c&xk^`Ck z{Rj1P%|kTa9`idPnk(vm;yKu63`3Pt+b>vX0|(B)-}%Pbc(Ia9$FkVq*UM&dI`dIwR0Eu zGU*(9H|kv3<|&=fS{vqfg-4gIZc4m7yBuFqZ+w_9!W^yD6NNN&EO;M^D}rCQJ8rUx zFbJ(s9jibuDzD@3VP00*aQo&_{PK}^K22miq(ly{7cQ~UHlL1=8Q25<;t<(Ii#X4SuxdjJk^XfQR3g1BayNUOX+IUM|odP^RafDbQ>QkwqmB%Wb^EB zym45{YcO+L@h=(;;*P}Un_Q@;q3*w69a)E$+0y5&AJ+L@w$Gb|t%q_9Ak862ORQ$v zU+p6)s7O%o$U^XOKv*`z8}Lj%pG8VcrVawGdn6}C_f}RF@NP)^arV( z15waGy)=pgs`B!0-byJyiSv(Esq?lf3)q6(n{7-7^JoiSh?RjmBl;b95YQ|MHeuF` zI$iB|Zc`WnILVs58Lz%)u;rHR9C14wgMe8+RzBLn_rV6c@nHMceb`DJ0I(+}VJzZJJGG&Qp3vc6Y{D1#(!o<+Cju#mf8cJ}hYYa}+jhts z{iRN7>&5t8SoZ4%rW)c|H_69m2XyHadjlb(h^-FX5Yrr}Ye7LD<^}>J>x9en{BXLli zpOJlnaO%MCr+|EI@*-ur)~8Vf}g#jrVW*W;_yt8|FhM+*HHWW4LFDmBB83#Th= zk1xPz&~u2C0@Y=htrwKrxomkOO0$fR`7XO>22Pn7m;Hwdqq@^v?|TFz8x1^t#vN@@ zCY5OB;zo}&nRG$mT8h1PmeszuKd+h?n-ab(i|S|F( zxBLYcJ=|38$b++W2wDF>|BY1l_RV)OAgYHrW2|a zq@(D@oIj5N)-flhH}GOJT2$eo@I610`9^ic@ODrOqYeHiPz{lm|L@7lQ0v7@6nu6C z*q^#ch71eo>|34F6?Pim5s@g2ciuLC|49iqG?vy{;Q= zS>)BtZ!u%2C%t7QzUw=J?@Z$ARxjqe(YK5rj|K zUT4;2MkV5NMu%TWg2?jypyXL|f*9odo=e6E5$oNGB}$QO`eP108It(T z_{`H4Fto8d@N~+*Nf8HH&{W{qMc9AQIp~9QM#e2P03=~R$4h)F8J9YzH~i>=yTMMJ zDC(lpnLl4p#{<@re&00oqmWffk=-nE(bXw@k|?DO9aIq%(U?|%bt%Ww`L(~DxA8Me z`_&4`QfN$Xky_J7Q~hhL;OAiP8I#xJl^vT=+sg^3B~!DEjMhRGKPZ29&NfHo;kfRq ze76I_b_U?bD?A9U+JF4yVRM=f3ar)7b_s`(c{fE@DfpSn$HlN({#E?Lwk{vOWl+iw{E9C z%=Tg5#H6W$RK&6XXm)$s_Jnfy`?W|!gA8q8P?QP=0}s2O3_lgrQF8aI$*I>nayR@Y zfP5@)>xOmsjUrx+bhu4&I00PRhSEeAj)ZL9j3uz3ib2*6LHl3=X6*j-wpj60-FtJr zv(9A+E*}X?w%SmO^|&DbY07aiu=eBi>Vgr5Y40Vxxd~Z2UMS!nxkBcEdC_HywlL9L zb+DfSOGOkqUoYqBb$6z|<}Gg9?z4y^dx5GoB3ksL*Q($axhDoB+()s&Df#>Y)%g2K zco?W-vDUtC*qG2DYzQib`D&pe)Q|6q3wjbDT^cQ}^86a*r5!v7Mn(cBuU0VRVEq~=dvN+dRf>lxzUi!;_kOzuv|TI|&jkxyA^n`_}IlXni< zB^R_QecNS_s^Wna6G`A)PZB?DV zJ`cdR!x;H6xbLJCBGD-81z8Zr7Z1lrD68;&s|%HW6yzwApgsdof_dEAQzXFamrjyK znKHCpD^HY|1v359E3H+Lm2j<^zGLq>kaZp|_%Y$Zu zgF1M{vP+PW4N?CQ9{&P9^*8kGPLuU!Fy)VwUbM4-InXrX)c?zdg`D))#aO4TdcrC4 z8OqSgu+QRW{4nBfA18>QS}iPMY|>eAq4jqbW_sNh^oSrWni;-R( z)vm@P{4l{RF>JSl3Gb!*Dn7ChsBdzRj?8W{B~PIl@F@`wf5U8z_L6q?t5j9ebsgq-aXMZB4PlIKeEtCOP$iB}=VFO2nD^w7xYM_HsuYBq zVVkM-JK)^OF4gNZJV0KWBuItqT%Ca51xp4!R{y&)k`aOg%&QIOrhw;E2`OQ zlSz*dFz}!j8_4)XP?!;_5rtwC$0mvc%C9`1^6Nx80inKB9}! znOWLYD||=6Hn1)Vicw-6V+D+KYwecbt*vc`3Md@yxD^UT`)j71GLv8$OP_{6q|r7a z14h2PM+pwrg;16lJpJ`EDEkb_*a{HIoQROzR2gz`A_+9I@xSi|+Cy}KAG~|!5O~w7 zR9JH6ssGryTze(w8rLiS+2rSZD{^%v*4)0ouP9#XU+u4${^-OYHf;m0Cjf0~j|vjx z+Gpe`49!y!Vi9a%q1tp*X*w&q?Fuo$uQRlWs-@?YrWD-3OAq?8El^=D z#yr#;-yr(fZH3l935(OjYA{cw{c>mNr|yX;A3Q=LvQGN!_G)gi<@1)fl|*?@py!M; z=~{sDZ@zfVy=bo4NRY?0`xss7V6}clj{JKfkX9QWDBIKYQzScUKeh`fer9xj&%Z7e zr!Q}_P=g;$LEcU0?WW`lsy?;R>$wHUFR2N@818(-$xKF;J98KRI)Iqk{VlYqAF)OY zHYFG7hjj(W?Q!w0sPKaLB5>Y)Xcclji3w=&<=LYNKD`IXhc@Rv)fw+hsxDmBMN=yA zof1+(dxuR;i>hDk;1{{}K8?aFvYkVwtZl*|NkIZ&X+Z`!O1RHb48Vlx6^|x zHd%CQkx*3|lO%|GV|~`PjG|SRRfDV$BpN8{cS8#T=0~p@>GZZLx)m5IjgK|SC4<~3;4TnBidc07hS(^|w=fGVDeL)}m7wRn0 zd9k5|W*pk6EX3fc?N1aQeS9lle#D%^)2YUAY+pA9I^QvHqGkCr2h3idJrt)!-!IOQ zW)Dknf6=A7jsYFz56&+5)Sh}EYm`$S5gK|{g!ntQEuEkBVHKd`5^q?vGq_Lvk9zpJE-6RmLZlq`Nyb#!LF_AmgwRC-+ea3%;~wK+wyUmUY!NEBM2 z{0v!|!a5?Wcek@bBO( zKzrbNk171Zd6|YNid7jAT7+*pW%^^px^xVj_$3l{lxJ+M&^ed#101>T2E&yLeB=|S zsZXim+jBIyTIvuA56FTsVT1|C*{O8vTFroTu|}cwutFpfQS%l_ z=B-c{#GCal@pPex8yS}6-KMeB?9)^&P!J@}w_9Y0#goB1^IqCfr4s~eUyHhqxGQXCDQ!^fHQlI*%Z&O)p>891cQTMH7rrCLFlIii3yOb|zt zhBxFF>39&U|KdlCk;_{PqlTeesF)xjVeNctlo zYfbNOW^^{&L9jWv@81?B#R=V?di||4q^h@Pib?^0O?c! z`!g9kRA;`Ybx+rxi`Pyej1!vhgKQXh>4l9D$lH52n`VcFTR@PsQiI*|HiWIH`%H$x zf)`W^axZ;;|B68Ifk3YEP8hNsaY{>cQM=z)`@m_}VzBdk3}Hsm30{zsTFGV`>O2CY z7~>8Hhe=6M%BH^fPI950(Uv8nL-!#65%N*!njKXv&;z0k*!1PDCJYTxpwYS95p+E4 z@I#uevW`jayT7IsYsGUJ>WkzX4%sxztlwqqCvvZmMWaYO)m0OWyg23VXWY9ytvwn!`F{==5u zO5&?!`pJCAsU{7phDuJJUw_R$kd*eK`8-0*$0X2DMiTmksWp-Agg<86Yy^mx{U%@2 z+BcQtdmwpTb|?-NgzxKr&50_%#JmoIWZW3Z88eamH8Md$rv5b4BP${$^q1QiLziwd zv^a5G(c=a3fNnXI&xOeR4dhe6?a_ojTz=nTi50O^?=dC|KZh__bKI9WOUI;!lGKWf zcO|ZPf}Fr0>#?e`)-Xb&+F5b#ndL+pnxiIVxA^_^meGpjd=J|~_0{sy5}U42{o&Set2W-hRb}9w!#=&`FxEceyd>>0p&$DOXu0$8h;wQ_sgs)oHqjw|EkLT$xu)-} zXOrq#Rj!S^x{4Y&S|}93C1#8xRbJDk!NR^gXDcwm-klirTvV5}pIp8Vo8lC1OC|($ z#VWGXw-)lmBt6P-{_)b;ugE<%630nRq-WDv!0)eW&yXovU@6U^L;z2T19n#Z^_z^c z^fs4aYq}GSH2IJ40hrIw9BT?ahb+ngz&W!=+O&bB!HmYjO{RwL!S)1g9!lZk#ngs8 zeOk70x#`Ias9gy7T6XE@*K*hkRQ_p|M zwUWoMX!mqe=IIBPiBA5b*<}eh@o2=?A1eRCP3j$w`B$eFI4b#9iFSE18Qi#n6d)TP zM@kccS$GhqfpzUW2XW0OO$;V4O-xkji4Rirq7#2*sIxH@V3O}HdS~snFze2NX#7lG zu%yj8gpjLLMjAxOqKEge^n?ln@RU)Kk0By7z-6T)L4r zEJybVTV0sx(XJ;c%b51JWi1DU^js)3O|iBWmf)}Fp=(f|m38=9E1bu{CfnBk6A}b{ zcI<9K)!klj>i_A#tj%5$ij~b*bJOpHtac1l&5&ouv*7urFCANWMi2uk zHyz15DUnRAtL(W`T!(+;GlT3^2cV^+uBrVj!qdSfBA|YpNh~Hr^QU$Pkk$%|Cq*8_ zW-14ddtr2uS@#!j#FDDY5`KwgZzCK^^87Rkt+)$A{E^4?$;782C4%CXd|t5R=70)9 zhN`=@uV6w9C?cWBQ93HaE0g zvwgWUqC_;`EPMPKW$9h=WH4#cceS>}XyiCu1YYwa(gXUYN>_#QoRUVc#zNyPlybg*=hzXKr79s;4dOh8 zz8rI4n4hnB7A#2S0x_;Vw}z%WnZlpKj*HHdD6rc=l=#Yb6{x`XzY$>|&nG_QY7 z0-nZH@CJI^HdZ7OdQbkwJxk_JZ(1w<4v3QHw(HT(tf`V+RvZ-EQ9^t3k%4@r)o(#3 zA(Af>pQ09(=sBeZXB(Z66Sqi_%Ww+9gtMp~1-$;r%07LA!LIs;zYl9Z4qf#QkNN?y zg#RS}ZPdW4qpL;K5~^Tv)WFkM!7%&v%${r}a_Ms!B&sMN#(+&U_|Ayx!F4(0~E@=aM6rXHy)nchj&fA93l*s>&GP-poJG1yIt> zjQ7q$0G}Z3P9#|xK9~f7VCN-IvJNN(h(=sNmjCvkAH;qMRaDZ=?nhksslpZ>0Ul(K zkZ_EOD`C3u?J66@qZ178NrTjF9Le~{?M=;q0CbI20rCFo6f=YDDhyb{J9El>Bp&TH zv|Zb@=uB@0!_cer;bm4tQ)X~)k9)@oQ3k|G)~7)T*cs6|Gw*IxV`I#|LBEIM50`&4(|*OHKxPTB+;&H01|negb+{C=rs}- zm!}Y;{KU48@IBf5z)m`ceGT+oZ{%fg*Q_}P(<%rO@GhrwI|d=*ap?5~K*f!``2uqF zSaaM4A;o01!BSc+O2YL(!uONix}hR%{S;NlPYv zY~LaPR?FSk1Ir>*f##3%Mu`9S?i&B06dyog!DnbSOjqbwWY+CUE+pDkK3pTnjME=o z^+&vY*XaYe2q~|fhGc+x8TVUnh@#_^uXoU#6L@+jQFYnj+eic6$E!J26@7R}Tr!L@ zjmA9w?y8a`NEVm%I3|=<%z!gv0v@S_^Iz_dK{~a_@Zg}NlCZbVu*sU@&$m;) z_Xor-Q?439BqXJ;6tQEIf@tW{2F}KV=6^{q$wQF3OE4kbe;hz&l#o8?e@9Oz4S;n4 ziM~+F+;|Iwy2F~?FMGP%(bPt;K>tuQKudOh`MDIWya_#V-CdOBi@FFP$B{T+Vg~kQ z-Td^I?U+bnGFSeFRN3%E7;}G~?kF==<6d~juhh_u5VKx=#n75WHYn;?@b8JInzu4; zKyOr0ntv-Wy>sewo<{na2|-re*?b$%= zxIwC_gkr{%u48dOzw4*}Q*vsd=bucuWRBLEBXV=?c(GD}L=E(D1&#* z3*8>}AU7eWLT+uevb=&jSY(6U>|*`Pmq1)Wr|7jKjBlhfZy>Vp7J{%zYsw{(`FkoSV{$)Z5)KJ)2bx( zQ_dy48n4Zx%mlxI80zYy|A(Q5<ahwT*9i|cpCvjx3Ti@VQhiFg;4%Q)OI^8iO?ylYmP2{`;qnZ%1Y`maJ zS9K2`7yYFcWeG-?y?Y{sEi1~upsr%&Jl(wV{T4D1O(g}^l3LCa{F5n+FcY^^QbwoC zwE3J)x-&7E=&K;*@W1(|KtB%`Z2)JZw)}C>@GBWYHaOJ2xpLk z*%{P{t&!rFf;IY^Kn*^5D?+?gA(3;H@eACRwFUqTh|^G{S$p5O6p{b+3#6d@%^2x@ zh3BYC^!0d8Bh&BO&{}A{lWkPq&iL^N`&0gXfCRhAYMDjgzm6j)cFL9pllvxyUyOF7uE0%}D5c@*%Q>nx5vRU2hYTqKg@)<0MjppCLkkD79 zXx!uam6j2bkb6=^Ji+aZ_SwV&HMzzEx8BDNh>em|`@xJwz+hmBqAbXZEI%+xn<*9p zqTN2i(EzYq0qZ-ZQd@H2tH`;l-9CWBadpqyWN}(MO(xqB8?ncu=nLjQqp1n0{ED@` z3ReMsXCkx~$;XSP`PA0P#x|lJ*Wa+r`M8Ftt>nIitpARVH@Wngz>D}7SxcBk+GPDK zl}NZ+RWQ_5)UJ`FT!5jEba&NIJ>e_w?^?T$^37+BWvMZ!%2OiOi(bKrfHh^I0xLdU z+R*Raw4XAdEm4_b7LCh)6yN;k^?PgcTcoJ@{*pvXS`6}-rRhXMMicN(ocRk9p&*Va zv_1i`WW~44VgHIVCHXN<1RPH`DZb2WR}-i+P@s{%+Oe&bkQV7j+lCzQK!iBOikbg~ zO{Xb&V;69KBjd`zLPg>Q_0Vgh-|Sf#;<$Ayt5|x<k-zmwxU9C|XVqoM;`=~2k|;5% zUq)dqd?_qsY-Vsb)ZJG^z>Lr4G*?o8(43FQk+!nMo8;n*!RJ5=^6hDS^63LI)NZ|{ zHySC8z!-^NAk&4^K(xA@8Ta7hJ2Fquj}3k5G2Osr7|z}4ZChf^3WMp=4}lwgIs=7D-~a{$gTGYs zn@abxTMTP_tNlSn$Z?a}8e$9h$wZ#g%(+B+BOk$~AWNq(tQgg|q3Eo$p~Wf%v3e0` zowMb)!#Z*x@$({K0XF)bBM)>tsaZi zR{RO!o|Pw~B%Qloh#Op?%mVPfw1)J981(d@PGBAvde_Pq-Lb z%B{|eyndZ~^S8~`!))6Rw#-xhkP3*U@u92Zkftfm?$>6LMsh9&B1@zRak09eOk51Y zTHAC2>in4NFFsM>qhNfX|m>bF1$H0p}5bZ`_t{FDQ3 z^20?P<*^x8UcJVy#%@5y7q;k}&WIZ0!q>f$$xspqFLl=IN#xDgUb|-$JhU)LBUn0U z+2Gil!08G%>E$D|UMF#8-x*RqHU+B zoAL9lNK9a`SemOKN&Dv{XTTrFCVFzqAhpkCGkhy zGJ;M7&nFnr&H(BJSpI=9Q;!Mo%kGs9TpZ5gz~-j$0E5q5-!2YC``{(l)QoCZE+rbO zX`Tly+~}D-ka(6qP|>%d>Snd|J%YdFb8Q}$mGIJkn?EndOqNkT!E~&9Wf#8ZdLtw3 zYzH#Gh*4Phh{^4&YBlm-V;t9%eYl^fvj90En=dgGC0J#!}0c^r`#+wQUC0g z$|t76z*a(5^r8_PwBmzF5@Nv{-`bS#a}Nx~SLe_n$Tb__%Yo4Ch*J^JK_t7&`J0q{&# zo8G3=n|C4HHBsODL>|GKwR7e|QDshDtjtJ1pH>1$DH@v0apCvx3B^dbY8=$QLK-sYWOd+Aj`1|LU!6mzGs8*oxM4k4{<&{H zf3aC=IzVY8%NR(CL_=NhV&I+ z4&4gXzTBRTRxtGZD{Ut-pBXx>8czsyiWui@Lnf7)2s4nj2;ZetYx2aO_5s6hOi%6sh3j zoZT)-pN~6jGc=3m!n!5|EnUr2Og=N}Z+9m~W1h!_e@90-w?oxz2V)gtg`OcR2|OPL z>pea4yl16kQ<&fsV&(j3!aC(|}=;8M_-f#d3{6~n(F;`8cP1{msui7KFn zWFWvT2+1kumunqWx!gP(BDJ;FM8jy&7_M+a%N9v1l_7YM3atW0Qo`NwF8c0%JgTY_ z_QmYrb7fH4^D)A+Xsc;0x56M%_=bHCmaIAKkNv+IJL{;pw&l;?5D4z>?ykXI8x8L6 z?lkV&Xh?t%AP^+M-Q6u{g1fuB&*9$p=FOY+o3-Z8UTnJ0-e*hgs&9SDdg=L1O&xJZ zSxYjn-@S6l2vgURM3zKe(Kfnv7Di%j1svZuF4}uWqw+pjRHLa5la{u6pBeZ=ky%XX#<65;?5QtOtR;R}(;^GcVxBe((?ohfxqQuN z1#w-F)RAD-<4ok^UKtaY0aitH%z?YqVofTEv>a>-Zr#%suGY#KOyjyjo-L{p3@$ZCAO&S_2jqd$z?MTKE&+-~2HYY}UB~>dI8v@TQ zpuSXJi8Q#a4o6M-yUVABf8Uvo2BIbYTsuh5H82?hPQi{N+=J`@P$42Jw{I zh$c~)NARf0$AHBNtgN{2?`^8*;E~2&F}2kI;3LR4RTELngdTduh!j1vXxiis{RP2_ zb~6rvJH^lxQyswVF6xD+an0K$~G3@>Y>c};pr_u7DGf(Oi(-iish>XPgfim#uWSdisN zFkT;?AAgn)fmknM_56(s9JhW`6+X5Ok@REL4JLbZhGkM;`7h0|ksmnCj^+IKn1>l7 z9t)|vXgK6?Q4^;9KiXeHyEqUbV3`l}*)#CtKUdMnr}_I0{HK*!+|wmBR-+oV>j@9WW+V`scehxLMZ-b_3ZR1u%{uGu*41 z9TdDOXC^v7v&~C-Nge!T5mx+rVE&tb0?#{Hi;d9Ob>+nNVd%5{zQGHxks_d-a&-v_ zn*5G&D3BEWWE0LNptC5G$M};Jj~{u2a=ud5If|!e<1C5v_Hj{$?{%~hF?eZt%0WGZ z7dK>lGhrIVb*A&{xvS81Aqio&6#CRlq0wBbbvarIzOUD!0T?*m1_uGlW7b}^o7kRD zCd8+QVaKzzjPSIF$mK_pP=v3(_&Cr-)#+KxZ#>~L()oyH$W7rVgn|gfS57ccmtJvpMzeT`zT>* z9t}$QiTiTF^VJs)FmgbFh;_pr9H|nyl9#LUP7)2$hbTJ9$I_yyBiqluGW5JI->E7b zto5KInftvB@cD+w8JnKr-P2#6KIF(IJ^;2=FU(UmA=c&V4p8(tB^mWcc7I?NLBiO# zmcClQ>mKOKgbz6q)x31Q(5BTo6;I9xDfV-a^3>zr?L5D0YYZePreLg&HitJvFrL#i zVPN@y0Ccn>8JX611fM5s=|G&7oJ-LpOM2quo}!Zpql<&UtVB;fyP?kc%+zc&xiN z^+t?on4X~&#YFeUvd4r~)HaKRvgvj#+_9$_=)-1b%dR3XhrISzv~}hqjCB(hmd>?U z)il@Y)4Rt~YaVMf5+aPTY2HY$*|QC&7vt5q?ulVE{j0(dogiv7>dhSXbw^ZxQTVgE z@ok6GJ>pgucPEUth4k>?4It|}nphR43?LCm zc^$5aa|=duf|c?hAq{KEY^$06>pwANU8ouX{1hA@4JbxHUOEmW4VMatZ z0hP@A;I%snhKZo!Wc&NcCOXcCDg(WK%O&9yO2H6P3s2tm$8G@4?{XLHcY3N(w$&}u zx6#M4c4#;`3p~YO&|Yz~i9EfzgSaf)IvC|O0th4%r&%i#wtFyZMc7t zl=Od7-`D@9^ETu7Q~!)?kSfo)oRKK1YG1Ry_hp!^#Fw!r%(wgpODF*p?^#L>xPLHK z+#EM$ZKI#2Vt~)m7vIv!8$bC$Fk16Nrp{$o3ae54-tQlj?CgLqK-qIo$FKg0xrNP< zM;!qfa<2sR{*AZ`3mOUGUt^yW<;UzjjF*dA%Ouwo`_;DKCfEQ&lvnKmhMWmh6ou3KiO>qk?@e0kc_5N0?x#p%tcSYp{pK27MU(^`^$SA-!cX+|1abE^M%yCwpJ-)} z2wi-AN{%brx9MSAk^0-DFcvnXA;4MQRx+5UX??ajZWD!SHv;Sv3WP^KOV6s|X~5?4 zFV<8cXTJEm)|Bn7S0^kSxySYGl>>W4v9F;!M$79Kv9GWnJm3#rN(1tj)|Qo%oQ6!3 zR*kDtn+)Ig3+VLJzqWyjSO3gYx(lsPhXK4ji)qu0ArWZ9Q$zSKO@Q0$-#Dv%`<)32 zEW)SUJlOb8It%eZ(hSDbP z^x$rpGLcT=!wu=Ej&k}xOeK%&jeRaW=4W0@%NFu!_M`vqZlbP>GnBBtGZdgv#nmxAF`I8O13zv&nNs9r{S%rk#Et{S zFR<8k>N@)KOJ+N?1OqYKS3X!_t(ZaXc)rHZgF4O9j#f-Pv7E%l`P7e^7NRdHX6P1h zWxCoQBD7=eMc@${W2F2buMaY9Zd#VcMX~JRD>-Ie6Z^Z-(a6^I3sJ7XVw8%av&M!7 zV-4V^Czv_7f&D)hSr4jAxDdQpki&}6DuhEPNI#Z|ieUcvJru$VU9Cb$Lh?qE*5MN5 zfN1>$d==Qwc9l{W1(~e+=@o|0#bzy}=4N+yzLh5I&yRtLsT0Qa2L=N!VX_fbq6|4$ z@H;-@$MjPR?^T4~XxAP31M169s_3f8)s4O2^9u{O-*(zP0z^uDpFkwbf?$g}e&>_h zwgw+rD;ygsI~JSI@fKK@ohgVe}%lf->U#6jjEgir+6l zp32xmrP<6i&5IF5*5-&Z=|}e*G#ZRHIWW*m+&O?|(80=JF_KI(m#O)9@@?jyQ#NNj z12Xnr2-v#d$@ALskblk@ZiA>gdyZ>5*V$CrrGSYZ9QU7v1x(&TER~UGBmXOt+<5J(#`4Ayqa2?#RmVIA8qjC`DEJB!iH-HiA ze6vubW&GY#bYNsme2xmouyy`tjyVCHZhY8ULF(G#B?42wQndgOWNhYf>U}tHb9wpH z+0ddazsOgBIQ{H~xAzB`9C!5$y32|=-1|fhugtgQJUlu;31|7uJ)>igN3Y&0lP3;! z-WlhXN5JlqGZIx&5w!bx=Z}@|a@k@v8#ThroQsJGJ<`|+GQGUhq zQ^~;&!3?XV*jZs7v(#X7_lL`6D&Ar3Q;JW-ETM}T2)-a32-@T=^>Z~LiUd|)J)#wQ zc0~`g*rK4dpjyPxhB*hHPdy%d>-{6L;zq%z;)&urxYtQe;*S%;hdR2bPKVJ1oCl+n zE8^B}0+(+iyHXnPT3X>!+Koa@2RO&CnsLv^@6FXo;tjsNEjg{t4n{4<;z+MhFu+XWyTc zch6Xc-$+oFC_ql~Owj(*PH#F-wmFS|ymJT>lJ+b4;yP#i(+Hr_L;#K4Iylg@1wPZ` zG*(Q;VlCd!`8b1+#iw>KqLv&Z1q=8B0(^+vh0{(XI2@keU0P#mnK73%0ZBYrJ9E$< z?=eAb-`;k=4U||aiC79?A++G65w(6|5~p4HaOVYL>RpYMD*MWz0k^rvUb{dhkdR}3 z@1Uaeh#&DLv-1aYq|4EVu*ECE%}Ly#9r91+L01lJIJlpbC^E}rtG^if@p0G{tLLU4R;@|2NS2y6 zV}O;X^B1DZfJGD>qEu9YBP<({C!qog32MfA38=N+BiHEL zCdx}iTLa0-;yh>$iy^Gw9dDP&N~kPgT>M2VNEd!V;r8c0(qXl6>sMHOs*>eC926hC z3=khIUx{uH?ZsVFxD4vAvuJ z-3p%1`YUzT-AQydJnD4@xJDX3AAQJi_qZw_ZadS-Iqe<{sjjJOpBZ`0MZY`u46fJH z?rG<{PzzrlbHZwvJii&ZuvaQRU1@j1Js;xG4g)WIDG^`mdmmt03R$T2PG)nt#~>g3 zRT*_VH^4`!;BzLhc9VR3@|P7S{L4c?=IK)eW>?k*ewQ!j3tHQ<+GbNZ57*gGSUXD);>MB6+%$WJFZ;i-GM{qyaXR~Vg;8Psr#^c*@i#*@vwC;QF z-d}mFc#(9fLVe*!;RqD+m5qXRQ9{wJ#TYv8R$ssITDms>nO@%=R(E_NdcTJ`t3wb% zC{r)95ut@ms?CZE>7w2?Yu!QCUhH+GBIM#$3Pl61FF-wgt5qS|@_Ph==K2!=d^@ z^z*QqBkTM7-b=Z)9$1#v9L!SDraT*&tXXZ#!FNn4`Zxo-Jn^^+!WF zlH7f`4ihw?`jfWT;(zFwicbh__)p~?%e zmp*VHgK*U1I>gK5AuZ;dI1YcKpb^pHGtJ;JxFz5i~h>bxtdd}`+ zg>^59UnM&stB?=3G?&2p8E;4(v9sAXl@=Z|_U)r#fPN{MA({;r>bNQC+#DFv?S5=3 zxD_;mel~=G7g3#jw!aPMZOcbtXb9u~@_d{odDw|}oDORslEMF2P@=wp2QB$#IyPHqCU9T(CzS{wn$Xcv8$m0vYI3O-Rd8^e9 zul#5ccrS1!1fb@UT8>J`ucIO?u1)nO~8vmt4SAmxSbyMwIRuwt)Euy6ec&=fDI$MEx%hAaNacr`>n)K^OKP ziw%;D<-#qt^nc2yw##O znsYYg;6u{6AW2N6#oIfbwkQ`DanEZu82t$I+v!_ZQVUd{QG2B0d0>yehPUiD&ON;} z%=(4|Kfai?U;Kz3sc()23^G;`lZ~I(_y&AhnXaRJs(H!HFVYpYJ-`3*_%mb8BoXsP z=7r}9@0wXyc*Z_F)YguhAxoJILlN&%O*pu&6)MGHMPgOEg~l{QB_jHO#a~VVnzcJc zKPFL58IqZF-b%jL6vwvr4FoMu({PAfaVU(q^a(X5rpU)qxd6kL2_K>_GE;NwKOi84 z=+JEKSv~;9OS!VkZae10lQz=4YQho^(RR1SyLP;Tb$Y2FF=7#j9j1-#PHbp$;Q0lE zpH}b8$YU|G@``%a3OG$XdM_d%&rw7q`ZbrnJanh4am`H6U;`lLYQeucZ(!UTtAb zu$eTZ8_H^kq_XvG%bXjg?OcW>(by6Mv+-edm6+Ecanm+cX9@cqoDJO`!{Kekv`X`J zUudNMp?7a&y$kSd02yOIHgZ=#^5Xfiyf`EJtX`{U!O7?HE&Va|TEbKHsp5K|N!4fL z&Mf>}lO5kg$dZ$|?1l?qlI%OCicK}$7uWeqRiu;s-pwy1humgyUf`7o1c=Q(t%w(# zpM21tX1*{sdviy1Tu<4f!NW zl3s~NjM<__R-Ms{@h-DY;Sr_D3$2A*s2tkMXrg`|nj&f;xMm%*L}jNDLc*HIPK8$J zE)RsHYhn-8y?u`;xoOdoiVLQsB2^4@Lq=jhvO0?2CI?2fCeHgiv*5Gm4F2)>dA2H% zGP2TH^(>E9_JqpiPh0uZ>PW#L6vkH=$Cvc|>O2W*72O#PWF%O^swJ@Y)>kBNW8&Z| zMaGEnE_3NR1sbl7N7EZfchsDoVc)CXg{Po)0uMvhQHQ^6;Tb23Q~;47{N#YyfGzlz zveQ!hEEYa0t%<^zXL^~6n1X?0CnOa?ar%u>3gXVXS9XgH!6yIMweZj!ALI5>TD;!9 za3q@p7Q`@3J-t~tzbGG~q)2UJ3R~-O52&n>a$EC8&DOh*h(Ldy-Awh0DhP_A zvD%4@roB&!N=}hdMv16sR3mTsP!21}Gk!<#2+^ALY&J5j2hs)KMzm8WrkB4xQ)`(5 z%pgwHRR%Qdssvo4V5nEXQA?>zNWQYQH2CwYxAcGwCvW~~MTjH}NUn6ut znDRe`Gte-c^Co=A)1BM#hJ5G?u^7%gWNt@9(O>-$L6io9;GK@t39b@s2y`T!zuMya zTV%Q|`yxcrS8oF4X(MmlnZu#ZQ{Zg7=&AHpBWYIE~FGf4Xt z*^E0Z2Dao^EsrpO*8#81nA5Y|50S@!wx|9evMcY#WqmwAvYs$LO`BWE9{iSCi2+Vd z_SG%TMqT(7oqd%PiP(j5UAHEpRZIA@?e5Q_V z-gcRBusb%v;>EI_E`t;^r`k;%rQb!B>ZU~B2C3H?JI%jKJH-9Q(ac`y#&f8+vipy7 zBN1`+gDpt9ZKQABQAtL)Dz5Ee9R#xl2#UAQ;EVpCkK~wz`uw7n=`7aE_D+WL5-pST z`!R8vpqARRUlH{!c;$ofs~aFadlOSRpOvDsn%VVMPwX z#Ctdb5Pe7~U zs_bHaYV3vYQV-;T0Cgh9VE@6lo<23m+NCzLiom&QeK#5(itWhV)({_7ekuA*(32Tz zBHAks0^pBXopvviPBd;%{=*-;-5Gah;;kBOU7AJ_ z5Tla-b>jEP+?bH!@nLb`gWZHn%Id5JEL>+a>M)y^Aa=fjDjOU{PgiAEQy^yPy02kB zWCBB?Gf#qsB=CDawxUXtLo4dmS)fZR7%KVVMTpd5ocX+D)ukoM1CgvkigGr1T}8XF z!a36h4T$G4~iX& z(gG3QA6qs*7afC^v;oKC3ocU$OzocJk^%lII9CU_Kj#;1Bt>-AQ_^_rk>Af3?LuZb z=`HKw{}DMmtoL`AFGIL;(KCby?M5@iFI2EO;$T)ARH;Z_NFr!vOw_A+4$LTygnO$r+qcSW}Z&i|EcL&sR#z! zBFb&52Z4qlU-QuAeXeOtParqsTgZctQN&SyTL)`@s+s7^%-OF28#>S(g2>6syCUtg z=Yaq4$%v#E1yL!TvLMf)eGozh&(t1oxG&yE;}mr*E4>@`>v~ap_e4zzV1&0$_jn)p z$B03c+{vSXO8_^nC%o1?X%($7q1QijB@g)NglOOQN9ju;&F@-57JL^I;_p@HxX&8u z#)=!NSKRP0R$%u3N)uVxKv3$yn8&JG|4mmZpIc!L2RR7CY!1NjwYQY4BY}2cYhx9r zaFDbh5xl{s%Tv962C!@7wHDS9Z2jv=IqSFuI?Kpsdi_#d54UH_hB*SR{wes_ms*>1Ai2VRFw8@*m#-d)L0G zp~>vr+=y~^Ll2$fKt=^`I(eOiU~?pxuwS5Ze^wa~+g8guCl;{+0ytQu(}ZP~*Z=B$ zBc%?T!N)~NTOSzQu=abLB9?`vMcrNr5yi?;MlO*$k;9H1#nK+;)tLrw&QP;^#c>}G zh9{0_3)3tmSV*)KXL*g6z4~}ZzHrLSR+zlSVbWercbZxGPg2h-UxJ_|qlFl_N#fD? z;$vZ>j>D40@919$wX+?||HS$%KVYPwvxd)<$v5C(I`tJ-hj^7XC;aiR?=BHif(dTA z6HWbbcL zWc)6cwIl%=nH(htf+F84%n32(=RQB8f!Y1^UzYBXgsuGNh6EcVg$czMw*gsh9ol|4 zA!%VPdq?4sE;=U#c$4%IQY{jt$e}DX^mccoAhNbKCwOS!qRL?r({a&)1_v(wK<8(b z*Ri0$B;GPI$QOox@D1&`c^##5zoNq0*=H}1F){Kxo^Zfe$`CWBwY zH9K&;(*4)Xd+1;1q_Nliacg}u=UX61)wzpMA{weJCqkDXbx1<2x9}^}FqP&>}mIByKju+k)IiU6U+Aqn< zo1|a5qHXk;0%}^-IOJ{Q)QNz!F)|J9CeNxn!dC3Qp z(z&MGsy3IJZS-&bkn@@IcJR%tx6{umGLow)l@%hmlXz4nB5pyn7x`H;2N$>|k_pqh zD{|>QMI3t`vgJQ0l1riU>nH$v+B{iUjJy=}G}MgTdb;Fpy66v%iWI#ZzMT*uiM^|D zB2~;|X&*Kon4*zwKI}O$wYE>1frGpM=tQ3GXJ(>ddmdYXPBiSTAt2v=lrw)8w3jUI z&nqf}a}W9^3uc%itl{fB1vJMue3~s9^+y_JywS%1Fd|X+!R97XmQ}bHAetqRhD9$O zqdPT<$V~R_JrKX_fZ(|CC$4Rguv1+vrYU|c+{FV*>?&CcRg8wp88d6b- zEf17uHyR=sip-9@C-|4?#aNz=n~VP1e!Gn9CK~M~zl$!x=CcByKk5M-;9G~A1h^@i z>={jBOD1ZSJ@%b88K`H$Tah!bBden#UCEkmK;2->OdpJJYv}jH_o~d+IN7Fjwn$yj zk)N+{OKZH70*G@sg|_Hc{+b$^=KXtbT|_f@^g%~{6u z6^Vxbue+1#mLYy@+&w1)KrECuCi&X@N{%Eh+#T%pJnF^Tpyo;s1im&Y4wX9bJj5J_BLKR|NxZ&Ifw-pYl+X=OE)+fe$LS z5xv_TqU_SLY=A*6%M9?%aE{WYDsh~rg5sm#a?X#a8i3!(!%+vpI1m`GDd8`M;@<7|eLbXY0rXLH{r(@H&Y&f*tMF}sj; zt<{&AsmTp~@>(b;PoImlmG$Y7s>jV-0P9Ygyto2MtzAEuKG&B_VX*_Y>g|szV>f5; zSql0;Hx_B7FKMC}KTL`kgKx!HwLxU+qnXf6zd}^kj&A~RU1MUIFd8(E6mXV}D!$Y{ zCYKL4&d8Y2?KoBvhEK@^;_xmHQnX3E7M`IJmBn>6T)vyOWktIBA5t{mY>M~NvvLDx zGa3e2p10BV;{_>0fh-WOb39b~YWI-?_z?9Cqk-eG$&a>fQ~&n6Wb;HU^UGoX*Ol`A zuYt#XgWuD4K;(JsfRQ%fJxU@FT0TSC6{0ctlRjKq9)bO+ScBIZsrxM^tntq#ymFRV z__FQ0FK{;Qz&Hd^&W@Dz>P=IVNGIQ-<>h6HMdvq3z<54BUig?x*LOap)=P;i(3Kr7 zVcgME{cIuz;~9|5TD{7YLBv)M+0j&!ddx_>c?!pkIOZA%fp5Xk;uW^O*(X2 zhXPs1U(4EbEHE|DYZPhn9jRO0Ev69yN%lsoG29tJLd{Cw<@z;uCkkNJv3Qu4GUJ8- zFC`&n!gsmf;L9jJE!d(41p?i|pr(EM>MOF)CGzZ_zt%4Dwe^XV-VokwZOpNvtk*il zlhNnL3Mpe8LY1d|?~Va(`X5K#W0+q8+jw(Z7cX98%1CdOWW zkjHe+?tR;;f6{O~-fcMDvI`Mae`|zt(Y@B;+%_H38%F!>MMi>;wGkg3y)$s^ zf4hQiJn?k{eWK%JV!f0;ty%DGj?W97x;EK$7ue-TX@+63Gxg^a!()I}!djV8#ihl) zTxj-`n`YWUjTUW8|GDi52wul=3gBC;i0n5pqee$Gjd9Z*yE8vuTsXG84BP_L@tUXR z(?2wf0scPFdGS48%pT4*TPVTDX9sLNoll&c3Dj$4#W76AYEOH3!-_Sq%YfklXmB&W&hg)a;(!Af;T<^6RyRgQ67hR-hcPcwqAm zMB>d)Bol7Q*RSVMQOIC)0fhn)zD!t`mkeUTXHrCcmpZ&PLjAZ@ulaDNwkNgQ^>0i# z=+J5bDb5Fh+X!l@nK*%=3uTI5!z;^KFjPx+igM-^V7fM({1{s*4>#KnxWhy=Hfrbj z*0Np}m|KKSemYYu(eY*;YspT5qr;TGCkN$5IYysG1_V71S&^eAU{WHV@O_Q*p7oiJ zFflOIfgYEUHqp+&p+(6M_kgy&K5Ytw^Cgl?4exxgW&~CtJ@%NdaJFMG<@e}W)lSF|yJ2)~L}K2!EDMC^Bc(>exFoXV3=m0#T%!xt(Poy` zn4gl54<62@9+PZg6JjI4_ILV&zVugi`zEE8cgnxN{A=W+Qi~!Q^e0_LB(gJh;9+*L zxAQ6Z`a?YIyw*ilN(*aR9u^Eg&3fA}`r=G}CK2*=7w7a9wq#yOFv4}}+|e64oi|I( z)e0wX1!P&ldO(y*MtD--+WRGt2Obkdw)oR02|%sxBWD0;!NM zB?-YiPOc1cua)WfE@83T6Vcy>rapwreUTKn!-^@>q)Oy!iu(pj<$iqe4bjh0A>*P! zs!U>-k+T&4TU_z~fH}&GPIpmk`ssvQq}^{JEC%SXh7r~<5u{yb4us5YcR=c60k*x{ z#aDqsoD@_QAd23SUGXPG`Q;YYvY4xf{q*W&k*Gd)Z%`M2Wa$_EI?SrO=|sI=TZ(KZ zr_o1lm1ZzId4c650Ag`4BsNlm*7#HK93pfXfc^$ht{N>J1_#T{^WN&+p;gCqSAQdw zbWB3V5rB;Yy(!g=MwpB$2I36P76IK9$WIhOgeR5d*#NuWt%dVEkgB5zzhq1gh* z2NwnXoD|q|dgXTW3?4Y0+Vrb1Lz}5q@6io_0uyw>;D5_BWpUM?pQjywCIuraK_m|A zKpBNqC-^Wx{s1UT{)dWlZ^T)WqaihO*9jL>uc8?$Q7(9!VAu$SP*9HPzqxJ8w*A2y zR+aWQazq1M_k*?Lol*q!gqRqtORlhfv4AD8`5j?$@)W2XV`fo>?gBy~eR0Cqw9=bx zrTTk7^%F~}P|KfBQTYvpp;`5pu;2Q4b;T{!YFC{@6a7Jkw|)I(ZcN_CT6<0etcivu z8K&n_4ptyhA-(ehRAU}d1~M#1ohaI)*)bt78i@ot2p1J}g-A~kH{xnXjT^_k(vSnN z;8a;d4Z;ZDt-ziyulLQb#I{%No@(x&Pt|l0D5>?R!Apacy)o5%IAJnU;(NW!QToSW z6JLTa*VU$2s8jsvw)t=(;qj+R7?~~nNG{hjcFMu<=(;tlj z`hWN;$Xa6{cz9P;=(rShgq*!s?m`UB)fvB7emRh5_jU*u)APt>jppI%m5^A85HB-F zH`bZ-n*G)mUdzDwRZ%6q>vWpf2qN^uZK3I#S76w07`YOSsi2e%z*hlf1eiK3A_yTLbKjSN`$d5y&?ISQO%IGMC zBLJdaUluFZ@>yyF>QE#~_f*68gzP!J#BjO@v0vl}n~L->Ukr#X-YH@%-<^!$5f%J( z;%vOrX2xr9h;_5goERy+as7cf$7%K2eE#vOB+c|aj!ha+1aU3X>lt;S&cO_QQpFt( zf~G1|4D&-#@VhSe-Ft7@I%1HRwYb_PyX;p~lwzFHJ_0xB0{v-1)YyZlP?b;_mpk|ETD@?KkJKxT)Ed^I{h)2w@6%o_)a?8M>h zaT%bnV5HC0BHS72r4>z>x^l6!T0Rod$tV?+Q~yV5-w)G8LL?+aJI91{wYw|g;iM@f zqM$GCC5G>6hi>|}2lD|0JMgey-e@7)MF@SpSv8k_qsg=!D^gXP=5m6zJM0#w5PeWa z8uwvM3Va6GA0G1q$7G#;<9{QDp#iTL;bNDGAi3>WT%49b1v^%?`uO-mXrxZUYCc@8WsXlT=GV< zk=N?5)It*W1dbZ3Q#VkF@b4!y6#u^NJhr(8KCTl3Ul!yvSjaN&3VF}HJM(@0a%2f9 z>`)xQ(et|J11L%x$vj Date: Wed, 9 Jun 2021 17:47:24 +0800 Subject: [PATCH 14/24] asdf --- examples/benchmarks/TCTS/TCTS.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index e8cbc8005..aa707b26c 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -9,10 +9,12 @@ Sequence learning has attracted much research attention from the machine learnin ### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. + + ![The optimization workflow of one episode.](workflow.png) -At step $$s$$, with training data $$(x_s,y_s)$$, the scheduler $$\varphi(\cdots;\omega)$$ chooses a suitable task $$T_{i_s}$$ (green solid lines) to update the model $$f(\cdots;\theta)$$ (blue solid lines). After $$S$$ steps, we evaluate the model $$f$$ on the validation set $$\Ddev$$ and update the scheduler $$\varphi$$ (green dashed lines). +At step $s$, with training data $(x_s,y_s)$, the scheduler $\varphi(\cdots;\omega)$ chooses a suitable task $T_{i_s}$ (green solid lines) to update the model $f(\cdots;\theta)$ (blue solid lines). After $S$ steps, we evaluate the model $f$ on the validation set $\Ddev$ and update the scheduler $\varphi$ (green dashed lines). ### DataSet * We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. @@ -20,31 +22,31 @@ At step $$s$$, with training data $$(x_s,y_s)$$, the scheduler $$\varphi(\cdots; ### Experiments #### Task Description -* The main tasks $$T_k$$ ($$task_k$$ in Figure1) refers to forecasting return of stock $$i$$ as following, +* The main tasks $T_k$ ($task_k$ in Figure1) refers to forecasting return of stock $i$ as following, ```math -$$ +$ r_{i}^k = \frac{\price_i^{t+k}}{\price_i^{t+k-1}} - 1 -$$ +$ ``` -* Temporally correlated task sets $$\domT_k = {T_1, T_2, ... , T_k}$$, in this paper, $$\domT_3$$, $$\domT_5$$ and $$\domT_10$$ are used. +* Temporally correlated task sets $\domT_k = {T_1, T_2, ... , T_k}$, in this paper, $\domT_3$, $\domT_5$ and $\domT_10$ are used. #### Baselines * GRU/MLP/LightGBM (LGB)/Graph Attention Networks (GAT) * Multi-task learning (MTL): In multi-task learning, multiple tasks are jointly trained and mutually boosted. Each task is treated equally, while in our setting, we focus on the main task. * Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task T_1, then T_2, and gradually move to the last one. #### Result -| Methods | $$T_1$$ | $$T_2$$ | $$T_3$$ | +| Methods | $T_1$ | $T_2$ | $T_3$ | | :----: | :----: | :----: | :----: | | GRU | 0.049 / 1.903 | 0.018 / 1.972 | 0.014 / 1.989 | | MLP | 0.023 / 1.961 | 0.022 / 1.962 | 0.015 / 1.978 | | LGB | 0.038 / 1.883 | 0.023 / 1.952 | 0.007 / 1.987 | | GAT | 0.052 / 1.898 | 0.024 / 1.954 | 0.015 / 1.973 | -| MTL($$\domT_3$$) | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | -| CL($$\domT_3$$) | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | -| Ours($$\domT_3$$) | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | -| MTL($$\domT_5$$) | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | -| CL($$\domT_5$$) | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | -| Ours($$\domT_5$$) | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | -| MTL($$\domT_{10}$$) | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | -| CL($$\domT_{10}$$) | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | -| Ours($$\domT_{10}$$) | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file +| MTL($\domT_3$) | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | +| CL($\domT_3$) | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | +| Ours($\domT_3$) | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | +| MTL($\domT_5$) | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | +| CL($\domT_5$) | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | +| Ours($\domT_5$) | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | +| MTL($\domT_{10}$) | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | +| CL($\domT_{10}$) | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | +| Ours($\domT_{10}$) | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file From 5a3dde93a893f91c69e54f495a67cebb3f2f36ab Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:15:06 +0800 Subject: [PATCH 15/24] update --- examples/benchmarks/TCTS/TCTS.md | 40 +++++++++++++++----------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index aa707b26c..b0389dc9b 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -4,17 +4,17 @@ We provide the code for reproducing the stock trend forecasting experiments in [ ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -![Temporally Correlated Tasks.](task_description.png) +
![Temporally Correlated Tasks.](task_description.png)
### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. -![The optimization workflow of one episode.](workflow.png) +
![The optimization workflow of one episode.](workflow.png)
-At step $s$, with training data $(x_s,y_s)$, the scheduler $\varphi(\cdots;\omega)$ chooses a suitable task $T_{i_s}$ (green solid lines) to update the model $f(\cdots;\theta)$ (blue solid lines). After $S$ steps, we evaluate the model $f$ on the validation set $\Ddev$ and update the scheduler $\varphi$ (green dashed lines). +At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). ### DataSet * We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. @@ -22,31 +22,29 @@ At step $s$, with training data $(x_s,y_s)$, the scheduler $\varphi(\cdots;\omeg ### Experiments #### Task Description -* The main tasks $T_k$ ($task_k$ in Figure1) refers to forecasting return of stock $i$ as following, +* The main tasks ( in Figure1) refers to forecasting return of stock as following, +
+ +
-```math -$ -r_{i}^k = \frac{\price_i^{t+k}}{\price_i^{t+k-1}} - 1 -$ -``` -* Temporally correlated task sets $\domT_k = {T_1, T_2, ... , T_k}$, in this paper, $\domT_3$, $\domT_5$ and $\domT_10$ are used. +* Temporally correlated task sets , in this paper, , and are used. #### Baselines * GRU/MLP/LightGBM (LGB)/Graph Attention Networks (GAT) * Multi-task learning (MTL): In multi-task learning, multiple tasks are jointly trained and mutually boosted. Each task is treated equally, while in our setting, we focus on the main task. -* Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task T_1, then T_2, and gradually move to the last one. +* Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task , then , and gradually move to the last one. #### Result -| Methods | $T_1$ | $T_2$ | $T_3$ | +| Methods | | | | | :----: | :----: | :----: | :----: | | GRU | 0.049 / 1.903 | 0.018 / 1.972 | 0.014 / 1.989 | | MLP | 0.023 / 1.961 | 0.022 / 1.962 | 0.015 / 1.978 | | LGB | 0.038 / 1.883 | 0.023 / 1.952 | 0.007 / 1.987 | | GAT | 0.052 / 1.898 | 0.024 / 1.954 | 0.015 / 1.973 | -| MTL($\domT_3$) | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | -| CL($\domT_3$) | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | -| Ours($\domT_3$) | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | -| MTL($\domT_5$) | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | -| CL($\domT_5$) | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | -| Ours($\domT_5$) | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | -| MTL($\domT_{10}$) | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | -| CL($\domT_{10}$) | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | -| Ours($\domT_{10}$) | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file +| MTL() | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | +| CL() | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | +| Ours() | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | +| MTL() | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | +| CL() | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | +| Ours() | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | +| MTL() | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | +| CL() | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | +| Ours() | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file From 02d0eedd68e75b437e38c32c3f52022a73a62ccc Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:21:16 +0800 Subject: [PATCH 16/24] update --- examples/benchmarks/TCTS/TCTS.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index b0389dc9b..f500fc155 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -1,17 +1,18 @@ # Temporally Correlated Task Scheduling for Sequence Learning -We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://www.overleaf.com/project/5eb8efb42dcf710001d781d6). +We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_nn.py). ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -
![Temporally Correlated Tasks.](task_description.png)
+ +

![Temporally Correlated Tasks.](task_description.png)

### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. -
![The optimization workflow of one episode.](workflow.png)
+

![The optimization workflow of one episode.](workflow.png)

At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). From 38c7b7303adbb1d5061e5f556870aed24d10e159 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:26:50 +0800 Subject: [PATCH 17/24] dsaf --- examples/benchmarks/TCTS/TCTS.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index f500fc155..d450cfff4 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -5,14 +5,16 @@ We provide the code for reproducing the stock trend forecasting experiments in [ Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -

![Temporally Correlated Tasks.](task_description.png)

+
+ +
### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. -

![The optimization workflow of one episode.](workflow.png)

+
![The optimization workflow of one episode.](workflow.png)
At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). From 4c4e77b11ffe3c6e36f6c3b7c642b5b17efd0329 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:28:31 +0800 Subject: [PATCH 18/24] asdf --- examples/benchmarks/TCTS/TCTS.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index d450cfff4..f3113f56b 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -4,9 +4,12 @@ We provide the code for reproducing the stock trend forecasting experiments in [ ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -
- + +
+ +
+
From bb6c1572cadc111af94e012eae2756aa0980e433 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:29:55 +0800 Subject: [PATCH 19/24] asdf --- examples/benchmarks/TCTS/TCTS.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index f3113f56b..eb644ff4a 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -4,13 +4,14 @@ We provide the code for reproducing the stock trend forecasting experiments in [ ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -
+

-

+

+ +

+ +

-
- -
### Method From 89d53853e5630fa339b3c463f6010d6bce69f953 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:30:42 +0800 Subject: [PATCH 20/24] asdf --- examples/benchmarks/TCTS/TCTS.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index eb644ff4a..eab8199e4 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -8,18 +8,13 @@ Sequence learning has attracted much research attention from the machine learnin

-

- -

- - ### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. - -
![The optimization workflow of one episode.](workflow.png)
- +

+ +

At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). From 073fe4668e1cd970a0b3bd89711012823359b623 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:34:31 +0800 Subject: [PATCH 21/24] asdf --- examples/benchmarks/TCTS/TCTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index eab8199e4..581d34f90 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -5,7 +5,7 @@ We provide the code for reproducing the stock trend forecasting experiments in [ Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other.

- +

@@ -13,7 +13,7 @@ Sequence learning has attracted much research attention from the machine learnin Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2.

- +

At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). From d199256d3489368fc03054987b1f3453ef95a0ba Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:35:14 +0800 Subject: [PATCH 22/24] asdf --- examples/benchmarks/TCTS/TCTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index 581d34f90..66c569734 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -5,7 +5,7 @@ We provide the code for reproducing the stock trend forecasting experiments in [ Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other.

- +

From 65ddca133f3d3ae28fe6051134eb1e850a5290d1 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:36:12 +0800 Subject: [PATCH 23/24] asdf --- examples/benchmarks/TCTS/TCTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index 66c569734..5579c42a8 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -1,5 +1,5 @@ # Temporally Correlated Task Scheduling for Sequence Learning -We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_nn.py). +We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py). ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. From 567e42840c5d1a3bf0565ab016a20ea2bf6f3a6e Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:37:25 +0800 Subject: [PATCH 24/24] asdf --- examples/benchmarks/TCTS/TCTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index 5579c42a8..ee67ffbeb 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -1,5 +1,5 @@ # Temporally Correlated Task Scheduling for Sequence Learning -We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py). +We provide the [code](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py) for reproducing the stock trend forecasting experiments. ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other.