From 45f73361e3bbcf6625600c52dccff9b3cdca723c Mon Sep 17 00:00:00 2001 From: lwwang1995 Date: Thu, 18 Mar 2021 11:17:42 +0800 Subject: [PATCH 01/45] add tcts baseline --- .../TCTS/workflow_config_tcts_Alpha360.yaml | 93 +++++ qlib/contrib/model/pytorch_tcts.py | 393 ++++++++++++++++++ 2 files changed, 486 insertions(+) create mode 100644 examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml create mode 100644 qlib/contrib/model/pytorch_tcts.py diff --git a/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml b/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml new file mode 100644 index 000000000..589f4b43e --- /dev/null +++ b/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml @@ -0,0 +1,93 @@ +qlib_init: + provider_uri: "~/.qlib/qlib_data/cn_data" + region: cn +market: &market csi300 +benchmark: &benchmark SH000300 +data_handler_config: &data_handler_config + start_time: 2008-01-01 + end_time: 2020-08-01 + fit_start_time: 2008-01-01 + fit_end_time: 2014-12-31 + instruments: *market + infer_processors: + - class: RobustZScoreNorm + kwargs: + fields_group: feature + clip_outlier: true + - class: Fillna + kwargs: + fields_group: feature + learn_processors: + - class: DropnaLabel + - class: CSRankNorm + kwargs: + fields_group: label + label: ["Ref($close, -2) / Ref($close, -1) - 1", + "Ref($close, -3) / Ref($close, -1) - 1", + "Ref($close, -4) / Ref($close, -1) - 1", + "Ref($close, -5) / Ref($close, -1) - 1", + "Ref($close, -6) / Ref($close, -1) - 1"] +port_analysis_config: &port_analysis_config + strategy: + class: TopkDropoutStrategy + module_path: qlib.contrib.strategy.strategy + kwargs: + topk: 50 + n_drop: 5 + backtest: + verbose: False + limit_threshold: 0.095 + account: 100000000 + benchmark: *benchmark + deal_price: close + open_cost: 0.0005 + close_cost: 0.0015 + min_cost: 5 +task: + model: + class: TCTS + module_path: qlib.contrib.model.pytorch_tcts + kwargs: + d_feat: 6 + hidden_size: 64 + num_layers: 2 + dropout: 0.0 + n_epochs: 200 + lr: 1e-3 + early_stop: 20 + batch_size: 800 + metric: loss + loss: mse + GPU: 0 + fore_optimizer: adam + weight_optimizer: adam + output_dim: 5 + fore_lr: 5e-7 + weight_lr: 5e-7 + steps: 3 + target_label: 0 + dataset: + class: DatasetH + module_path: qlib.data.dataset + kwargs: + handler: + class: Alpha360 + module_path: qlib.contrib.data.handler + kwargs: *data_handler_config + segments: + train: [2008-01-01, 2014-12-31] + valid: [2015-01-01, 2016-12-31] + test: [2017-01-01, 2020-08-01] + record: + - class: SignalRecord + module_path: qlib.workflow.record_temp + kwargs: {} + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 + - class: PortAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + config: *port_analysis_config \ No newline at end of file diff --git a/qlib/contrib/model/pytorch_tcts.py b/qlib/contrib/model/pytorch_tcts.py new file mode 100644 index 000000000..9f44ba31c --- /dev/null +++ b/qlib/contrib/model/pytorch_tcts.py @@ -0,0 +1,393 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + + +from __future__ import division +from __future__ import print_function + +import os +import numpy as np +import pandas as pd +import copy +from sklearn.metrics import roc_auc_score, mean_squared_error +import logging +from ...utils import ( + unpack_archive_with_buffer, + save_multiple_parts_file, + create_save_path, + drop_nan_by_y_index, +) +from ...log import get_module_logger, TimeInspector + +import torch +import torch.nn as nn +import torch.optim as optim + +from ...model.base import Model +from ...data.dataset import DatasetH +from ...data.dataset.handler import DataHandlerLP + + +class TCTS(Model): + """TCTS Model + + Parameters + ---------- + d_feat : int + input dimension for each time step + metric: str + the evaluate metric used in early stop + optimizer : str + optimizer name + GPU : str + the GPU ID(s) used for training + """ + + def __init__( + self, + d_feat=6, + hidden_size=64, + num_layers=2, + dropout=0.0, + n_epochs=200, + batch_size=2000, + early_stop=20, + loss="mse", + fore_optimizer="adam", + weight_optimizer="adam", + output_dim=5, + fore_lr=5e-7, + weight_lr=5e-7, + steps=3, + GPU=0, + seed=None, + target_label=0, + **kwargs + ): + # Set logger. + self.logger = get_module_logger("TCTS") + self.logger.info("TCTS pytorch version...") + + # set hyper-parameters. + self.d_feat = d_feat + self.hidden_size = hidden_size + self.num_layers = num_layers + self.dropout = dropout + self.n_epochs = n_epochs + self.batch_size = batch_size + self.early_stop = early_stop + self.loss = loss + self.device = torch.device("cuda:%d" % (GPU) if torch.cuda.is_available() else "cpu") + self.use_gpu = torch.cuda.is_available() + self.seed = seed + self.output_dim = output_dim + self.fore_lr = fore_lr + self.weight_lr = weight_lr + self.steps = steps + self.target_label = target_label + + self.logger.info( + "TCTS parameters setting:" + "\nd_feat : {}" + "\nhidden_size : {}" + "\nnum_layers : {}" + "\ndropout : {}" + "\nn_epochs : {}" + "\nbatch_size : {}" + "\nearly_stop : {}" + "\nloss_type : {}" + "\nvisible_GPU : {}" + "\nuse_GPU : {}" + "\nseed : {}".format( + d_feat, + hidden_size, + num_layers, + dropout, + n_epochs, + batch_size, + early_stop, + loss, + GPU, + self.use_gpu, + seed, + ) + ) + + if self.seed is not None: + np.random.seed(self.seed) + torch.manual_seed(self.seed) + + self.fore_model = GRUModel( + d_feat=self.d_feat, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + dropout=self.dropout, + ) + self.weight_model = MLPModel( + d_feat=360 + 2 * self.output_dim + 1, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + dropout=self.dropout, + output_dim=self.output_dim, + ) + if fore_optimizer.lower() == "adam": + self.fore_optimizer = optim.Adam(self.fore_model.parameters(), lr=self.fore_lr) + elif fore_optimizer.lower() == "gd": + self.fore_optimizer = optim.SGD(self.fore_model.parameters(), lr=self.fore_lr) + else: + raise NotImplementedError("optimizer {} is not supported!".format(fore_optimizer)) + if weight_optimizer.lower() == "adam": + self.weight_optimizer = optim.Adam(self.weight_model.parameters(), lr=self.weight_lr) + elif weight_optimizer.lower() == "gd": + self.weight_optimizer = optim.SGD(self.weight_model.parameters(), lr=self.weight_lr) + else: + raise NotImplementedError("optimizer {} is not supported!".format(weight_optimizer)) + + self.fitted = False + self.fore_model.to(self.device) + self.weight_model.to(self.device) + + def loss_fn(self, pred, label, weight): + + loc = torch.argmax(weight, 1) + loss = (pred - label[np.arange(weight.shape[0]), loc]) ** 2 + return torch.mean(loss) + + def train_epoch(self, x_train, y_train, x_valid, y_valid): + + x_train_values = x_train.values + y_train_values = np.squeeze(y_train.values) + + indices = np.arange(len(x_train_values)) + np.random.shuffle(indices) + + init_fore_model = copy.deepcopy(self.fore_model) + for p in init_fore_model.parameters(): + p.init_fore_model = False + + self.fore_model.train() + self.weight_model.train() + + for p in self.weight_model.parameters(): + p.requires_grad = False + for p in self.fore_model.parameters(): + p.requires_grad = True + + for i in range(self.steps): + for i in range(len(indices))[:: self.batch_size]: + + if len(indices) - i < self.batch_size: + break + + feature = torch.from_numpy(x_train_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_train_values[indices[i : i + self.batch_size]]).float().to(self.device) + + init_pred = init_fore_model(feature) + pred = self.fore_model(feature) + + dis = init_pred - label.transpose(0, 1) + weight_feature = torch.cat((feature, dis.transpose(0, 1), label, init_pred.view(-1, 1)), 1) + weight = self.weight_model(weight_feature) + + loss = self.loss_fn(pred, label, weight) # hard + + self.fore_optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_value_(self.fore_model.parameters(), 3.0) + self.fore_optimizer.step() + + x_valid_values = x_valid.values + y_valid_values = np.squeeze(y_valid.values) + + indices = np.arange(len(x_valid_values)) + np.random.shuffle(indices) + for p in self.weight_model.parameters(): + p.requires_grad = True + for p in self.fore_model.parameters(): + p.requires_grad = False + + # fix forecasting model and valid weight model + for i in range(len(indices))[:: self.batch_size]: + + if len(indices) - i < self.batch_size: + break + + feature = torch.from_numpy(x_valid_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_valid_values[indices[i : i + self.batch_size]]).float().to(self.device) + + pred = self.fore_model(feature) + dis = pred - label.transpose(0, 1) + weight_feature = torch.cat((feature, dis.transpose(0, 1), label, pred.view(-1, 1)), 1) + weight = self.weight_model(weight_feature) + loc = torch.argmax(weight, 1) + valid_loss = torch.mean((pred - label[:, 0]) ** 2) + loss = torch.mean(-valid_loss * torch.log(weight[np.arange(weight.shape[0]), loc])) + + self.weight_optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_value_(self.weight_model.parameters(), 3.0) + self.weight_optimizer.step() + + def test_epoch(self, data_x, data_y): + + # prepare training data + x_values = data_x.values + y_values = np.squeeze(data_y.values) + + self.fore_model.eval() + + scores = [] + losses = [] + + indices = np.arange(len(x_values)) + + for i in range(len(indices))[:: self.batch_size]: + + if len(indices) - i < self.batch_size: + break + + feature = torch.from_numpy(x_values[indices[i : i + self.batch_size]]).float().to(self.device) + label = torch.from_numpy(y_values[indices[i : i + self.batch_size]]).float().to(self.device) + + pred = self.fore_model(feature) + loss = torch.mean((pred - label[:, abs(self.target_label)]) ** 2) + losses.append(loss.item()) + + return np.mean(losses) + + def fit( + self, + dataset: DatasetH, + evals_result=dict(), + verbose=True, + save_path=None, + ): + + df_train, df_valid, df_test = dataset.prepare( + ["train", "valid", "test"], + col_set=["feature", "label"], + data_key=DataHandlerLP.DK_L, + ) + + x_train, y_train = df_train["feature"], df_train["label"] + x_valid, y_valid = df_valid["feature"], df_valid["label"] + x_test, y_test = df_test["feature"], df_test["label"] + + if save_path == None: + save_path = create_save_path(save_path) + + best_loss = np.inf + best_epoch = 0 + stop_round = 0 + fore_best_param = copy.deepcopy(self.fore_optimizer.state_dict()) + weight_best_param = copy.deepcopy(self.weight_optimizer.state_dict()) + + for epoch in range(self.n_epochs): + print("Epoch:", epoch) + + print("training...") + self.train_epoch(x_train, y_train, x_valid, y_valid) + print("evaluating...") + val_loss = self.test_epoch(x_valid, y_valid) + test_loss = self.test_epoch(x_test, y_test) + + print("valid %.6f, test %.6f" % (val_loss, test_loss)) + + if val_loss < best_loss: + best_loss = val_loss + stop_round = 0 + best_epoch = epoch + torch.save(copy.deepcopy(self.fore_model.state_dict()), save_path + "_fore_model.bin") + torch.save(copy.deepcopy(self.weight_model.state_dict()), save_path + "_weight_model.bin") + + else: + stop_round += 1 + if stop_round >= self.early_stop: + print("early stop") + break + + print("best loss:", best_loss, "@", best_epoch) + best_param = torch.load(save_path + "_fore_model.bin") + self.fore_model.load_state_dict(best_param) + best_param = torch.load(save_path + "_weight_model.bin") + self.weight_model.load_state_dict(best_param) + self.fitted = True + + if self.use_gpu: + torch.cuda.empty_cache() + + def predict(self, dataset): + if not self.fitted: + raise ValueError("model is not fitted yet!") + + x_test = dataset.prepare("test", col_set="feature") + index = x_test.index + self.fore_model.eval() + x_values = x_test.values + sample_num = x_values.shape[0] + preds = [] + + for begin in range(sample_num)[:: self.batch_size]: + + if sample_num - begin < self.batch_size: + end = sample_num + else: + end = begin + self.batch_size + + x_batch = torch.from_numpy(x_values[begin:end]).float().to(self.device) + + with torch.no_grad(): + if self.use_gpu: + pred = self.fore_model(x_batch).detach().cpu().numpy() + else: + pred = self.fore_model(x_batch).detach().numpy() + + preds.append(pred) + + return pd.Series(np.concatenate(preds), index=index) + + +class MLPModel(nn.Module): + def __init__(self, d_feat, hidden_size=256, num_layers=3, dropout=0.0, output_dim=1): + super().__init__() + + self.mlp = nn.Sequential() + self.softmax = nn.Softmax(dim=1) + + for i in range(num_layers): + if i > 0: + self.mlp.add_module("drop_%d" % i, nn.Dropout(dropout)) + self.mlp.add_module("fc_%d" % i, nn.Linear(d_feat if i == 0 else hidden_size, hidden_size)) + self.mlp.add_module("relu_%d" % i, nn.ReLU()) + + self.mlp.add_module("fc_out", nn.Linear(hidden_size, output_dim)) + + def forward(self, x): + # feature + # [N, F] + out = self.mlp(x).squeeze() + out = self.softmax(out) + return out + + +class GRUModel(nn.Module): + def __init__(self, d_feat=6, hidden_size=64, num_layers=2, dropout=0.0): + super().__init__() + + self.rnn = nn.GRU( + input_size=d_feat, + hidden_size=hidden_size, + num_layers=num_layers, + batch_first=True, + dropout=dropout, + ) + self.fc_out = nn.Linear(hidden_size, 1) + + self.d_feat = d_feat + + def forward(self, x): + # x: [N, F*T] + x = x.reshape(len(x), self.d_feat, -1) # [N, F, T] + x = x.permute(0, 2, 1) # [N, T, F] + out, _ = self.rnn(x) + return self.fc_out(out[:, -1, :]).squeeze() From b24af7fff6311a2ff0e8e5456b359febf5d6099c Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Mon, 24 May 2021 05:07:38 +0000 Subject: [PATCH 02/45] multiprocessing support --- .../model_rolling/task_manager_rolling.py | 10 ++- .../online_srv/online_management_simulate.py | 15 +++- .../online_srv/rolling_online_management.py | 22 ++++- qlib/model/trainer.py | 88 ++++++++++++++++++- qlib/workflow/online/manager.py | 29 +++--- qlib/workflow/task/manage.py | 7 +- 6 files changed, 145 insertions(+), 26 deletions(-) diff --git a/examples/model_rolling/task_manager_rolling.py b/examples/model_rolling/task_manager_rolling.py index 4f3ac04b1..89233b37b 100644 --- a/examples/model_rolling/task_manager_rolling.py +++ b/examples/model_rolling/task_manager_rolling.py @@ -4,6 +4,7 @@ """ This example shows how a TrainerRM works based on TaskManager with rolling tasks. After training, how to collect the rolling results will be shown in task_collecting. +Based on the ability of TaskManager, `worker` method offer a simple way for multiprocessing. """ from pprint import pprint @@ -13,10 +14,10 @@ import qlib from qlib.config import REG_CN from qlib.workflow import R from qlib.workflow.task.gen import RollingGen, task_generator -from qlib.workflow.task.manage import TaskManager +from qlib.workflow.task.manage import TaskManager, run_task from qlib.workflow.task.collect import RecorderCollector from qlib.model.ens.group import RollingGroup -from qlib.model.trainer import TrainerRM +from qlib.model.trainer import TrainerRM, task_train data_handler_config = { @@ -122,6 +123,11 @@ class RollingTaskExample: trainer = TrainerRM(self.experiment_name, self.task_pool) trainer.train(tasks) + def worker(self): + # train tasks by other progress or machines for multiprocessing. It is same as TrainerRM.worker. + print("========== worker ==========") + run_task(task_train, self.task_pool, experiment_name=self.experiment_name) + def task_collecting(self): print("========== task_collecting ==========") diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 4bb5022ee..de6dbcb21 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -78,8 +78,8 @@ class OnlineSimulationExample: provider_uri="~/.qlib/qlib_data/cn_data", region="cn", exp_name="rolling_exp", - task_url="mongodb://10.0.0.4:27017/", - task_db_name="rolling_db", + task_url="mongodb://10.0.0.4:27017/", # not necessary when using TrainerR or DelayTrainerR + task_db_name="rolling_db", # not necessary when using TrainerR or DelayTrainerR task_pool="rolling_task", rolling_step=80, start_time="2018-09-10", @@ -113,7 +113,7 @@ class OnlineSimulationExample: self.rolling_gen = RollingGen( step=rolling_step, rtype=RollingGen.ROLL_SD, ds_extra_mod_func=None ) # The rolling tasks generator, ds_extra_mod_func is None because we just need to simulate to 2018-10-31 and needn't change the handler end time. - self.trainer = DelayTrainerRM(self.exp_name, self.task_pool) # Also can be TrainerR, TrainerRM, DelayTrainerR + self.trainer = TrainerRM(self.exp_name, self.task_pool) # Also can be TrainerR, TrainerRM, DelayTrainerR self.rolling_online_manager = OnlineManager( RollingStrategy(exp_name, task_template=tasks, rolling_gen=self.rolling_gen), trainer=self.trainer, @@ -139,6 +139,15 @@ class OnlineSimulationExample: print("========== signals ==========") print(self.rolling_online_manager.get_signals()) + def worker(self): + # train tasks by other progress or machines for multiprocessing + # FIXME: only can call after finishing simulation when using DelayTrainerRM, or there will be some exception. + print("========== worker ==========") + if isinstance(self.trainer, TrainerRM): + self.trainer.worker() + else: + print(f"{type(self.trainer)} is not supported for worker.") + if __name__ == "__main__": ## to run all workflow automatically with your own parameters, use the command below diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index 25b8b2a0c..40da30db7 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -13,10 +13,12 @@ Finally, the OnlineManager will finish second routine and update all strategies. import os import fire import qlib +from qlib.model.trainer import DelayTrainerR, DelayTrainerRM, TrainerR, TrainerRM, end_task_train, task_train from qlib.workflow import R from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.online.manager import OnlineManager +from qlib.workflow.task.manage import TaskManager, run_task data_handler_config = { "start_time": "2013-01-01", @@ -80,8 +82,9 @@ class RollingOnlineExample: self, provider_uri="~/.qlib/qlib_data/cn_data", region="cn", - task_url="mongodb://10.0.0.4:27017/", - task_db_name="rolling_db", + trainer=DelayTrainerRM(), # you can choose from TrainerR, TrainerRM, DelayTrainerR, DelayTrainerRM + task_url="mongodb://10.0.0.4:27017/", # not necessary when using TrainerR or DelayTrainerR + task_db_name="rolling_db", # not necessary when using TrainerR or DelayTrainerR rolling_step=550, tasks=[task_xgboost_config], add_tasks=[task_lgb_config], @@ -104,17 +107,28 @@ class RollingOnlineExample: RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD), ) ) - - self.rolling_online_manager = OnlineManager(strategies) + self.trainer = trainer + self.rolling_online_manager = OnlineManager(strategies, trainer=self.trainer) _ROLLING_MANAGER_PATH = ( ".RollingOnlineExample" # the OnlineManager will dump to this file, for it can be loaded when calling routine. ) + def worker(self): + # train tasks by other progress or machines for multiprocessing + print("========== worker ==========") + if isinstance(self.trainer, TrainerRM): + for task in self.tasks + self.add_tasks: + name_id = task["model"]["class"] + self.trainer.worker(experiment_name=name_id) + else: + print(f"{type(self.trainer)} is not supported for worker.") + # Reset all things to the first status, be careful to save important data def reset(self): for task in self.tasks + self.add_tasks: name_id = task["model"]["class"] + TaskManager(task_pool=name_id).remove() exp = R.get_exp(experiment_name=name_id) for rid in exp.list_recorders(): exp.delete_recorder(rid) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index fd76e6728..07bb839a2 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -12,9 +12,11 @@ In ``DelayTrainer``, the first step is only to save some necessary info to model """ import socket +import time from typing import Callable, List from qlib.data.dataset import Dataset +from qlib.log import get_module_logger from qlib.model.base import Model from qlib.utils import flatten_dict, get_cls_kwargs, init_instance_by_config from qlib.workflow import R @@ -190,6 +192,8 @@ class TrainerR(Trainer): Returns: List[Recorder]: a list of Recorders """ + if isinstance(tasks, dict): + tasks = [tasks] if len(tasks) == 0: return [] if train_func is None: @@ -213,6 +217,8 @@ class TrainerR(Trainer): Returns: List[Recorder]: the same list as the param. """ + if isinstance(recs, Recorder): + recs = [recs] for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs @@ -250,6 +256,8 @@ class DelayTrainerR(TrainerR): Returns: List[Recorder]: a list of Recorders """ + if isinstance(recs, Recorder): + recs = [recs] if end_train_func is None: end_train_func = self.end_train_func if experiment_name is None: @@ -315,6 +323,8 @@ class TrainerRM(Trainer): Returns: List[Recorder]: a list of Recorders """ + if isinstance(tasks, dict): + tasks = [tasks] if len(tasks) == 0: return [] if train_func is None: @@ -329,12 +339,24 @@ class TrainerRM(Trainer): run_task( train_func, task_pool, + query={"filter": {"$in": tasks}}, # only train these tasks experiment_name=experiment_name, before_status=before_status, after_status=after_status, **kwargs, ) + # FIXME: reset to waiting automatically + for _id in _id_list: + is_prn = False + while tm.re_query(_id)["status"] == "running": + if not is_prn: + get_module_logger("TrainerRM").warn( + f"A task (_id: {_id}) is not being trained by this Trainer. Ignore this message if it is being trained by others." + ) + is_prn = True + time.sleep(10) + recs = [] for _id in _id_list: rec = tm.re_query(_id)["res"] @@ -352,10 +374,33 @@ class TrainerRM(Trainer): Returns: List[Recorder]: the same list as the param. """ + if isinstance(recs, Recorder): + recs = [recs] for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs + def worker( + self, + train_func: Callable = None, + experiment_name: str = None, + ): + """ + The multiprocessing method for `train`. It can share a same task_pool with `train` and can run in other progress or other machines. + + Args: + train_func (Callable): the training method which needs at least `task`s and `experiment_name`. None for the default training method. + experiment_name (str): the experiment name, None for use default name. + """ + if train_func is None: + train_func = self.train_func + if experiment_name is None: + experiment_name = self.experiment_name + task_pool = self.task_pool + if task_pool is None: + task_pool = experiment_name + run_task(train_func, task_pool=task_pool, experiment_name=experiment_name) + class DelayTrainerRM(TrainerRM): """ @@ -395,6 +440,8 @@ class DelayTrainerRM(TrainerRM): Returns: List[Recorder]: a list of Recorders """ + if isinstance(tasks, dict): + tasks = [tasks] if len(tasks) == 0: return [] return super().train( @@ -410,8 +457,6 @@ class DelayTrainerRM(TrainerRM): Given a list of Recorder and return a list of trained Recorder. This class will finish real data loading and model fitting. - NOTE: This method will train all STATUS_PART_DONE tasks in the task pool, not only the ``recs``. - Args: recs (list): a list of Recorder, the tasks have been saved to them. end_train_func (Callable, optional): the end_train method which need at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. @@ -421,7 +466,8 @@ class DelayTrainerRM(TrainerRM): Returns: List[Recorder]: a list of Recorders """ - + if isinstance(recs, Recorder): + recs = [recs] if end_train_func is None: end_train_func = self.end_train_func if experiment_name is None: @@ -441,6 +487,42 @@ class DelayTrainerRM(TrainerRM): before_status=TaskManager.STATUS_PART_DONE, **kwargs, ) + + # FIXME: reset to waiting automatically + tm = TaskManager(task_pool=task_pool) + for query_task in tm.query({"filter": {"$in": tasks}}): + _id = query_task["_id"] + is_prn = False + while tm.re_query(_id)["status"] == "running": + if not is_prn: + get_module_logger("DelayTrainerRM").warn( + f"A task (_id: {_id}) is not being trained by this Trainer. Ignore this message if it is being trained by others." + ) + is_prn = True + time.sleep(10) + for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) return recs + + def worker(self, end_train_func=None, experiment_name: str = None): + """ + The multiprocessing method for `end_train`. It can share a same task_pool with `end_train` and can run in other progress or other machines. + + Args: + end_train_func (Callable, optional): the end_train method which need at least `recorder`s and `experiment_name`. Defaults to None for using self.end_train_func. + experiment_name (str): the experiment name, None for use default name. + """ + if end_train_func is None: + end_train_func = self.end_train_func + if experiment_name is None: + experiment_name = self.experiment_name + task_pool = self.task_pool + if task_pool is None: + task_pool = experiment_name + run_task( + end_train_func, + task_pool=task_pool, + experiment_name=experiment_name, + before_status=TaskManager.STATUS_PART_DONE, + ) diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index 443cd61ad..ef6cb8dfa 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -18,10 +18,12 @@ There are 4 total situations for using different trainers in different situation ========================= =================================================================================== Situations Description ========================= =================================================================================== -Online + Trainer When you REAL want to do a routine, the Trainer will help you train the models. +Online + Trainer When you want to do a REAL routine, the Trainer will help you train the models. It + will train models task by task and strategy by strategy. -Online + DelayTrainer In normal online routine, whether Trainer or DelayTrainer will REAL train models - in this routine. So it is not necessary to use DelayTrainer when do a REAL routine. +Online + DelayTrainer When your models don't have any temporal dependence, the DelayTrainer will train + nothing until all tasks have been prepared. It makes user can train all tasks in + the end of `routine` or `first_train`. Simulation + Trainer When your models have some temporal dependence on the previous models, then you need to consider using Trainer. This means it will REAL train your models in @@ -103,17 +105,21 @@ class OnlineManager(Serializable): """ if strategies is None: strategies = self.strategies - for strategy in strategies: + models_list = [] + for strategy in strategies: self.logger.info(f"Strategy `{strategy.name_id}` begins first training...") tasks = strategy.first_tasks() models = self.trainer.train(tasks, experiment_name=strategy.name_id) - models = self.trainer.end_train(models, experiment_name=strategy.name_id) + models_list.append(models) self.logger.info(f"Finished training {len(models)} models.") - online_models = strategy.prepare_online_models(models, **model_kwargs) self.history.setdefault(self.cur_time, {})[strategy] = online_models + if not self.status == self.STATUS_SIMULATING or not self.trainer.is_delay(): + for strategy, models in zip(strategies, models_list): + models = self.trainer.end_train(models, experiment_name=strategy.name_id) + def routine( self, cur_time: Union[str, pd.Timestamp] = None, @@ -139,20 +145,22 @@ class OnlineManager(Serializable): cur_time = D.calendar(freq=self.freq).max() self.cur_time = pd.Timestamp(cur_time) # None for latest date + models_list = [] for strategy in self.strategies: self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") if self.status == self.STATUS_NORMAL: strategy.tool.update_online_pred() tasks = strategy.prepare_tasks(self.cur_time, **task_kwargs) - models = self.trainer.train(tasks) - if self.status == self.STATUS_NORMAL or not self.trainer.is_delay(): - models = self.trainer.end_train(models, experiment_name=strategy.name_id) + models = self.trainer.train(tasks, experiment_name=strategy.name_id) + models_list.append(models) self.logger.info(f"Finished training {len(models)} models.") online_models = strategy.prepare_online_models(models, **model_kwargs) self.history.setdefault(self.cur_time, {})[strategy] = online_models - if not self.trainer.is_delay(): + if not self.status == self.STATUS_SIMULATING or not self.trainer.is_delay(): + for strategy, models in zip(self.strategies, models_list): + models = self.trainer.end_train(models, experiment_name=strategy.name_id) self.prepare_signals(**signal_kwargs) def get_collector(self) -> MergeCollector: @@ -297,6 +305,7 @@ class OnlineManager(Serializable): # NOTE: Assumption: the predictions of online models need less than next cur_time, or this method will work in a wrong way. self.prepare_signals(**signal_kwargs) if signals_time > cur_time: + # FIXME: if use DelayTrainer and worker (and worker is faster than main progress), there are some possibilities of showing this warning. self.logger.warn( f"The signals have already parpred to {signals_time} by last preparation, but current time is only {cur_time}. This may be because the online models predict more than they should, which can cause signals to be contaminated by the offline models." ) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 658eec4d6..0e495bb0f 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -69,7 +69,7 @@ class TaskManager: ENCODE_FIELDS_PREFIX = ["def", "res"] - def __init__(self, task_pool: str = None): + def __init__(self, task_pool: str): """ Init Task Manager, remember to make the statement of MongoDB url and database name firstly. @@ -79,8 +79,7 @@ class TaskManager: the name of Collection in MongoDB """ self.mdb = get_mongodb() - if task_pool is not None: - self.task_pool = getattr(self.mdb, task_pool) + self.task_pool = getattr(self.mdb, task_pool) self.logger = get_module_logger(self.__class__.__name__) def list(self) -> list: @@ -288,7 +287,7 @@ class TaskManager: for t in self.task_pool.find(query): yield self._decode_task(t) - def re_query(self, _id): + def re_query(self, _id) -> dict: """ Use _id to query task. From 6222940b9c387f0cca8a61dc25f93bc8bd678f97 Mon Sep 17 00:00:00 2001 From: al Date: Wed, 26 May 2021 17:50:49 +0800 Subject: [PATCH 03/45] Update README.md fix typo --- scripts/data_collector/yahoo/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/data_collector/yahoo/README.md b/scripts/data_collector/yahoo/README.md index b9fd9123c..0413f32b6 100644 --- a/scripts/data_collector/yahoo/README.md +++ b/scripts/data_collector/yahoo/README.md @@ -121,7 +121,7 @@ df = D.features(D.instruments("all"), ["$close"], freq="day") ### Help ```bash -pythono collector.py collector_data --help +python collector.py collector_data --help ``` ## Parameters From b884c8c571602482a7355df955ff2f14ee16b5f7 Mon Sep 17 00:00:00 2001 From: al Date: Wed, 26 May 2021 18:00:23 +0800 Subject: [PATCH 04/45] Update collector.py fix typo --- scripts/data_collector/yahoo/collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/data_collector/yahoo/collector.py b/scripts/data_collector/yahoo/collector.py index a6e06613e..b92f7773b 100644 --- a/scripts/data_collector/yahoo/collector.py +++ b/scripts/data_collector/yahoo/collector.py @@ -191,7 +191,7 @@ class YahooCollector(BaseCollector): class YahooCollectorCN(YahooCollector, ABC): def get_instrument_list(self): - logger.info("get HS stock symbos......") + logger.info("get HS stock symbols......") symbols = get_hs_stock_symbols() logger.info(f"get {len(symbols)} symbols.") return symbols From 9b431bc5035c64a60e02b21ede72e5d7788c76f7 Mon Sep 17 00:00:00 2001 From: al Date: Wed, 26 May 2021 22:01:15 +0800 Subject: [PATCH 05/45] Update 1min demo data in CSV format --- docs/component/data.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/component/data.rst b/docs/component/data.rst index 0a650c523..6bc55cf6c 100644 --- a/docs/component/data.rst +++ b/docs/component/data.rst @@ -72,12 +72,14 @@ Converting CSV Format into Qlib Format ``Qlib`` has provided the script ``scripts/dump_bin.py`` to convert **any** data in CSV format into `.bin` files (``Qlib`` format) as long as they are in the correct format. -Users can download the demo china-stock data in CSV format as follows for reference to the CSV format. +Users can download the 1 day demo china-stock data in CSV format as follows for reference to the CSV format. .. code-block:: bash python scripts/get_data.py csv_data_cn --target_dir ~/.qlib/csv_data/cn_data +For 1min demo, please refer to the script `here `_. + Users can also provide their own data in CSV format. However, the CSV data **must satisfies** following criterions: - CSV file is named after a specific stock *or* the CSV file includes a column of the stock name From 5a382d7e99a96c7af05948f6e586dc47a9662ab2 Mon Sep 17 00:00:00 2001 From: al Date: Thu, 27 May 2021 12:40:55 +0800 Subject: [PATCH 06/45] Update data.rst Update csv format according to feedback --- docs/component/data.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/component/data.rst b/docs/component/data.rst index 6bc55cf6c..ff0d0c2d7 100644 --- a/docs/component/data.rst +++ b/docs/component/data.rst @@ -72,13 +72,18 @@ Converting CSV Format into Qlib Format ``Qlib`` has provided the script ``scripts/dump_bin.py`` to convert **any** data in CSV format into `.bin` files (``Qlib`` format) as long as they are in the correct format. -Users can download the 1 day demo china-stock data in CSV format as follows for reference to the CSV format. +Besides downloading the prepared demo data, users could download demo data directly from the Collector as follows for reference to the CSV format. +Here are some example: -.. code-block:: bash +for daily data: + .. code-block:: bash python scripts/get_data.py csv_data_cn --target_dir ~/.qlib/csv_data/cn_data -For 1min demo, please refer to the script `here `_. +for 1min data: + .. code-block:: bash + + python scripts/data_collector/yahoo/collector.py download_data --source_dir ~/.qlib/stock_data/source/cn_1min --region CN --start 2021-05-20 --end 2021-05-23 --delay 0.1 --interval 1min --limit_nums 10 Users can also provide their own data in CSV format. However, the CSV data **must satisfies** following criterions: From ca0363ded804ad97d21d2d151ef823df9336a7c5 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 27 May 2021 06:04:46 +0000 Subject: [PATCH 07/45] update trainer and manage --- qlib/model/trainer.py | 38 ++++++++++++------------------------ qlib/workflow/task/manage.py | 34 ++++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 07bb839a2..ace3031ed 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -283,6 +283,9 @@ class TrainerRM(Trainer): STATUS_BEGIN = "begin_task_train" STATUS_END = "end_task_train" + # This tag is the _id in TaskManager to distinguish tasks. + TM_ID = "_id in TaskManager" + def __init__(self, experiment_name: str = None, task_pool: str = None, train_func=task_train): """ Init TrainerR. @@ -336,31 +339,24 @@ class TrainerRM(Trainer): task_pool = experiment_name tm = TaskManager(task_pool=task_pool) _id_list = tm.create_task(tasks) # all tasks will be saved to MongoDB + query = {"_id": {"$in": _id_list}} run_task( train_func, task_pool, - query={"filter": {"$in": tasks}}, # only train these tasks + query=query, # only train these tasks experiment_name=experiment_name, before_status=before_status, after_status=after_status, **kwargs, ) - # FIXME: reset to waiting automatically - for _id in _id_list: - is_prn = False - while tm.re_query(_id)["status"] == "running": - if not is_prn: - get_module_logger("TrainerRM").warn( - f"A task (_id: {_id}) is not being trained by this Trainer. Ignore this message if it is being trained by others." - ) - is_prn = True - time.sleep(10) + tm.wait(query=query) recs = [] for _id in _id_list: rec = tm.re_query(_id)["res"] rec.set_tags(**{self.STATUS_KEY: self.STATUS_BEGIN}) + rec.set_tags(**{self.TM_ID: _id}) recs.append(rec) return recs @@ -475,31 +471,21 @@ class DelayTrainerRM(TrainerRM): task_pool = self.task_pool if task_pool is None: task_pool = experiment_name - tasks = [] + _id_list = [] for rec in recs: - tasks.append(rec.load_object("task")) + _id_list.append(rec.list_tags()[self.TM_ID]) + query = {"_id": {"$in": _id_list}} run_task( end_train_func, task_pool, - query={"filter": {"$in": tasks}}, # only train these tasks + query=query, # only train these tasks experiment_name=experiment_name, before_status=TaskManager.STATUS_PART_DONE, **kwargs, ) - # FIXME: reset to waiting automatically - tm = TaskManager(task_pool=task_pool) - for query_task in tm.query({"filter": {"$in": tasks}}): - _id = query_task["_id"] - is_prn = False - while tm.re_query(_id)["status"] == "running": - if not is_prn: - get_module_logger("DelayTrainerRM").warn( - f"A task (_id: {_id}) is not being trained by this Trainer. Ignore this message if it is being trained by others." - ) - is_prn = True - time.sleep(10) + TaskManager(task_pool=task_pool).wait(query=query) for rec in recs: rec.set_tags(**{self.STATUS_KEY: self.STATUS_END}) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 0e495bb0f..167087260 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -108,6 +108,15 @@ class TaskManager: def _dict_to_str(self, flt): return {k: str(v) for k, v in flt.items()} + def _decode_query(self, query): + if "_id" in query: + if isinstance(query["_id"], dict): + for key in query["_id"]: + query["_id"][key] = [ObjectId(i) for i in query["_id"][key]] + else: + query["_id"] = ObjectId(query["_id"]) + return query + def replace_task(self, task, new_task): """ Use a new task to replace a old one @@ -223,8 +232,7 @@ class TaskManager: dict: a task(document in collection) after decoding """ query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) query.update({"status": status}) task = self.task_pool.find_one_and_update( query, {"$set": {"status": self.STATUS_RUNNING}}, sort=[("priority", pymongo.DESCENDING)] @@ -282,8 +290,7 @@ class TaskManager: dict: a task(document in collection) after decoding """ query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) for t in self.task_pool.find(query): yield self._decode_task(t) @@ -338,8 +345,7 @@ class TaskManager: """ query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) self.task_pool.delete_many(query) def task_stat(self, query={}) -> dict: @@ -353,8 +359,7 @@ class TaskManager: dict """ query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) tasks = self.query(query=query, decode=False) status_stat = {} for t in tasks: @@ -376,8 +381,7 @@ class TaskManager: def reset_status(self, query, status): query = query.copy() - if "_id" in query: - query["_id"] = ObjectId(query["_id"]) + query = self._decode_query(query) print(self.task_pool.update_many(query, {"$set": {"status": status}})) def prioritize(self, task, priority: int): @@ -401,9 +405,19 @@ class TaskManager: return sum(task_stat.values()) def wait(self, query={}): + """ + When multiprocessing, the main progress may fetch nothing from TaskManager because there are still some running tasks. + So main progress should wait until all tasks are trained well by other progress or machines. + + Args: + query (dict, optional): the query dict. Defaults to {}. + """ task_stat = self.task_stat(query) total = self._get_total(task_stat) last_undone_n = self._get_undone_n(task_stat) + if last_undone_n == 0: + return + self.logger.warn(f"Waiting for {last_undone_n} undone tasks. Please make sure they are running.") with tqdm(total=total, initial=total - last_undone_n) as pbar: while True: time.sleep(10) From c12c861b7a99093d09bf426f2bb89b1529fa69dc Mon Sep 17 00:00:00 2001 From: al Date: Thu, 27 May 2021 19:37:57 +0800 Subject: [PATCH 08/45] Remove repeated package from requirements --- scripts/data_collector/yahoo/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/data_collector/yahoo/requirements.txt b/scripts/data_collector/yahoo/requirements.txt index 3e3e0d1e0..5f08026e5 100644 --- a/scripts/data_collector/yahoo/requirements.txt +++ b/scripts/data_collector/yahoo/requirements.txt @@ -5,5 +5,4 @@ numpy pandas tqdm lxml -loguru yahooquery From 7ceec37848b7840bd4c4f995afc26ba07788038f Mon Sep 17 00:00:00 2001 From: al Date: Thu, 27 May 2021 22:35:43 +0800 Subject: [PATCH 09/45] Update integration.rst Fix typo --- docs/start/integration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/start/integration.rst b/docs/start/integration.rst index 3ecae1090..3d4043826 100644 --- a/docs/start/integration.rst +++ b/docs/start/integration.rst @@ -82,7 +82,7 @@ The Custom models need to inherit `qlib.model.base.Model <../reference/api.html# return pd.Series(self.model.predict(x_test.values), index=x_test.index) - Override the `finetune` method (Optional) - - This method is optional to the users, and when users one to use this method on their own models, they should inherit the ``ModelFT`` base class, which includes the interface of `finetune`. + - This method is optional to the users. When users want to use this method on their own models, they should inherit the ``ModelFT`` base class, which includes the interface of `finetune`. - The parameters must include the parameter `dataset`. - Code Example: In the following example, users will use `LightGBM` as the model and finetune it. .. code-block:: Python From e409bee9b9e779d73296c11b533d9cc6026f1872 Mon Sep 17 00:00:00 2001 From: al Date: Fri, 28 May 2021 07:54:45 +0800 Subject: [PATCH 10/45] Update report.rst typo --- docs/component/report.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/component/report.rst b/docs/component/report.rst index 7d8053c78..6f4bff4f9 100644 --- a/docs/component/report.rst +++ b/docs/component/report.rst @@ -101,7 +101,7 @@ Graphical Result - Axis Y: - `ic` The `Pearson correlation coefficient` series between `label` and `prediction score`. - In the above example, the `label` is formulated as `Ref($close, -1)/$close - 1`. Please refer to `Data Featrue `_ for more details. + In the above example, the `label` is formulated as `Ref($close, -1)/$close - 1`. Please refer to `Data Feature `_ for more details. - `rank_ic` The `Spearman's rank correlation coefficient` series between `label` and `prediction score`. From 02e34eb9e9f7d65d9dd782e8d023e44b9014619e Mon Sep 17 00:00:00 2001 From: al Date: Sun, 30 May 2021 08:27:21 +0800 Subject: [PATCH 11/45] Add import stock pool (csi300) in documentation --- docs/component/data.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/component/data.rst b/docs/component/data.rst index ff0d0c2d7..cd30ee98b 100644 --- a/docs/component/data.rst +++ b/docs/component/data.rst @@ -152,6 +152,16 @@ After conversion, users can find their Qlib format data in the directory `~/.qli In the convention of `Qlib` data processing, `open, close, high, low, volume, money and factor` will be set to NaN if the stock is suspended. +Stock Pool (Market) +-------------------------------- + +``Qlib`` defines `stock pool `_ as stock list and their date ranges. Predefined stock pools (e.g. csi300) may be imported as follows. + +.. code-block:: bash + + python collector.py --index_name CSI300 --qlib_dir --method parse_instruments + + Multiple Stock Modes -------------------------------- From 4ff0c4fb0f98abfd36faaf666cd8f6811c3a7cc7 Mon Sep 17 00:00:00 2001 From: you-n-g Date: Mon, 31 May 2021 08:52:41 +0800 Subject: [PATCH 12/45] Update strategy.rst --- docs/component/strategy.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/component/strategy.rst b/docs/component/strategy.rst index 0720dcdad..e4a5a94d1 100644 --- a/docs/component/strategy.rst +++ b/docs/component/strategy.rst @@ -111,8 +111,6 @@ Usage & Example pred_score, strategy=strategy, **BACKTEST_CONFIG ) -Also, the above example has been given in ``examples/train_backtest_analyze.ipynb``. - To know more about the `prediction score` `pred_score` output by ``Forecast Model``, please refer to `Forecast Model: Model Training & Prediction `_. To know more about ``Intraday Trading``, please refer to `Intraday Trading: Model&Strategy Testing `_. From 94ab4bbf3feb5496720c6359dc85cfb1766ed5dd Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 1 Jun 2021 07:45:39 +0000 Subject: [PATCH 13/45] add docs --- qlib/workflow/task/manage.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 167087260..dd42caf65 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -24,7 +24,9 @@ from bson.binary import Binary from bson.objectid import ObjectId from pymongo.errors import InvalidDocument from qlib import auto_init, get_module_logger +import qlib from tqdm.cli import tqdm +import yaml from .utils import get_mongodb @@ -72,24 +74,26 @@ class TaskManager: def __init__(self, task_pool: str): """ Init Task Manager, remember to make the statement of MongoDB url and database name firstly. + A TaskManager instance serves a specific task pool. + The static method of this module serves the whole MongoDB. Parameters ---------- task_pool: str the name of Collection in MongoDB """ - self.mdb = get_mongodb() - self.task_pool = getattr(self.mdb, task_pool) + self.task_pool = getattr(get_mongodb(), task_pool) self.logger = get_module_logger(self.__class__.__name__) - def list(self) -> list: + @staticmethod + def list() -> list: """ - List the all collection(task_pool) of the db + List the all collection(task_pool) of the db. Returns: list """ - return self.mdb.list_collection_names() + return get_mongodb().list_collection_names() def _encode_task(self, task): for prefix in self.ENCODE_FIELDS_PREFIX: @@ -109,6 +113,16 @@ class TaskManager: return {k: str(v) for k, v in flt.items()} def _decode_query(self, query): + """ + If the query includes any `_id`, then it needs `ObjectId` to decode. + For example, when using TrainerRM, it needs query `{"_id": {"$in": _id_list}}`. Then we need to `ObjectId` every `_id` in `_id_list`. + + Args: + query (dict): query dict. Defaults to {}. + + Returns: + dict: the query after decoding. + """ if "_id" in query: if isinstance(query["_id"], dict): for key in query["_id"]: From ab6b88ce14814fe5679e4f5b5f9a016a2397c1a6 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 1 Jun 2021 07:48:14 +0000 Subject: [PATCH 14/45] delete useless import --- qlib/workflow/task/manage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index dd42caf65..7a85036da 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -24,9 +24,7 @@ from bson.binary import Binary from bson.objectid import ObjectId from pymongo.errors import InvalidDocument from qlib import auto_init, get_module_logger -import qlib from tqdm.cli import tqdm -import yaml from .utils import get_mongodb From 8d05cd2dafcb8f8dbf2bcdb453b5d9236d3bd766 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Tue, 1 Jun 2021 09:40:53 +0000 Subject: [PATCH 15/45] modify tests.config.py --- .../online_srv/online_management_simulate.py | 64 ++++++++++++- .../online_srv/rolling_online_management.py | 6 +- qlib/tests/config.py | 94 ++++++++++++++----- 3 files changed, 138 insertions(+), 26 deletions(-) diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 5f024192f..8650859ff 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -5,6 +5,7 @@ This example is about how can simulate the OnlineManager based on rolling tasks. """ +from pprint import pprint import fire import qlib from qlib.model.trainer import DelayTrainerR, DelayTrainerRM, TrainerR, TrainerRM @@ -13,7 +14,63 @@ from qlib.workflow.online.manager import OnlineManager from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.task.manage import TaskManager -from qlib.tests.config import CSI100_RECORD_LGB_TASK_CONFIG, CSI100_RECORD_XGBOOST_TASK_CONFIG +from qlib.tests.config import CSI100_RECORD_LGB_TASK_CONFIG_ONLINE, CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE + +data_handler_config = { + "start_time": "2018-01-01", + "end_time": "2018-10-31", + "fit_start_time": "2018-01-01", + "fit_end_time": "2018-03-31", + "instruments": "csi100", +} + +dataset_config = { + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": { + "class": "Alpha158", + "module_path": "qlib.contrib.data.handler", + "kwargs": data_handler_config, + }, + "segments": { + "train": ("2018-01-01", "2018-03-31"), + "valid": ("2018-04-01", "2018-05-31"), + "test": ("2018-06-01", "2018-09-10"), + }, + }, +} + +record_config = [ + { + "class": "SignalRecord", + "module_path": "qlib.workflow.record_temp", + }, + { + "class": "SigAnaRecord", + "module_path": "qlib.workflow.record_temp", + }, +] + +# use lgb model +task_lgb_config = { + "model": { + "class": "LGBModel", + "module_path": "qlib.contrib.model.gbdt", + }, + "dataset": dataset_config, + "record": record_config, +} + +# use xgboost model +task_xgboost_config = { + "model": { + "class": "XGBModel", + "module_path": "qlib.contrib.model.xgboost", + }, + "dataset": dataset_config, + "record": record_config, +} class OnlineSimulationExample: @@ -46,7 +103,10 @@ class OnlineSimulationExample: tasks (dict or list[dict]): a set of the task config waiting for rolling and training """ if tasks is None: - tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG, CSI100_RECORD_LGB_TASK_CONFIG] + #tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE, CSI100_RECORD_LGB_TASK_CONFIG_ONLINE] + tasks = [task_xgboost_config, task_lgb_config] + #pprint(CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE) + #pprint(task_xgboost_config) self.exp_name = exp_name self.task_pool = task_pool self.start_time = start_time diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index b4f7245b7..99a91e027 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -18,7 +18,7 @@ from qlib.workflow import R from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.online.manager import OnlineManager -from qlib.tests.config import CSI100_RECORD_XGBOOST_TASK_CONFIG, CSI100_RECORD_LGB_TASK_CONFIG +from qlib.tests.config import CSI100_RECORD_XGBOOST_TASK_CONFIG_ROLLING, CSI100_RECORD_LGB_TASK_CONFIG_ROLLING class RollingOnlineExample: @@ -34,9 +34,9 @@ class RollingOnlineExample: add_tasks=None, ): if add_tasks is None: - add_tasks = [CSI100_RECORD_LGB_TASK_CONFIG] + add_tasks = [CSI100_RECORD_LGB_TASK_CONFIG_ROLLING] if tasks is None: - tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG] + tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG_ROLLING] mongo_conf = { "task_url": task_url, # your MongoDB url "task_db_name": task_db_name, # database name diff --git a/qlib/tests/config.py b/qlib/tests/config.py index 80461f6f9..c61b5651e 100644 --- a/qlib/tests/config.py +++ b/qlib/tests/config.py @@ -43,17 +43,29 @@ RECORD_CONFIG = [ ] -def get_data_handler_config(market=CSI300_MARKET): +def get_data_handler_config( + start_time="2008-01-01", + end_time="2020-08-01", + fit_start_time="2008-01-01", + fit_end_time="2014-12-31", + instruments=CSI300_MARKET, +): return { - "start_time": "2008-01-01", - "end_time": "2020-08-01", - "fit_start_time": "2008-01-01", - "fit_end_time": "2014-12-31", - "instruments": market, + "start_time": start_time, + "end_time": end_time, + "fit_start_time": fit_start_time, + "fit_end_time": fit_end_time, + "instruments": instruments, } -def get_dataset_config(market=CSI300_MARKET, dataset_class=DATASET_ALPHA158_CLASS): +def get_dataset_config( + dataset_class=DATASET_ALPHA158_CLASS, + train=("2008-01-01", "2014-12-31"), + valid=("2015-01-01", "2016-12-31"), + test=("2017-01-01", "2020-08-01"), + handler_kwargs={"instruments": CSI300_MARKET}, +): return { "class": "DatasetH", "module_path": "qlib.data.dataset", @@ -61,48 +73,88 @@ def get_dataset_config(market=CSI300_MARKET, dataset_class=DATASET_ALPHA158_CLAS "handler": { "class": dataset_class, "module_path": "qlib.contrib.data.handler", - "kwargs": get_data_handler_config(market), + "kwargs": get_data_handler_config(**handler_kwargs), }, "segments": { - "train": ("2008-01-01", "2014-12-31"), - "valid": ("2015-01-01", "2016-12-31"), - "test": ("2017-01-01", "2020-08-01"), + "train": train, + "valid": valid, + "test": test, }, }, } -def get_gbdt_task(market=CSI300_MARKET): +def get_gbdt_task(dataset_kwargs={}, handler_kwargs={"instruments": CSI300_MARKET}): return { "model": GBDT_MODEL, - "dataset": get_dataset_config(market), + "dataset": get_dataset_config(**dataset_kwargs, handler_kwargs=handler_kwargs), } -def get_record_lgb_config(market=CSI300_MARKET): +def get_record_lgb_config(dataset_kwargs={}, handler_kwargs={"instruments": CSI300_MARKET}): return { "model": { "class": "LGBModel", "module_path": "qlib.contrib.model.gbdt", }, - "dataset": get_dataset_config(market), + "dataset": get_dataset_config(**dataset_kwargs, handler_kwargs=handler_kwargs), "record": RECORD_CONFIG, } -def get_record_xgboost_config(market=CSI300_MARKET): +def get_record_xgboost_config(dataset_kwargs={}, handler_kwargs={"instruments": CSI300_MARKET}): return { "model": { "class": "XGBModel", "module_path": "qlib.contrib.model.xgboost", }, - "dataset": get_dataset_config(market), + "dataset": get_dataset_config(**dataset_kwargs, handler_kwargs=handler_kwargs), "record": RECORD_CONFIG, } -CSI300_DATASET_CONFIG = get_dataset_config(market=CSI300_MARKET) -CSI300_GBDT_TASK = get_gbdt_task(market=CSI300_MARKET) +CSI300_DATASET_CONFIG = get_dataset_config(handler_kwargs={"instruments": CSI300_MARKET}) +CSI300_GBDT_TASK = get_gbdt_task(handler_kwargs={"instruments": CSI300_MARKET}) -CSI100_RECORD_XGBOOST_TASK_CONFIG = get_record_xgboost_config(market=CSI100_MARKET) -CSI100_RECORD_LGB_TASK_CONFIG = get_record_lgb_config(market=CSI100_MARKET) +CSI100_RECORD_XGBOOST_TASK_CONFIG = get_record_xgboost_config(handler_kwargs={"instruments": CSI100_MARKET}) +CSI100_RECORD_LGB_TASK_CONFIG = get_record_lgb_config(handler_kwargs={"instruments": CSI100_MARKET}) + +# use for rolling_online_managment.py +ROLLING_HANDLER_CONFIG = { + "start_time": "2013-01-01", + "end_time": "2020-09-25", + "fit_start_time": "2013-01-01", + "fit_end_time": "2014-12-31", + "instruments": CSI100_MARKET, +} +ROLLING_DATASET_CONFIG = { + "train": ("2013-01-01", "2014-12-31"), + "valid": ("2015-01-01", "2015-12-31"), + "test": ("2016-01-01", "2020-07-10"), +} +CSI100_RECORD_XGBOOST_TASK_CONFIG_ROLLING = get_record_xgboost_config( + dataset_kwargs=ROLLING_DATASET_CONFIG, handler_kwargs=ROLLING_HANDLER_CONFIG +) +CSI100_RECORD_LGB_TASK_CONFIG_ROLLING = get_record_lgb_config( + dataset_kwargs=ROLLING_DATASET_CONFIG, handler_kwargs=ROLLING_HANDLER_CONFIG +) + +# use for online_management_simulate.py +ONLINE_HANDLER_CONFIG = { + "start_time": "2018-01-01", + "end_time": "2018-10-31", + "fit_start_time": "2018-01-01", + "fit_end_time": "2018-03-31", + "instruments": CSI100_MARKET, +} +ONLINE_DATASET_CONFIG = { + "train": ("2018-01-01", "2018-03-31"), + "valid": ("2018-04-01", "2018-05-31"), + "test": ("2018-06-01", "2018-09-10"), +} +CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE = get_record_xgboost_config( + dataset_kwargs=ONLINE_DATASET_CONFIG, handler_kwargs=ONLINE_HANDLER_CONFIG +) +CSI100_RECORD_LGB_TASK_CONFIG_ONLINE = get_record_lgb_config( + dataset_kwargs=ONLINE_DATASET_CONFIG, handler_kwargs=ONLINE_HANDLER_CONFIG +) From 811d2c975e4c277651e3e87220ac4c36eb63d8d4 Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Wed, 2 Jun 2021 08:56:15 +0000 Subject: [PATCH 16/45] update & fix --- .../online_srv/online_management_simulate.py | 61 +------------------ .../online_srv/rolling_online_management.py | 1 + qlib/workflow/online/manager.py | 7 ++- 3 files changed, 7 insertions(+), 62 deletions(-) diff --git a/examples/online_srv/online_management_simulate.py b/examples/online_srv/online_management_simulate.py index 8650859ff..bd7c4675d 100644 --- a/examples/online_srv/online_management_simulate.py +++ b/examples/online_srv/online_management_simulate.py @@ -16,62 +16,6 @@ from qlib.workflow.task.gen import RollingGen from qlib.workflow.task.manage import TaskManager from qlib.tests.config import CSI100_RECORD_LGB_TASK_CONFIG_ONLINE, CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE -data_handler_config = { - "start_time": "2018-01-01", - "end_time": "2018-10-31", - "fit_start_time": "2018-01-01", - "fit_end_time": "2018-03-31", - "instruments": "csi100", -} - -dataset_config = { - "class": "DatasetH", - "module_path": "qlib.data.dataset", - "kwargs": { - "handler": { - "class": "Alpha158", - "module_path": "qlib.contrib.data.handler", - "kwargs": data_handler_config, - }, - "segments": { - "train": ("2018-01-01", "2018-03-31"), - "valid": ("2018-04-01", "2018-05-31"), - "test": ("2018-06-01", "2018-09-10"), - }, - }, -} - -record_config = [ - { - "class": "SignalRecord", - "module_path": "qlib.workflow.record_temp", - }, - { - "class": "SigAnaRecord", - "module_path": "qlib.workflow.record_temp", - }, -] - -# use lgb model -task_lgb_config = { - "model": { - "class": "LGBModel", - "module_path": "qlib.contrib.model.gbdt", - }, - "dataset": dataset_config, - "record": record_config, -} - -# use xgboost model -task_xgboost_config = { - "model": { - "class": "XGBModel", - "module_path": "qlib.contrib.model.xgboost", - }, - "dataset": dataset_config, - "record": record_config, -} - class OnlineSimulationExample: def __init__( @@ -103,10 +47,7 @@ class OnlineSimulationExample: tasks (dict or list[dict]): a set of the task config waiting for rolling and training """ if tasks is None: - #tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE, CSI100_RECORD_LGB_TASK_CONFIG_ONLINE] - tasks = [task_xgboost_config, task_lgb_config] - #pprint(CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE) - #pprint(task_xgboost_config) + tasks = [CSI100_RECORD_XGBOOST_TASK_CONFIG_ONLINE, CSI100_RECORD_LGB_TASK_CONFIG_ONLINE] self.exp_name = exp_name self.task_pool = task_pool self.start_time = start_time diff --git a/examples/online_srv/rolling_online_management.py b/examples/online_srv/rolling_online_management.py index 99a91e027..6abbbfb0e 100644 --- a/examples/online_srv/rolling_online_management.py +++ b/examples/online_srv/rolling_online_management.py @@ -19,6 +19,7 @@ from qlib.workflow.online.strategy import RollingStrategy from qlib.workflow.task.gen import RollingGen from qlib.workflow.online.manager import OnlineManager from qlib.tests.config import CSI100_RECORD_XGBOOST_TASK_CONFIG_ROLLING, CSI100_RECORD_LGB_TASK_CONFIG_ROLLING +from qlib.workflow.task.manage import TaskManager class RollingOnlineExample: diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index ef6cb8dfa..dc1186038 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -163,17 +163,20 @@ class OnlineManager(Serializable): models = self.trainer.end_train(models, experiment_name=strategy.name_id) self.prepare_signals(**signal_kwargs) - def get_collector(self) -> MergeCollector: + def get_collector(self, **kwargs) -> MergeCollector: """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. This collector can be a basis as the signals preparation. + + Args: + **kwargs: the params for get_collector. Returns: MergeCollector: the collector to merge other collectors. """ collector_dict = {} for strategy in self.strategies: - collector_dict[strategy.name_id] = strategy.get_collector() + collector_dict[strategy.name_id] = strategy.get_collector(**kwargs) return MergeCollector(collector_dict, process_list=[]) def add_strategy(self, strategies: Union[OnlineStrategy, List[OnlineStrategy]]): From 8222795ac4ae51f504aa1ace432fe0ff69b3bbf1 Mon Sep 17 00:00:00 2001 From: Young Date: Wed, 2 Jun 2021 09:16:46 +0000 Subject: [PATCH 17/45] fix format with black --- qlib/workflow/online/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index dc1186038..d3cc0cbf8 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -167,7 +167,7 @@ class OnlineManager(Serializable): """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. This collector can be a basis as the signals preparation. - + Args: **kwargs: the params for get_collector. From 1320e53f8172a7abc54dc8a6753f417edc47430a Mon Sep 17 00:00:00 2001 From: lzh222333 Date: Thu, 3 Jun 2021 03:23:48 +0000 Subject: [PATCH 18/45] fix DelayTrainerRM --- qlib/model/trainer.py | 3 ++- qlib/workflow/online/manager.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index ace3031ed..28d854477 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -350,7 +350,8 @@ class TrainerRM(Trainer): **kwargs, ) - tm.wait(query=query) + if not self.is_delay(): + tm.wait(query=query) recs = [] for _id in _id_list: diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index dc1186038..d3cc0cbf8 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -167,7 +167,7 @@ class OnlineManager(Serializable): """ Get the instance of `Collector <../advanced/task_management.html#Task Collecting>`_ to collect results from every strategy. This collector can be a basis as the signals preparation. - + Args: **kwargs: the params for get_collector. From ed54f1213c717950b38ca9da7be95f64b20c9ddc Mon Sep 17 00:00:00 2001 From: Jactus Date: Mon, 7 Jun 2021 17:13:36 +0800 Subject: [PATCH 19/45] Fix exception hook bug --- qlib/workflow/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/qlib/workflow/utils.py b/qlib/workflow/utils.py index 596ff0927..5a93eacca 100644 --- a/qlib/workflow/utils.py +++ b/qlib/workflow/utils.py @@ -17,7 +17,6 @@ def experiment_exit_handler(): Thus, if any exception or user interuption occurs beforehead, we should handle them first. Once `R` is ended, another call of `R.end_exp` will not take effect. """ - signal.signal(signal.SIGINT, experiment_kill_signal_handler) # handle user keyboard interupt sys.excepthook = experiment_exception_hook # handle uncaught exception atexit.register(R.end_exp, recorder_status=Recorder.STATUS_FI) # will not take effect if experiment ends @@ -39,10 +38,3 @@ def experiment_exception_hook(type, value, tb): print(f"{type.__name__}: {value}") R.end_exp(recorder_status=Recorder.STATUS_FA) - - -def experiment_kill_signal_handler(signum, frame): - """ - End an experiment when user kill the program through keyboard (CTRL+C, etc.). - """ - R.end_exp(recorder_status=Recorder.STATUS_FA) From 7d9544fb91fc350f29b2dba4bec5d6424f75e8ef Mon Sep 17 00:00:00 2001 From: al Date: Tue, 8 Jun 2021 09:35:36 +0800 Subject: [PATCH 20/45] Remove non-existing parameter from doc Remove non-existing TradeExchange parameter from generate_target_weight_position doc --- qlib/contrib/strategy/strategy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qlib/contrib/strategy/strategy.py b/qlib/contrib/strategy/strategy.py index 4f8eb0ab1..74290b7d6 100644 --- a/qlib/contrib/strategy/strategy.py +++ b/qlib/contrib/strategy/strategy.py @@ -148,7 +148,6 @@ class WeightStrategyBase(BaseStrategy, AdjustTimer): pred score for this trade date, index is stock_id, contain 'score' column. current : Position() current position. - trade_exchange : Exchange() trade_date : pd.Timestamp trade date. """ From 492a62a569e2a64dfb9e79e61e61cffcce8ae61f Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 17:32:24 +0800 Subject: [PATCH 21/45] tcts demo page --- examples/benchmarks/TCTS/TCTS.md | 50 ++++++++++++++++++ examples/benchmarks/TCTS/task_description.png | Bin 0 -> 25240 bytes examples/benchmarks/TCTS/workflow.png | Bin 0 -> 29877 bytes 3 files changed, 50 insertions(+) create mode 100644 examples/benchmarks/TCTS/TCTS.md create mode 100644 examples/benchmarks/TCTS/task_description.png create mode 100644 examples/benchmarks/TCTS/workflow.png diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md new file mode 100644 index 000000000..e8cbc8005 --- /dev/null +++ b/examples/benchmarks/TCTS/TCTS.md @@ -0,0 +1,50 @@ +# Temporally Correlated Task Scheduling for Sequence Learning +We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://www.overleaf.com/project/5eb8efb42dcf710001d781d6). + +### Background +Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. + +![Temporally Correlated Tasks.](task_description.png) + + +### Method +Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. +![The optimization workflow of one episode.](workflow.png) + + +At step $$s$$, with training data $$(x_s,y_s)$$, the scheduler $$\varphi(\cdots;\omega)$$ chooses a suitable task $$T_{i_s}$$ (green solid lines) to update the model $$f(\cdots;\theta)$$ (blue solid lines). After $$S$$ steps, we evaluate the model $$f$$ on the validation set $$\Ddev$$ and update the scheduler $$\varphi$$ (green dashed lines). + +### DataSet +* We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. +* We split the data into training (01/01/2008-12/31/2013), validation (01/01/2014-12/31/2015), and test sets (01/01/2016-08/01/2020) based on the transaction time. + +### Experiments +#### Task Description +* The main tasks $$T_k$$ ($$task_k$$ in Figure1) refers to forecasting return of stock $$i$$ as following, + +```math +$$ +r_{i}^k = \frac{\price_i^{t+k}}{\price_i^{t+k-1}} - 1 +$$ +``` +* Temporally correlated task sets $$\domT_k = {T_1, T_2, ... , T_k}$$, in this paper, $$\domT_3$$, $$\domT_5$$ and $$\domT_10$$ are used. +#### Baselines +* GRU/MLP/LightGBM (LGB)/Graph Attention Networks (GAT) +* Multi-task learning (MTL): In multi-task learning, multiple tasks are jointly trained and mutually boosted. Each task is treated equally, while in our setting, we focus on the main task. +* Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task T_1, then T_2, and gradually move to the last one. +#### Result +| Methods | $$T_1$$ | $$T_2$$ | $$T_3$$ | +| :----: | :----: | :----: | :----: | +| GRU | 0.049 / 1.903 | 0.018 / 1.972 | 0.014 / 1.989 | +| MLP | 0.023 / 1.961 | 0.022 / 1.962 | 0.015 / 1.978 | +| LGB | 0.038 / 1.883 | 0.023 / 1.952 | 0.007 / 1.987 | +| GAT | 0.052 / 1.898 | 0.024 / 1.954 | 0.015 / 1.973 | +| MTL($$\domT_3$$) | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | +| CL($$\domT_3$$) | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | +| Ours($$\domT_3$$) | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | +| MTL($$\domT_5$$) | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | +| CL($$\domT_5$$) | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | +| Ours($$\domT_5$$) | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | +| MTL($$\domT_{10}$$) | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | +| CL($$\domT_{10}$$) | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | +| Ours($$\domT_{10}$$) | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file diff --git a/examples/benchmarks/TCTS/task_description.png b/examples/benchmarks/TCTS/task_description.png new file mode 100644 index 0000000000000000000000000000000000000000..7a9005bf24869c6c88bec34513a67704743492ee GIT binary patch literal 25240 zcmdpd*I!f57cHI8n+ix5Lk(RafOLc}M7q*@N4oS9x*$po5J0+!fItYnccch}CN)&) zz4vzG??1Q?_vJq1BzfXKEME9+&t#>GaC++Lz0;c#_fP<^MNrEB-yN3! zp1hg5kEHB0aL<5ka924m`C9BfFS=3l=?G2jR0Q8LJ@)~GCf2NzGE#G@zqIh#>pi;-;mjy#ILrTJKT=}l1 zsGTp9VHISHWzQqrl&s-UFTYS&Kyneo--@kt` zHSNVmN)(Yh@p28~?&ZqeaDH%CT(f_4?%TaDtS@(ajZ%G0jbu>gSoxijSkKldM*=ti0?^50IG` zCTSe+d8ntlxl|kGowS8c)rSL557QwSXUxPQj(yyaaV6a*Cc32SHb7ZP_POC9Z#7F5 z!Si|UG2H_sTC!2JgX%pm#6spRY-uTy!L%S^U#g;;dWxF)<_bCj#{~1b z>vX&uZZB!je%P@6oO;-UtL^3Q247OJ{;FQ)X$Cs)+P z&kS^{D7^oqLa58yWP_4K}8N7B?*1Dj{B7WI3AOq`+5R09L0J zm&u@zbw0>u$$21&tFguY+i%b}f!0^>+>CwH_d1MS(K5Dw#! zyZcgZI^WludWm3N<~ivA&Mv3wmcywA0}>8P%1q}!t?4e_!6!;6JEN1HU6iOTPC9#| z^-+zkDgtPIe`F@cJcIBaE2clJSH`-_KS$fH)(zBSk35HA8>TH1{_t^LxW29$_Hk@q z$hoVjzpP{aopj50kwAE>bVawqf6AtZvx{n>zuqm`Anu@@HEko_7d%@9GID=XQ+>6A ztTH5%F9%n-Bix9ew>QpuwtxT6h$+!*<>5Cq1?!%T0P4;9wty<#Y7Z=rb-)K<$)qeZ zCVyD0`(MJAotbp?cgIS+cSNNw=F_Mgonv<((`r0jZ)Mnl&a;HON?17$5ymq+FN+yb>bYn2)CNCuc&J{y^x>m= z)7KeOqfkj1w5$rHQKshUuv?7B!@Oz~ zb3J5CkE4KDEe*f>;>Nh0i$0k!T&HtTuw=Q@d=V<$o6BSqNn?p5XN%C0|M9wKEkpHB zce7Q$lcyNI{yxXs44a1KpUc;aO^WTS7fTBTiX4tiL`av<%DVMc#}J5gQ+GbD(%shz zPv_zt6=3;xEYdJkbF-Yx24VeU62EBnfFIaXB`Jb$^?9-BP%475(<@Tz3EgFE)hISG z#~jnflWRmF(GJEH!C|;!k9`N*c>~xPGkmxb8}374$)7Q^U~T@<`Et}Qw1Wwv1f_E7 z|Le;PH!P)8$MsUU5ICxm*zc*vBWT>{s9P2UIR-qLgrqq65B>*OmZiotJXK)k1I7$Q zeU6>zD9YFhJ3unMG|Yf^cB)r9SNr_#{7XpCx5FO(PMjm}`iT_OJddDVk$C)<#H=S*n(qo%|z7@%&fq)MvV+ zRkCtge1vrHqkz~j?T$Z}9uh&p!%UtOe14zAI-dpk9Da$rH(8|uW-z~f8dv!?$M9Kr z0ai9Ze?mhVX+20BNvWQIr*FYB7|AEz*A1Y&Hc+yVnsuB9SG`DDOFy%(Q#+G$Xq%_9 zPn>*B@2~>?zuqzR@YUq;!B zF6i0D6Xl0U+}wc5BDM&TXV|*`_VZ4nBrM*4Pr4kT67p}4re&+mh&#zT!jxv`ei=XZ-9cy7 z)&|Vps&S|^{pCWyF9Q}(YWfj+bp$)$x!Rs6L3Xp3-_7^vs50vxmjNF(V=2AauqS7Z z8G&uu$HXOi7;{j(@mvi)Dq*Ea8X_d>$=<^pP}l%-9l@f}a_++r&vR7|Gvm7w_y<2Q z=G)x+e7W`PcTBOy!FVBC1d<&h_xLIu)&yVWK>qIzsdU_u9A3Y0STgE=O&CArs;W&0 zz~_b-{s)PceQ&k<=i~3m{&&Wf8BjUN$S;}wd(H>@?sdNm;~hh1!_qHE_Ri)a0|wL_ z#QpD1l-OroDLT!TUjhE?j3hEl$5p^K;|~~6WaWMXTNKF4VOfsF3~!-lonYFw`b)PQ zA=9SY8Jabf^SseM(4_w%IdEiqVy%5xe!gK9RC&Fqe>@BVs1CZafly=9q2Ez0s zB|jPW2a3X2P?Xtus@G9~sfIwu1$opFrT+b3M!*irCEueEO01Va^?hv|ANf+9(IRk~WH+FG=F1ayn7j9?!&`u5|%Pc>Wnuh{mZViH(Tm9?Twi&M1uS;etfkuPE`lW-NmHyPn>Yr=BxFZBgKV-ouc&|EPgz&*4xnml(;CGe*dkE*n@7Ak) zs^A+9qZWyux0sB-PE{#VDA_m9$TG<~|GJe?tqZxZ5l`3;=5IZz;c=zLx_A>mK(k z)XRc>3?6mFzX9~3g`d?QH?+%%DnyC_~Vi7Y--i`*K9RDT5Re6sXAz*y*d01h?~ zCOoYyc`*F2Gf~1`UD4kq`Ol~10VIJRh&E2)C3<4YCh);tjpS;PFq=*p{))FEc1_ax z5x7ZS4tScCr0~F}K77BQ>7y!LLji<#273jn{Lh4W5hjt=L{nS-__^gPvDgN*2gBGq zg{i*5rHM4MfV6dQZGa2fw3B-9PCt5W>Wgpc`krca;hP<59CkVBEc)`{%AC^Tn8m>P z10(s3MccD&CA|0WM+{IK?U8{mIL#LVR}-+RB0W+}I1EJ&K5r5tkY*3Vo=FlXIqU?6 zv5VZMi}7gL0VQp=T<57jm&QVko~$52!^VGPb1QbYXD5F0Eq}$AHEZ$rV&9Gc(NI6c z1H$m>!-i3(d~BOFp1{7bb>_df^@0e6Int|w6_h)FyDrF@PQ-5-&Tby1zd zA6eEt)^1Z<8%z+!8#KiFzO}V`h^{gjGxeQ@VCDhsZ4LO+_dtZ8iTlB)0_o7src%@S z%$t;=jH5dZzC^7wE%0DyKXKrG&BJ41*OeL7?1RlkkB>1?vm%fPX3YjX!aa$N0tra4 zuuwcG9T#Vy;(ZMs_4bVEcWWYunXpJacSY89emTfMUNetuv3YeFh+{;-U1RKLI+DHk z))EJakm3UlAW>d6pT}!{^>{hm<~EN}3n zCQ7vOMNzF<*%Y;_;Ws6f#JV*L1OCv#i`3E*uf~PGRqo$DBEPhkIAy;Z!HbnQd9o&- zxV+MjBAuv90c09qFn)&yMML#}B)>~BseT345FM%UfzwE3E12nj^E~;~%JtDs@N#(D!Lp)>F1KK9?rfC>zNeCX&w?dZ5I| znm((UD65cP%Z|ehEnm^e_0sWmbhx0PunXbVt|us@6BR(>0|@-XHcf0gH>vr4!x7?{ok% z1mEUet_u#bb-onDZ&>p3DB;3j@`0!DXt2pk&Ak`a>39a23520R=KbZV00PiQ_x1PcuNb~0C2?0caZzp$ zekzT_g48;aHiJHjpZgj$H_{&%M7o2pr?knw|hX|1pI66mF-8R zC3A80#jMomEc1nz2L3t}Yom|$s~L9=HW5JSc=`&^hJl4}0en6C?YBu)vOK%@a!R{Y zK&gh@q-zMJ7(poqS*6v=5MWjxX;VDHR8d(Dr?C4oNaU@?{D%uA=Y_#N5OTsEIrmPu z4(KuJ3`jfzjg53cFqxOS9oemZ_G{P9%B+$LH}P@=XlYHypf?Zt>aLrp{&s3V1_BSz>? z5_Ra|*s&AFLSC#w3(;YEfuZV_4H5b{lvBq8{L(fGBum41IT*BY@bY}??oCOm{FV37)>bGcFybi|+IFl65rtI3pY|OM#<~zZ1rbrlA;rG=ehV81_8Ax{UCs zzfnJYJ`N@vj%`}Cw&}UfJCWb%l8}<$LHj~y?Nns;y3Wt*RAl&QuLJK(hwV&Qw*!RR zt`*2qi6RH}Fwf_>%Bzj-2{N%;1E^SZD9RC91tUh2UQT}I{$MMO5vi)|cw|f1qDRYgh z!pEJ!9PQ41I^{aPnaHEdC}5k(hXB`15$Wdcn!pK6Yjfet79flEF+K3X_}CFx$4#`X z`i$h*6=+t0!Tc8O50x0kah-JNUGJMipz1whwH(rewF87?8y;_!=ta5r!t5{%pMJ9! zQ5QcArFy&`w|K0Oe0fm8>V1ixa-Zd$LO(;PsY!5l;i^sCz0_ko9AW7;1IrIcNQhS* z?18`H?jT?#(|v;)>U}U#QnjoF8nUWzwqk$_^-L;n=)t(?M? z=D*wTz>lSiB1U~rFbT|;x0O~Idl5F-n!<$oy|2yB)PMp=f3HA(h{=-QYwrt5h`Lq7 znmPvfAN!Z+ZG>Yo$<+K{jTBeDnV0R<$x!?Ls1&#QaxzVeBhRIrYn2bJ`U?K>^F5rw7sPSRcnj*>2mZ z>3hdz*R@19nh!03PGcfvrdcc?pZ?>AH~m)4R4$$Ks{Ck0h~`}f@WJpnnK~#oQ;EX> z4Grx7Qfitd-9!IScB`lovh#Obz=&mkNCjViENm=M;O((8uubzDvubPyOY)1S_Zbjo z6HEr!rgz*5)Bs(L!WxRS%Q`iZi&$Vm)fye>x@5q}QzmXe)t|G%1+M!7MO6a{xX$Pm z=SOc3#Bhs$4byMiVjP06Yj0jxK=xeG+4ulzXGsab)O&`5l85Q)7Z*;z>j?V;FL0gL z)H7di(2d(%q=oMQHFV__)TK#{e6pd*Wn(RPN*v{DTsS4oGOE5%7u@>{<)~z2PlMhQ zX+MiTk*{mGH`z#QIq*RD&kZS!x$m9mV%*5N1iF4Q%4z!*+6t-n-mqmr^VgKu9;V;L zqa~wyD!w1d)F|F~OYT`N?Ip!c2dShs4qZQObY4Gdj_??bY`K%}NT}UBuVU%-k(%v* zT`E9a^uDn;jYH?AYva}0;5}C3==WmI6>4?613}94eZT?8{rxySW14cjrD zH=fB$mO4s87reiF`7X3Bn!WEe@jHOI3M4w^;>uun{j^L^wRF+jPmbS)_7<<=8#k^` zTP|vxcLv!oS!aBTJxi#rS?YDe$%~oB!?`>~PyR~-dCnl8=$gB!v!0ipgTe4VKoRW0qS^tBhsuaO*Bjy6Yny(H_U(46P*_3>QW3@iKD1bc*CQA5ICN#|XY z>vgdg^-@n?>UfF)E^iAVq&>Z<)3qYuF>r$f&r+PX9i^Sa1*9Pa-c)>Ol>H;6}}f=j?wXVSTK*DTl80=saICAkQ+x5uMffr zIM(rFtkMA!Fae%X{;VC};) zwf_$dJ`MfGt6G@&p~YO=kBF1=h8_1gZ(`Efn-p+sP^Ga8?C1d4vMOh{kk@_DTDfWm z*+ejq_nZ_km}ws$!d$toPN*g3eF7!K#FB5VqYWx{Msqkor|a&(7_QB1=^wZerkmx? zap9&Ct?qkYAKQ6QTB%K#Kz4UhRT=e+{O^Q-JDChI>?j!}@1Apqs*R90tl?HnShX}aS1-N}098?g#{;rBvui)GzjV}u(k#x);3YRcRl z7Z)B%scGgoQ<254hJQ1eHUHia4=PW;Jy&jW(1`ES3Rn+dN2RPdXH^JDRF&oCkRcb9 z;b5P}<(dlDxd#3fovbp+Uo+BHULX>oQJ)~E5(b~sl8W&aSxGt)r__oYT_Akdn)>SL zfrEhDSD;j{8^4QXy4F|t%#t!z>jILh$@LkJ{F6t% z{f@1EZ^wu4F;9CNxZ5iqv4cnWW!`oA3Azvna`uAhu+BS#`l?kbgr8-y;r80AUT6n- zG0c|RImNIQ-dq6pJdY4S9PYZ`UH3!w)oXO-4!y$y?m<=|1;Nq{5uxKUBXKd6tc|H3 zhVJPk3BG!mMcr|$uk|qwyfmtONAg_O>Xxp7MPQV}D#R;?8eM}*dweI`4^E>uiYvEU zWlRr&sXP5_TXvF0bc+ii8S`o3#%5Fg>%X^>q*}e^+Z7#D zs1dW>xZC~7+~DDTylCqwMu>zrEOav9?%#XPH~mR-OY4bu@~IPmPQY;x&VWsPoY>Y1 zft4a53wPOKPI%qf%1ACh&23%IKk@LxR< zYX88vGTDm$L5#xJ=utJRJJ0jLX_=(oQ>$aj=X{poX57nyPK~|zdY{o8Xb~;>Zsxn` z&=Ti#(vgsqtFa}HJ@wl3>?k|ev{fUAFU{_@ne;Xb#5mr4AZLUMvbfvQzqILdF2|Sa z=$|uDGX85?SX#}uC%2z&bOTZ1H5Q;7y%}wOL%BPCq<&%x2pRd{P7NX-V>`2$&d<*R^j=_F{N75=opdS;9EjIiHefV|H zdosXG2(Mj(9nN(^E4;2r@D|73nW1?lm) zXp8=IvMr4T^~uiEWDgPc-p!xkZ2y3GPrF6R#F#gZ(9o3Fx68pApw3q5p?gz7HruX0 zLm&m8cZez^Q#-E3*YANzKN=j~G(!)3n$B@%ifWAmgk%=>tBVlmBNV(n7BiqRT2S^R z(eox7#f?fLG`O|&4;>yLuBCkO20CMUKqPz(UZG|)+QH1bC5dqrX#5iqp;W-xL`l9% zL<~BUKEbcIe-K zE@}m}p1f`A-=4T1Ve?Lpm2rmSfP!NQ?;{VMCK_0GO(*)59)!h+OEPe%^I+xWl1i_M zg{LhmlF9abj-lH{DM&zj-k7`{64DIh9uflETZ;@HTZ?j0uc#%G<*K=nqZ}8#DJcz6 zZY^kJ8Mfn-SEiD^oM4Mj>))&#oYm>zZ^bo4gnASy`+e*5)u|Zinh_>4_W1|A^g@ys zfLN1+bYvNpMZ~hlE4g9!CE2ss+GXpSVTnP+pG$WFVMau()y8wSqNTT6{nPxyX0soq z<03!w_)%VqoEJL!IMIt>Gi^LCs)~5-ySJi$ZDY*n=pAM|bd3*BW>SXB63BX&xZmOM zL=Jn&KY=ME;%`}9<*>crZ}k37l@{+6nI%Lgl5xYzy1Gdlu=j9*cM_A=KgSnZYI)$J;mN>9_}cMed~vSo=(WYnZpT ze)H7&W!@g+0L*!ZPbesc%Idy;gz1_bJKRys;+-yAeTI;dGrRrvAX$uZQk4tt>4h3d zsp-4xbfOR)`tgjdXt)0RrdGTr^ADEk2P5uH=7Q3Xaq^A;_QnHZqqhNVAhptCU|M057P%o7S@@bjySu zkSGkf6IbHB!l6-W&idpOx>L*es`d*g&AUOlUv#57j{Gm4#OWKs!O+-VwlI#~4?$>q zT+uoUyZ1N7_?2*(KhRTm>#f1uJ%#n97gYN8MiR(JSVjvpf7}pJ=508#=~Nwfg_dxc zojD0>c~Hq$-&@OQ*)QSqlT%r-#$y(8f-S=NcI==+_n#-yEt7g4OGE?qo?fWa-7rTx zimkK)Q+oP=Z4OEZ6iLGPbE)@i(4YXC5WCa=_fvF6?Ph!Q!f!!?R)*eB{X{Jj(uHyY z7yM@>OIjf|!fsD$Tf>cL>{=a&ax&`saExldk4M=Kz(*c*znm_P(XBL9K+gTT&(E)l zCQ8mmq%^RQPEZ?gF%~2wAKQimE>Ui$?%@s5Y^O_C8A>ms+PQu2Z7ogrcF|!ls0Gd@ zJbhQw((&rYZ-y$DGlR+QAyF|KG4+gWGS$U{Fu8BiijX3I!%mW=G3-TWG3rx;Bhr1OmwyRixKTE^A+A?isZlC zYqH$uC0;(Gqq09WX}Ym=A}+En16vJvpewo9dm47d<`X`e(>|ISc<=lz{Fx8eFhI$; z>fZZqUe8KxSQL+@r)pMuqgtSqSM!m+qtR5CH1l+1Ug82{(=4TS*HYBc= zym20@q~M_T9BdF+_VqMC2c(^`K@w=;NyNxqxjKVzq@@F{!Mm zvxaW5jmVHseW2@gqBeiRbtBGxIJ?IGb+VyA^!o)|G%rYgU7lawgej=z{0SKtWm`4s&;gPTh#xt7=2PU(${ z^_ZM1e!0tTnls^DwksISZ5>dNEWWy$^<`-pOoLF^0v&9+X4);vT4$C7taWY1dQ_7- zPguU;U?jX*vrE>e8yC4IaDh(~uJPfN@|O!TzloI8DN@ouYT8{kz^k+S{R)fx*V2UV z=QNvwYh6c&0KvE2!XaF`*8Am6TXcfIe_8xi6-=|0W&9*JTbH{1ALRMieEzp{rpT3m z&t_f{`QOz$g{_A10vEL{39X`7bG6v5h?X_!?5DVn4^DSEK#$%0i8ZCX^T=8~s65^8 z5rvL3cu&gRP7l;ZveDNjwsK1w>a2G&Xpf)!(-o#|aY@;EzJKHw8D)E|S?F122AhYCkT@B=Aia;Ma&Gmb zTz-oOoL||LBjP~pb1`+E3%Liw&h^b6qQlnulaVU`vL{NE;b7N$YN?2JoCWSSbFj}I zH%~WXBkTv*cX{D`$z&Czh}EaQJ*D7$ufhs=?2Cj1YF`i<$;T_PtOw`pG$n?Meu$&pMJYsV$cfM@qjg4Fbb4C9*>5*ZpQ2Zov zj4Ro**MbnRZ)!F8N4>_w_7UBSP|B{q9!v)=3Es6!OL#gl+xlE6P4{k;a_E{b;VdAk zXgbh=$eM;!1pfmcdh!%MB=eK^mZTXgT@Btyomja05)wpIoyfG+|Ij{tb>XXlc~Gp?|;Rwp<2i|NVfcs{%ub2JFE6;P6c$mH|n=2+;`4l+=qs* z!(e(+oUa4xaxT}&8UZGhm;}>Kji+smV=or)WDX!>17o%|i{qK9f3F znZ?wvP(lkRs=q|}ikgU2?Yn@zKA^@$P2LIP%y89~`2JqiL$19az$ zlEMPfp0~0xdXtXp&Sy1qGsuWhAGWD&+tA3ZqP> zR0T2%NiU|$PY1-)72`I8d$U>OEWfQ7HuZS1yiu}l-8cEbt zI|GoMIM-wtlbI2`ejnDs5*u$VXnHofY2;JD;-PR!A|Vs^g(^p2ez2B&#&{1%(8tco z&l*LbpUKb8Tf|7XN6lo;o#staLkJ-j#81u)U^kM#qSvwX%L~4@>lRKnIuqBZ>Nsl9 zC*yp!GlE^ln#}&VUVKB{faDpbt}lB>D*|krYfXyb-7`g=G6h(*;XI{}_c{|m4VS72 zl|l`EQi9?;C^k4*Pa8_LSRj)2v=FS!kGm7Ns_Wi$PCpJausLFG?U9qC*l=&buE&Af zEJ}W)h^LK~@SabSdXQ&U9NmiY<`d5Ezagy|Ppt?AW_978MbZ-Sk*}O2gL`|m>;nkY zSZ<7ara=ZHPNLgVEBy@RRHfZ;QVAGL`S({Cw& zOO(kwymI)3>aKhT7 z{;Ysn7VMp(rZ9(&ypZ9I*}eu6EWDTn_EU$T-S(K;$<|$%(84vpZw_av2$Ok(`oQN| z#q+@((|aNEnnX~mhpjryU<x$&&+fY7afihp$(1shkCrKzo5es^tg5YgB6^pRQl}xUM1RSN z5KrlOz26thD4mlbH~BI+(-vQnFruh1-f=U64R7bjl#yts?*8bewS~1qxxs>ZB3C+i zs-Hi|PqcZuV`H(1wbNO}cSHS;vmfuOM3Ao4J8?T5d_lgWa09s>PIuL(H=w62wT%i=uWg3-@V zasImz`7>E7J_RsK{f=fEpf)v|f~M0?8LCA)#ao23KKh%7QfF(FFbI7dVC{D}(xg00 zvj6?vMX^#+_VuE$_u^e}alh;&yMp=Je|}=U?WmQE zcvRmA(Kay!&|>>cHjMtV;(Sz5a`Cpkj4n$(>I<&SC`Fs-mDzPc$d|~8yn>9_Os-xu5*6K8*U|RzfYhH4n)g9#;)& zTdmyI0vHK(CA_}A{wFM8UvfW>|{y)MI#ng9nR#<#JP|9!L1AjnTH=zhsd6 z{nAI6k&~UNd|1sQGzMs*YiB|6HBH7J)`8Gc%sA%;C2V1$$GkePREL!Q> zRTGQ^^{%zXdI}4o$#EFiOPyZsrh}*TfYF*t^irdkSq^Iby&C&Y@67wph>_BDs|OI% z4`)23f;wPs@GONug~ouVfR9>sopq$U-pR^|=|*rXtl*{hO~5@Ym9L`Z^qM4fP}8*P z_y9BU%aL|GTL+ZIw{Dem%`Dm<{+0PJ2*IT*tW#N+C>N?%LBDY)EyjQ%0Wk0V=a+TV za{DP8##0U#;)|R&y{@JdZ|IW00+679C>l&X4zt!XiH8m9 zt$8zns~Xr8j{{eFb=7b|z+DaEUJbbY!gH*TPgS}=as4$k1-h+R)fz8Ae^pCO50ZOK zcO8|Q_Kw<%niC)YNp-Q<48EMT^7(cb-g5yACHscm_fg=}PsDc4l1~Z(7Vnz{_?jNI zixRsypL_Iq&5>ZnNRS zSSRJi9M|xoqPpy23;4%=d#b0#&(%cNFvr16q7JbzA!MzU84eYQoua>~xwA{PAJAw# z|GMW8^H57KrE8Z(e`cxERw=W~Fgy8Fa^?K)-W}e$dvcG_vO1OTKlBfq(K!3UHiKJH zs$;InMMoa@`Q@W+*9$kEPPKJ6xp!?1iotgZDrU67r7WF97P7>M$pWC)&jJN>pgPg7 z+AOTyHdW6NC(`dD+@EtxnL@Ae59U42wgEX(KNLqqC46z;xvvtYg78J6zODaf&Sq{P z=G(y<42i~4q;ycbRkD16U4YzIXNUr%P~J_pq&~p z@B~Dr3w71~NoITz0LPRl<{+Zw_Wj$cx;LzHhs>wXyHw}W&p!fZ*Y<(2JJ22K4X(o= z>*!~ecWuSNbgkw5S6^T(+==*)1MNNhuRVaIy8xy;1U0w2^AD-u;_u_N{`GKnB3egh%~c7dK^DxD?TFeG~jZeSbrfjD#i)oY+_BBJBG z2M89wI}rn-lo$P5h5*<18gP5gMo#wKQHy3m*#TbbZs^5U>`9)_xy*J9QHMr~y~WH&V6Wmn+l9Nl=C) zRAFze6?)Py(7k^V9;h3AWNDN%9*~T5rf50O-aQ3m>`AKP%(b@%(cA4GGFXJ45oTG8 zSMz1jbD|-el6TgD;yxu*ri+D&O%i%HS_c?o%Qnok#^%Z*S{t?2JAk!QKC_Q!M~_Rf zmRKOuu|8bOk}+E6QNh&JM4toO&{|E*I7Y?$sv*BbiRg?(W8)P@dUjv7j7wi{wF1im zoFeZIs~kRrJixyei6;@%*Z&V+W5w7PYK5-9@ogw!|70+=rxH zy`w`jj|m(Dhj$lS#Z^@0(@FgaZeHw5hStB|56{Ng=KG6HjKlY z#0jkgN^PuqCQXWR)lq>4F2vgZWV8$_xBswv{E{+)nIqG*FG%TNM82f zYY&60I!S0#5bdkj+W3}$x6$si62)zA4yOfc&na8KN0pfh(ZhOSDesYgZ9&Y(bjSi~i! zncmFR5lfJm^kaw2FllRZcMzQnSfH0CBF-%)W|_{sW8UPS_L222lgWk!B1pDT3kjJJ zTyGurrN6W$ttC4YZx?2;x$Rw3k$7GdR?560PAgoeUSCpe=L$)i`TdP3!M_3Py0Zb` z54Hwe4n->Gzow;w_Wpg_9_`Bq-j`LvS_Iy4WhU>i-btPMh1%+Vg-K_ank# z=*ab;!^NH2ZN%FdlE!|Ll~ho1P+`Id&b;OTfv6#pIQ4 z>Oz7%#Vx$W#Z-X)(*CyBR;}1JrWW`9m2zL)R*6tM11`w2xlO5G+8)vo-tI(NWSu{S zuOAy7u`-|^FL5FbToB81%yduSO&!g1`$Tb^AWq;EP~r6TuQ$r_{UDAnzAeYbWFP) zo^05vp~CaCsXPt736rckf$z1p7(}Sad(3dVh2ev^?5`AvOxq$(1VvsnCGwjPlTKNf zYriFTVyOM-k#`!)`r9dt?JFS@?qC0Q`V0pHbqiKL+TWAaeYg5w6Ms*N3M*vt;>vUa zo-Q?++iuQ+9%2FJX9c+%;8VW6@8up$I5u(#?VVLA)&!tvue_q#l~VDv3X|qZeqn{U zsyJy15vMUhI?wpl5nl37R+ce?m)GD*CDgBq%?E(V)c#@mihkrOh`z1oCJdskfJHd4 z5~dp06u%biuUNO?o!^@uFrFjzgW&Zc?34*?I-f1-?b@MaNh$NRThU%c}Fd zZ*hXdfr6eS-J+xRJ}6&z1@K%^&Wh*tZBzb@b5qMKH-F9<+eTw#`25ck9W8@bVG$Dtn3Du`S*w4 z18fJi@iq47=sY$Yyd`SCV7+eA$C-O^C^kwyCPZh8jgV%DS73}98FsG5&Oncd3vb`G z77+C;XkYk(;!qWTT(dY34I=JU%QySv{?pfa2MO=Yl65=*klo8@fmLir$3;Wy?XZjE#HNzABtz- zfY22yE_9L(m!jzp1=10v?2WfpdYc?9AOj@c1OKJ=#YH1<`;CnQen4tAhNvHey{?3_ zz+=eQI_HSK8Eyqs%@oiB!P+9PnFM63PG9ohiSx6&64>TS5?1-W9%pPP?ak%Vh^4XO z$>pOKAv0U2dP2;NvR?3F?}E@w?>T(n=I&(z4d2uuWZifTv&yz?7S=+SFLsa zQoY3s`g66t4}f_ATi@6f`;`K#J)C((0ZAA6rFruE1S=-OxFnr#%8->$`G;4vF{2VO zM)cw}mK^v&4(1LYFl-i<7atClm`9(i$D5VoW9qDeM6>3qyxK|PEc8JAM4jT) zzy92On^dkfBVhAZK^MFw`G+i~}7~1#^QMbJ^7)r(2ANt?Bt_@py0?tB-Cy$;oY7f=2 z5H<|P;au}K+dMDWbRPc|x3a6jgC6(n4c)&j$kcE=EYIHw1`)Y*vGaiR0HSwHqj1|( zM3UC|l<;Prrwq!creo=aCW3bC`udkwI=+WtX3kE59%^DEN2_x6MZ5=Lu_1nMrBl~` zC0YqW71mPhbZd|5MzN#)tF;6y2+&>vq)+ZNd4*j!?i3v5yg!-}6HL5_s|hEQ5}fpR zus1s`*{wGBGoGD$Ov82OqVjHtHtT!I%6MBU<*TcOu^|DI;rupnd3r_k*N?;M{k7M& z%q+FEm!q`57IuWUAk&4iqtbss=^;Pwj=rvBote|dP(e2X@eHHBsV)78y?O<@^6ZIQ zkjQ?sFei#rKJJhCBTHoU4C&%`rpxpKymkln`Vhl0TPKD4{|;64{KueU0MHe^!JCeS znASYIuy*K@9iBeiFhdA|{AO*?iVNvA$E6ruBk7%{?3K$84PSmvr|$4zy7TyP2ALsB zxJy&NY+t2OzZcTG<5?VpQs_R=HK@AX2dHcR7?ifj`-;edMZfni?s3v?T$~R$Z-_Y2 zqI9N*QO)A3{`9tH;S{TTY4!fc_N1GSq>I9N59y!dtT6A;g7)9-o?^_nMJ(1UbX4hb zd#`G9JeELp_>7fAwoKQfOBYOX0$s(j^WyPf0t-zagZ(+e;?+^s&4F@0hxuc@vPUgd zr;bVmCYe=|;H`0frJC}*Bbi>FkX()PJePPO_nW#wlB^FPlwnh8#;@2-7Jm*dK?1x1 z&ko_H()Uw!+(lxdAbPHZ0*|;gf4rGqq3*ITnDVC#8nD$=xA>4;cF-gWntH%?I`4f- zj4IA~uzfI%KNA-DTyWELXpfx)IujQuJ3zd<`RNLa&P+YRBSzGHIolF1bSnM>E0zB6 zrqTA6H(LMqC;ewSyUu1kKlgk(&a+<~xUwT*a!fn8VnTK49=aEHL{J3PT-1Fl`by1Z zv#fT+_U@8g--6ONgX?lDnVG!@xpuFBd+e;Qz9v%&2ec|TZx}bl{4KSIzRa`hZIm7x zs}FgrTkANWjDN%ghs0opGP7dYf1s*d6 z+sft}*(NkR4c>`4+TBpoq6_V$CXmT@dLqR$CyJ*_%9~%;PZT07E~BKUJIrb&%iUC( zEL)t&B-C#02p_9|Pv5xEAwBf8y1414=fn={Cap3erbYR=+ivU5>?lU4eo6hHh= z-wQ9>Ft^GG<~Y=QBK3K%F)@*{eHJm*w8uWISfL3Z9wu+by<&<$Go>FE)`1AOWQz=; z{QuM1x&Jfy`2RmbXbDN=oY#=^Ea&q%%^@8fYs}f?d-<%vgI0;`sIpUNX4=Ow3&DD06h zhxKD^Eq_w_$)`u8uX^jg-;4#E^R<;U}{gYZpBXNg=owuE_ii)_Du1+ay)-XU%k2I*GO%q zPhlSExDLgc(s?%tM0k^`a@z#IiXoi=;-`RoEd>+jYo+v52ndCc?{BBvh9;dz!Rihg zEgZi;>54QTOEeoZKaC{+AUuk(l=6o=lFuKndAuZbTiYiy5*rNmPWqfwD6_jCM4D2d zR?>w#zzQ^?U=2MFYkvLi_~}Xe`1S?ISSM#=(q$Px>l`FOF721#i~rocpySG$oIhEl zVdg6TAqP4;`|5Im=i8$dH}n(*6B2@604VE&k;(0wZE|DrS=^n?r9UonIpe(43?LzK z?qTi^)*@6S_+oX`&Bo;kOOj92AHDzfs-CoRy-8b%)dtZ0JUCNfMOnTSban1-QbO>%(do87|v}P^Tk$TE^$i zwsa)CB$1k=e&zPnws}Erut#_+q06x`93;RPDDIA^WHHdg>U4C%#hG5a^Ew4`YpPfI z)+%c|toBHECMbgL^NaLUj3O(QX2k8Q9UIlg#2jRu*|*asWU5mzS?ENcixQS7pSvU) z+YCMX zQ|aU{1#Qq^ji|L1#Zg<&SrO}G4CwFJ!>|a(2o%{3noS2FPyF*;?9J=m7b2b*L2;vO zTIO3kr~Zx(o{Q@c1CLI8coxc;GhMqo6CpX5^burB7QYq69)x_j)}M?o4$K@dqv#h4 znJ1j!i+o{BFUOmSe|6eu9FT}( zpXfs@2)r*xgfZ3@2ZB^qWT+txe*}MhK;Z7V?{Bk+D}i5W>bjh#=db5GBBr-fN;KXZQ4Lp}P3NvO+TTT!cD2E~s zy~CZ(*_9myoRArvGGDK0#n}&_J~2gpX+dbAw570BE^94JIhJDo=A(0%Q)6CD#id5o zo{RDo_AUQIMS6$CNeaGF#&F7={joo%io;1)-qZ*{+Zt^Zl8NROgB5>co)XY zb#ni9oY;c#wtA%|riaOyY%w#F822vY9AKZOnMhLTHqidPh_>RL1I|628fnhZlYpv{ zlULYBvIQ5GJ7FA^v1`HDWne)pUw%#gI&+lz-J8tS47aPWEPsm%1I6dvB?f2C)lDc6tSnONFE*|p~mFThUUFzKQUoy7k!6yqfO#P$z-W<~Hb&&kD& zofp93$A3J5-E3TA;$e}&8h4m_IphqlqS$m`YF6u+_g+o=Fm(%dBdZj zQ?lY)dwePk)W2=iwNSPq&)r>jvFcgeZ{u5Y`|G^W&sM3aNgK`IpZ1xobwRN3z2ESO z%8+xkJ?&d$ylKlutL$NCW-CK1f+#wkTcq~XWHY)z2!sE>OZozNv_xK{a>7oufwA}h z{99mGU(SP^RJ(E2iNk1R=BJw%%;~4Mtjop)lfv8f)eE?oQR51rN7iq06rM&730P8w zTV*xVNE^M>?7^f0wMB>j4Kvq(U3Py4Gf%g1nk4&MEKCgbQ0TEs7i)xAQ)DQMZrQ^t znu(z^yoP0G_M7l2%thaFZz@OxWbawVg}xBTIK30Z->5QxHj9Z)6}0)JX(MCLS%*rj z7+93od|9HkK2AmJw5;4<7PWUX#8DZum2SaKdUog0@9&fbo4D_tZbSYInuu+o(Zmru z^FNf|WGblY40U=z!+q2C&w1dCWH;|WhDU`(4?w16t)$&zQP*!~@HwyIB|MTDUJc4_ zFYw~^rwjjETb3ufFe^>nvRQ1<)7`$NuJ8tbp{L0Mg1+1Kdh~8_k zN_CHb4`k?h4pqIj`}#I6SU0|fK9ZU8`t}2ee@pWS-Y)qmbSNEtd*$f0PWTWKDSB?# z8JI5fbc=3f3D*>e58-)iM+7whqa$uOzOR(2g?v{L+Y<*i`ccoh75cYTjdVl^Rn*w zgvleY_foffV_Emd%yDDbJwKSd!}}fQ88;mt-(&4^=fyS?w%!%|-y0b7RRH>UxQKEt zY^M|mJ41g4R8a+EsLl`#g#p|2zpOx5@wkR(mSdzVT(EV~>F4a+n)|J*#Gr24yj@w% zqFvbqGV?K_f_J7p`w;eWPw?mf1`L6FU*Ay#egtDq&q7-ED1=n_}!yy z-kC}TeQdv7=*9)&YUf@G>HWOx=|GICl#?WJknNkh`k|K+37@7P&>3^u1B{-MsH4K! zv@K#cH3ls_b2lIKPTRRQyHy?2?%O1!nd@ciPsVoL`(q1C!5A=Dm-&h}Mv?d#Q=486 z36Y^{@WScs-nxGQsVI>PJiycKOlgK~q(2P(Q6T0kR9_OX{V@CA)1mTP>$~rO!u_5( zNSVpXZa+&I-m{Be-WL3^=vlrEUF%``m?#-kh#HR*>MYPt-1;1=`1FYPN25uU#$(jf z8S>WEAIjUT;>_pF3^|V%Au%m(H@eww^ufdu>{0o~&0C=ENht)tEIP@Sew?YU$kW*w zIi&w6Y8)D>lQ?;>cX7a@1DNRIIA+61sMjkqf&RkC)owZcygDDcDm+TNYM4WX-Bvhc zoNUj3ev9V^$e!JZ?MdnYQV5mtm<1{xI2ON5Vo@Oci%i}9JDyq3m2{$Vt=Q@3Tw#`8 zbD_bN1d*c#o0-8OJEOh1`4H~)^I^_aEp8d)y)85{8D$;NPah8pDO%tNDV;}Z3pzHZ zpvin}M+GRtv|jf+c9h6NakpZ+@1>A@RDelCUxB&l;_=HkAm8%u=C2pGe2!M*;5#oIi7 zK%>&Rv4yXI;?I;igX58&A7MfVa|7xeG5%aPX5101_d?8Wo3kKr5$C>w7p|=6RP^sS z?{2*wToIC90FP-Hk#5JHIHVp}Ik4o$Mr<_4eOs|ISZ*!}UGamEQA-w4Z&vDDnDgpu zzxyMsEU3u!v>5lZ>dXJu56U6Ji*E_Oi0j*GlzK6y&eZoDl{3sC8t3R+NAtC~^qBi~ z{{QBRu&7uDjTDQ~rzU zUd7)nuCEQr6GK1KVu|cOK#?R0l(j6YRGc8z=pe9{R___FM3<$zK3~;6>qfQ#;!IXT zx=ca?Abw62lY(n&`qOBEd!mNo0mK_w3s+vetKM1wLT0qC83bZIV8XJV1|vu(aMgtM z_ipZzAZYe|p`lgf34lu9-#K6!Yaj{C9so@*+^6wx$a}HrOw;Pw6Yq8$G+vDbd7ov=;t^ZECqf;7 z;I_s(eNxX@Y)cOh7t z{KE~%fQ5zs|0LES^~Bo^Fw0E%q-JeeI>%r{ifk9{+jci+`e@X(As6y71 zRG})}xr^4sm;n$2USazt)AI$iBcC38dt6Ai5xq##YbKQ*Q z9Z_BALo##JZ*6kL2L`?34ELsbmW#2(722wC@Rg*sI1%fear=sU0gTxu_yXD@9`t2= zB6jUa3Mds`l8zhsQOj&_JeQ@i)r{f9Xa%nykDah{v%&&SR9rB3#bepmkG&+VFI{5t z>wlDfVxbJYzO?nco1iN6lK79n8=c0nF}t~(RBvhEGAfT&hOB)UF*1YoR0jmaGT7O> zGMc75!pPjA^s~7x9v`}*7-E=H^ngH(arvJtBXm#qjl05u^9hAq;ytg1RzS4I1_v%Y zKjtOL7bMf9ZR`^;s6$xOIieBnywW!QbyVKCR9zS>gjrRE$fxBAx-Om>o~(x{D5Q1s zE($}3?kICISBITR$i_G;8&~QxFED!iFnMs)S{oO^Ao9AiUfE}-90dD~y%h21AVEem zyckv>D*&%`kMT(SlG}#k%6Oyw+U_UiLq>OIdE{cI9%}zv%D%!N+%29)%{7fHvGu57 z04gffW|0(C|KVl!PBO{Pn2}w8+Zo=$^A|jaCOSHv2uaDSfah1C((T7}d{BzNp_7t! znt-<6CBTHL;WKPmj^uh8wn3Q40B`1Z53yV+R^;_uV8sd6;#Uh|2t7W-c}zlFQeR`s zy0OJ^GpD3nPt5P84N9BrWhw3+OGiNF8;o%y8fIF)O2qoKP$vF-B}Yu`YTRA#1Z>u` zB8mkDX>Bv*&grMGuhMg<-2#|?&hcSmVB`=()@Ia8g<0%XJ&bjx{o@%yFfF`4!h=et zW7B(}LSZ6}4^e>+!kpwKN)rosI4iln?8s+5UlAI6&U>?O%ue=OJUZR%OE#wJLz+En zN;i|rHMYF&WC=~Ba5aG@$p-G4mo1BUAaXj}r?wY=E&gamZ@b(5%>Yb-*y{;IRLIWQ+<%}?q zvzMNo^?Z&aajz}ep)Z*^t^L?Ue%>u6W}M9wR54B|A-;v_pn&HQB)j0ax~Z-O?GU6; zj!Jle?0|}qJ|#ii+j(?ejm+-|efl-17ziIu@3;n&jFREL4F7DNI%Lths5P=ckc2dV z8(M_3b_OK^8|Xd@RWbO7XbVrvdu}DPalJ?`zmNa*vh*gUCQeC(AZ8Gwk>WL5NA&m7 zW}|v*1DM?YvG(qI&V2-%VM9wrP5CD0Kk|m+a2L^Tau3s(>7EZ_av3N87OoyUa%lKG zV2YmjoirpWQV80?V?&ZZsw_9s7MXS|#KkMd;n#JA0!O12oMSKeuQQ1a<8&Bi!`0mF z!G)brTqhNLm@B=NlbHpmdy5j32CoaU^;v!&!2u#o0-7+1Zf7(U8UX|u*63_#@*M$# z7tgyo`$tmx-dyHr5Jbni#S|zpuEC)>_6HWOP%t^;!fZ0tgXUZ{(koxcq?5|mB!~+8 zjl|o#w)h;zE6Oa~e>hMd`W{O(jZ$nrUmhkSQqZ~^AaCYR(Qvy$`jLm#yrfLB^C-i8 zo2+-+v%Yk!RQ_{oJsT=t4rWdfpS`iE=@$?46bUyxMibNqp!Q$yT=dlU?C@+q&Upd= zaap<2-!$pFS%g2!y79yB=>K%jjE{S9e0oK*s&};WQYOlcywI5W1j@l{{Gz6y=3HM zZT$PLK=v;01(S{h;JX-nZAmp)AkBAT$b;>gdvUNHLV6I{g#GFNb+fZ#Q9-ESi(ds_ zRxnyiF^!RGF0q2b?=i1F~-g;L0}-M}k3;)O+n9|GLQDiXzt z(ZusHGFTKbt3k;e3h`r>RhyE{QjrMrbIj{~w)U1;o~w$gjpqCG`qA-M_Aht;4L@%J zIravn%P0S0fut39NXqhC2D0C&@$gOHdG#kVi0%}*2q%P3OBtk97o2xpu-9X z&zPjxGoQ}=a!Gta5Sf{`#(X>)iz*1Z@*|&MEb?t~gsA3r3DJt`n8axt3*Y6peIF|# zU2!*!j@mS5>J;LaFQk!l1+GC-(05p7VbfN{U%f;lt>^CJGWxPMvKa=c7J!~&t48G; zuEu|U%0+VzYmaX*>UZ z{d1c~qX`B1f*lT0&DtHmi!D?Br08*ZN=Lknf_)_F1%$05k-78=V@kev!Ju9ic$C+L{q3H z71|b<3ay?vl}rg_Y(}_rhp>;#B2t2`=N5kZ`8jE6tQIG}U!)y&HStFjn^4H4mP|t4 zje%BbPznHbRT}?SjYKfrWWx(&q%NJdyxkg192sf97G66)^t0XVj(jQ&?7|{gcuif7 zJ1*eooZ|!2xgYRZ-sfB3xp}D~&@Wwxf7`PbAI zOI0RAU8xMU?WES~tW?5PwrIHLv7DX1`O2mVo5{3c@r1FeW5WIz`^WN^KUmw!i*mis zPYn>?Yu>m1s>;6qotJ!;X*R;VkR!zp{hJ5Ril-YxS1Xwjrsi~ETGbLUEJ1|sXDI*% zpj%SZxL3$k1a1xuoP$y^aXL`&9-{fWIFH@z^hgvyhSH^$e}3=#jK<3Df#qujTb-&?1;TD!A0TgY>sw%A4pP?1{0_yYlQ{cF%#=r|e-BDZiNcpt@zU*A4pYobUVAojF zeL#ofhV}&ae|Dm-a5M8+{Te6{*B!2xaM4~R(RWk)tJG_M!mGZ8sxhN&ukMl7NF@e~ zu!~;JLAZ7cJZ3T{y@{G|d=sdDue@ji|l7=-S`J@cE(<;ySRN<`}`gwPD26c*GDpyzrQxm=pE(7J+} zU3Vz<0Y`3Z;_(t=WF;-g8ci@?lBn)c%Gyrr))bUxG#U0s0j?o)v$wZ?D%OB9;Oj(< z2C+3?Be3-Xi+n<0@%bd-9BcYw6y0k1sHOK2$x*veEeS0&oVi66_&AdAlXc^iTD4*#@)(cCOu(~H`0_dG)e#`r9aC#~m!@UfbU+n3&AJN0Oq=VO){ekb{hFJ7^ciuk=*IgP7;8g+9YVA3H>8 z_><(F84@a9H~vKwE`%=u@yaISWCWg{M~HK@xk*a=K;v9 z&^lG@%*;E4NX_26oi>DQ9Y0H2ZnpqLiZvnPr!YE$!(%0CBy^(~@hO>Stt@SI{oVe| zMe2k)eeTn@%11&qA^~utIAH$L6Wx7Wn~#2E#rOm)bnKd7t8NjCAg!a+m-d!;-;v_m zLa>`!g&doBHbqenK7b(;DWIsC2}xMy1p${C8UY#D6R-UcmRl%tbU#2xw67W8+EzK% zEC*43gxhG{BbBmSvdT3$`EcaGDVR@T@o%wf{^rqA5mV~_R1rlR6z}d6A^(JX8{T_w z5qK@~8!R}c_J-E*(nK7D;`n0tW*|E(G6U}+kBIFTUz1m@8^7K^iA0x!hs$nABJ}b- zVgOuc<@0gu!d1%AB5cxUZ(!kkcN6{} zD_%vpNst#nKHES4vVUfYGff{#F+KPMU`W7c?UuPCLIQr(x=(BRhU2%v)|c}wX?$@6E>PGDkVUZtfeo< zyzIIx#oip?Bh$g|?EM_L8VZ@d!Wv_fis}t(9`M0WjlVnl=wU*j_owxDgFtzg_pQGj zSX#s7_U~5z2p212@!@vsH`%@xx@R~&_5Gqf3gG!N?(4j+N80a~Q-17qQ_SxbIF_%K z@EPgWXj^vtNH}>fqIKPo-}*|p)caresw%lVO+oFhM{StxYs)G0%gu5-Jj(6-&9gqE zin@M?X{Z0D33*oP?Ej~; hQm=PV--S~#V@Ssl9}NGJ@i&Px7N*uF7$a!H{{yKCe^dYf literal 0 HcmV?d00001 diff --git a/examples/benchmarks/TCTS/workflow.png b/examples/benchmarks/TCTS/workflow.png new file mode 100644 index 0000000000000000000000000000000000000000..403a17de3923713e395c076b611f7e154f370d6b GIT binary patch literal 29877 zcmc$_WpEt9)+}i8h?y)LF*7qu7E6{z7Be$5Gue{G%*@OT7Be$5Gwt}^Z})BN#{2z# zbWBWl&x!6cRj2Y~W`!%tOCrL3h5Pd53*t8^G375`z~VqhF8~^Jrsp|}3G@Nxs4OY+ zrE&uQ81x0wR9H^<%a@v1_&0qh(05onDGkRjUy%C#9biMY#YSJgI7xpK6IOB4IbVm; zLpS||xw2h5p{Iw6XlvtT)$0vnc6HT%k>6l`$YOySkqPBJwOP`1PklAo?dMwuyo!gs}+Y(V##tNsyMut?Kze z&-Z1V)gt;xNQ$(;KHD^;-E&Vy(NL^ux)ml+t}c6F_iph}pZ`kO(@PU$8|pAautfW# z0j{a3{S#8l-A+Ps=+Kg7Re3Ytyl{0|tTD(@1SJlIK#84$CkG`IcrOAAN~lzaA~-Mz zMVV1YfjFQdYW+q{9^k^;CI?Di;uL2od~SUMle==iHUP;j~Duj)bn(1xMe8WXE*K`SMv5Gr6EOxQ`ioraUg<| znBFVI(?^}HL1$ocJ?N65p@LM8A2m&Ji&P^(2 z_oq*GG8S*X`o&HN_X~^UCM$g-C_d^wVD+Y7f8)iAw{X6R^R)}b4 ze3?Sc=#KiM(V_ZtZdfH!etNFetNZzmC=xDho%va--M2V@rH(2+5FWJaw%JlO;^OoM zwqpf3`!?#?ZH`fnR}`f%sa2%8SsWt=Z1I7a^29L)>P>Hm;!WXkWg1baXGQvN`#KoD zY+nn?eC&tDN|=0{AtFo#kS3gb3`z)keuK}xy(+$aC?-@sg^Pm;?Aj)@bE1h~$_3OV z9$xQXzh{f_lz*e96OpoGDJt43GBk<8Tg3AS<{iA|ix{O-q}A|);yFYr z9`Z%jZbG6vIam@z59>m$jdxUR0~CyE63ZK7qk4>a()aZLWj29qCqDgUWMwGeuu&?Nr2shGm~8F zBl%FMgh&5u+~5~9c24cFmtP>+Xw-WbPNrw!bM9lpD#?;7TiE_=Ok@816CU=xO^{U8 zkucN46IL_21>v(L_Yds>y}$C`{#qBTry2YOI3^l;Cqts(HIxx>=fi2p-oq-fs|HwbSDSRH{yc|v1x1~A$~VHBl|^>&8DKK+jYwXbUJcBhISV`Z{fF=83`s& z+z69T>{gghe-2?%AMnZSy)@J8j&0v5GB)R`(N&;&M%?lJg$p}AUS`O(5Na<`3UW&# zl%=pVaig~kaXiOge?M1|c^f8#_zfelNm{nUty{TCQEIhXz;z=_9RdBl+SWBKUM*># z;lUZy(`^(4s>gkouCMSJXf~a>z16HTm1q-m;@_iC!R9+HU~oCgSF7s%YW%Ss`v&Q5 z>b>#=X*3&%zmot^WnC9@} zm}=DxMSlmc44_nVG|=KfY``*$)R7_nmbN=)v*hv~QyqFx7j{^HML6`9@fkWK>C;Wp zzxuJ{gri!9rl2DjA%2nvh*i*rG_Lw^d$~#cBs>)8FvZ?jHW+Z7trDkQ6!7<}H6V?! z!dLlZHX@1=KccMuKpxQ%OLzUK`(ziPXG|%} zaP`%KWHv;EaGH(@E6}tv0h=_57NsCIZy$}G#y$UOy5c=Xp*Kuw0)u74OEjuU67B+X+IG94q-r5AZFEU^iOK;@U2lYZQ*{Z|7 z|5<$Z+>!5}RIplb z50*4%4kP>Q)Ki~35hA$tAb#Z&&67w=zP8!F%AH6ObzSq!KVO*p29c`%e)!9SESIu$ z&IxN_B5Y{fpj2pE<$35ht*gOxA(_{W3<(bcUNH7h`e{#^^f(iZ(NrB{gh)=PJZ<)p zku!{x;?)_DP8`Q6uQJ%~%$6(akm2mRlP@&tWToiQ)3*!6x>oMq%on`vN8gaCvdfTi ztUxD7``w8&cEyIvagrbQEtJ_OkHk&{nsrj^cBYL1vuvS--r$iaXz8tY=h2gdxB;+z zJi95pLpzCJ_Qeh+!Mplpc48VYHZr%#@mt%=kR-pGTA*45n5*g1UN4W~Ql#PCn4@u%Cx`*)}QZcDE3CWE^B?>4Eg#USHp(DHZ5M5W=b*@b+LBZ45o3z8A?BccDyM}^FVo}gG9Dh{_<4smmnMkOlNN11QIdZ8(T7AxXK zVcu(Wsupdg_ji7`nzEm?;4qy}Z0VFPeocja_AfY3zi;|}&=gC&!vGB_RHcyum@x;5 zi2R0Th^!gu%h_4~q)fb+aMaGpbfkkismz%ESaWwpgXGH+A}duqD)J(Ez*4EUdMI)x+Vhym6)uiu*cjKO3C{WaEdmo!ku4l|Q~d$)6}5 zm^PaUbEEW^O2|=kX+;bPSD3F%Z<7UcuYUrQJ$k;(4T~^#RaL;6h6lP%`unUQ#+@=& zpXS8`)GoJVm04|w%krJ&5zgR+@MeaT{&=5ALqRKd+E#MAdlv!L%lvND|t-1 zeI0Txp< z5(Y4rrSgX)z?B`{40S%N+KOb=*L6*XzpRl?Sq*y^xPmJb%p2hvmRrUDNxePkbG@uW>hJ6;O zOx#;)G!hy$Gm1=3+j*UIS;p@CZEcNy-ZLcbxl+x{)pD4!^>D_dvu@u@LTs{z>OIA1 z52Bo1kLGo!vRTsj^>JF3{Of zp|ZKkcf|5ZSb`)>?OYU|z4XAk(w)&SpRv7w;DV_Sv<>#@Rbf6^eyp8Jh{nas3|iI+ zxwsCr-`wJ)_3jLFrQ7tur*)<9yzgjS-XH#pEL`@aTy^nNx-T@V!yE_Apid)T|Bc6g zxPD6hF;)6K9DIbnC|-~NAJI)Zk3%0*0Z5{B}w|Gp5 z99D;R>K?21DB^!tWRb5~gpW)#>(Qi}!)VRB;oX3hh7*6p2qKM%?4)42IqXTA5uC5{ z8|(~TgS)v{hcu;+uVlkJok9WBC&VaCLH`iABnVJEyj(8EiU zWMg;br6%Jid~60@?P-WHQlfks3DzPtu1BJH3?2H{)I6}nxaJQ3K3vgTgi6YUyjY?r z9^$9*fos2YWvtmVoS!DLchs$+{5eV(BX2AeMiJ0pF=@`(qqX98aIz5GFI;dx*!r1BKl8<8N-}b~vO+pBpT& zKWPuB$Po*36RXyOFEf2YG2CFH{&=Z}`uU&E#Y@U0EQo~!D`s!}kA%jElXXs;0pHVi z!9E{~KjQEI>2a=s8d9#q^?d2L8gwLscjGy2$E3rzAO7~&d;kXsm5f1=b!WPSam`j~ z@e-bxkk(Pa@O?wF!idbjmNP7Nxc+)7szrAVg%?blEurF7nDx#lXzhc+h`THA?r^ai z*ZPYs)FgZ8c8H{Zu9sik7v>~Uf+GFznorQ64ofOS%2DFmVw&4cZlfM~Sa^5m3vPm! zJN8*g2}%@{9icAlEOKm5{fjB!;U=C0zd63ES7!<1Z=uC;GqwsnLiO3QOQl5%^1t>B z^$8z;l?ky;tt0ULRG@cb-qvuk^mvJK9m}vyO}_8g=#vl2o7!9jZmEF!>tE9-(tFDB z9|>8??$pC5y!aB0BH|oNu2n%MTh@y8_z)ToBzDFP>9+rQ&E6}hw1GWlL>E!){w6%X z-h5=cohe~9b3rt@R115D-88E0Vk7Qrg2QcNp;M))2SS+B3vId+-UK0lzO-@^u|c;f z4$m8Oe}fs6XSm;0CUUlTnZMb_xqg2?88JuCt+6hkguch?OQR7t#+!tl34G)W5-2&z zaSx38YwQ=1I2A3SN?VHA8f^@fjarrBO_shwR$*n|!Y0)}+i?qIKJ@4}b0I|-{3J`L zai0&PZ|rz(ep%BQSIF3|rtI(BaMHz`#Go%`@#JzlSv*=hjwoH7V3}WdbedYRrjg>U7rCmLloZqn~aiHX7Fq5SD1XPFX&EDCvX}Bhhin++@3^HB7oF-t5K4}t8_&D zbHbjbkysI2u3$6RYdC4*{D}5mGBfrH)WU^Uh#1!4hA-FK`?rN@aoQH4B<18@%v97X zQyAZ01R}8q$0e!``f`-bmiRN3Qe6Q7Iq;esB_1Dr>2-v(8$>Z|d6%JzXgih(LQl^Eee zU7^4So+yij5dPentzRvVGC0npQ;k0y@z>g*CvIaRG~nWMVlE&n0x}3t)2aA60A+%B z^qqniD4_1(Uu=yxjJ3^IhcmthC@hN4Xs~zPA=bRY9IAcVxAsrGVuPZs&d#RpsUrb? z2e#8!ItkfKN(RSi-zm46q`nJ-n@Jl9x6yA@m+4EakKHAw)Rl*6!14Yj3^`tr6ylF} z#_8O!(=RES$i|U`^BcNF1@4?e%iWpvYz5~D)ZZa94}*pDWHf0Ld_I$j?c}jwZMX|e zgC7_8$PzT+G|22K_+H|#UO!UGe{h;hCc(;>AQ{kKetYq5t~Q)R&7nH?>#nInxTM15 z@M2aOrgCh;v$$Ui7rD7;Dc|>XfUD@Wwmrue%C83WRjK>@|%9<($V{> zw56&t@{Tnh^$(&jY)~AJ8T*NhY_liBN`sWymRf9Fx8%RGG=jc5@>}jwv$U{;tw_Of zLjUfC-a6yvYPhrva>WS^Sl7+XpBuga=3j&6k`x5hoc$4#8O@8%GlxfvJN~wQbk|0k ze4gJt!+on%bspplKq^#T_CXoPnjEZyD;<$Gs@G9oUCte>>nhzB8P?@b4Yyiy6eM#B zTa;^G2%>tm9+(jrQ+dCS5rI`3Ht;THazLS$u`=+1(R~nMZ26`16sKK9ed;Gy4{UIG z>r6oA?%w%P5);zmgPWhY1>$s$dTUdnB%sZt1gIaRj3JpqnrIvX73j#G75@XsV)0Lq5$*bPvBY4CD8JS9iLDL zmaO|4Je1{UO<8-(BS5|R@jHSIKVd~KPhGOBS%T(~in9WE`jTf&>W#*zr%iHV#HNlV zZ17s!CT28)vTO+S!VP1Bkk`;A8bDi0ih-u$DCp`f3q9Dj@m=jL9dcp7FwZlzlV5YE>h?|o_?Qw<1S1>AJoX9z zpc7ig+;Gpb9iM9!t$#U|C!t)QX5Kd`$NGq~Fq`YZWlfuz@HZAqRLdX6ilIO)>~T#S z@dGV&e2RrqX-!=ZEpGE>hfeA;Wsx$emp6&y-E5`IG=TI8{eGhH+>+&Q6OII(co83) zpp?1U-@o$}$Gd|hP0xmsChx_)+(xB0vD=<>_VHLX%4iC-_EUvxNa{Ke1`<;h-YkE3 zES{DW&uJcgz0rz4**Fw&L@GyOwfy1y@k;kgC-N?})Uf8-&E5nWt6ku2GybUkt_hWh zw8M4&*G3tRC~Gmsq5PZlsdVfw!k+}ePwz3zQod=9n~rU>5#WxEr?$Hvo}A8z?Zt>Z z;TP@+Lc`+UM1!!2l6R#kSnLB=Wo5~l?mR1J2KC%$`0yuICNzIIN1O+qdhi{2w2Q5{ zy|q5F=3{0#HB3@^qrud-A{g?l?IDHbTWb3(JrhdzN1Hk4PhLJTxaLaM<J(iI8UzC8hj z+QikSaNCxR(9}B_oWM~YIeq#jMh*o`bgg@}c^SM*!*B1`^Sh6qX+rr%PxFn7mR?x} zV%~;TLsJ}uypd9&lHad(FOUZ^ZSF_3%VxG?J{Vg}vgusuOpz1uUk2|}=+FgW)Lgre z1ApocTy!u=6K8_z#au*v598*oT7>86p#tg8>yHzu5s4M}DyfrgE_X`H9|c&xk^`Ck z{Rj1P%|kTa9`idPnk(vm;yKu63`3Pt+b>vX0|(B)-}%Pbc(Ia9$FkVq*UM&dI`dIwR0Eu zGU*(9H|kv3<|&=fS{vqfg-4gIZc4m7yBuFqZ+w_9!W^yD6NNN&EO;M^D}rCQJ8rUx zFbJ(s9jibuDzD@3VP00*aQo&_{PK}^K22miq(ly{7cQ~UHlL1=8Q25<;t<(Ii#X4SuxdjJk^XfQR3g1BayNUOX+IUM|odP^RafDbQ>QkwqmB%Wb^EB zym45{YcO+L@h=(;;*P}Un_Q@;q3*w69a)E$+0y5&AJ+L@w$Gb|t%q_9Ak862ORQ$v zU+p6)s7O%o$U^XOKv*`z8}Lj%pG8VcrVawGdn6}C_f}RF@NP)^arV( z15waGy)=pgs`B!0-byJyiSv(Esq?lf3)q6(n{7-7^JoiSh?RjmBl;b95YQ|MHeuF` zI$iB|Zc`WnILVs58Lz%)u;rHR9C14wgMe8+RzBLn_rV6c@nHMceb`DJ0I(+}VJzZJJGG&Qp3vc6Y{D1#(!o<+Cju#mf8cJ}hYYa}+jhts z{iRN7>&5t8SoZ4%rW)c|H_69m2XyHadjlb(h^-FX5Yrr}Ye7LD<^}>J>x9en{BXLli zpOJlnaO%MCr+|EI@*-ur)~8Vf}g#jrVW*W;_yt8|FhM+*HHWW4LFDmBB83#Th= zk1xPz&~u2C0@Y=htrwKrxomkOO0$fR`7XO>22Pn7m;Hwdqq@^v?|TFz8x1^t#vN@@ zCY5OB;zo}&nRG$mT8h1PmeszuKd+h?n-ab(i|S|F( zxBLYcJ=|38$b++W2wDF>|BY1l_RV)OAgYHrW2|a zq@(D@oIj5N)-flhH}GOJT2$eo@I610`9^ic@ODrOqYeHiPz{lm|L@7lQ0v7@6nu6C z*q^#ch71eo>|34F6?Pim5s@g2ciuLC|49iqG?vy{;Q= zS>)BtZ!u%2C%t7QzUw=J?@Z$ARxjqe(YK5rj|K zUT4;2MkV5NMu%TWg2?jypyXL|f*9odo=e6E5$oNGB}$QO`eP108It(T z_{`H4Fto8d@N~+*Nf8HH&{W{qMc9AQIp~9QM#e2P03=~R$4h)F8J9YzH~i>=yTMMJ zDC(lpnLl4p#{<@re&00oqmWffk=-nE(bXw@k|?DO9aIq%(U?|%bt%Ww`L(~DxA8Me z`_&4`QfN$Xky_J7Q~hhL;OAiP8I#xJl^vT=+sg^3B~!DEjMhRGKPZ29&NfHo;kfRq ze76I_b_U?bD?A9U+JF4yVRM=f3ar)7b_s`(c{fE@DfpSn$HlN({#E?Lwk{vOWl+iw{E9C z%=Tg5#H6W$RK&6XXm)$s_Jnfy`?W|!gA8q8P?QP=0}s2O3_lgrQF8aI$*I>nayR@Y zfP5@)>xOmsjUrx+bhu4&I00PRhSEeAj)ZL9j3uz3ib2*6LHl3=X6*j-wpj60-FtJr zv(9A+E*}X?w%SmO^|&DbY07aiu=eBi>Vgr5Y40Vxxd~Z2UMS!nxkBcEdC_HywlL9L zb+DfSOGOkqUoYqBb$6z|<}Gg9?z4y^dx5GoB3ksL*Q($axhDoB+()s&Df#>Y)%g2K zco?W-vDUtC*qG2DYzQib`D&pe)Q|6q3wjbDT^cQ}^86a*r5!v7Mn(cBuU0VRVEq~=dvN+dRf>lxzUi!;_kOzuv|TI|&jkxyA^n`_}IlXni< zB^R_QecNS_s^Wna6G`A)PZB?DV zJ`cdR!x;H6xbLJCBGD-81z8Zr7Z1lrD68;&s|%HW6yzwApgsdof_dEAQzXFamrjyK znKHCpD^HY|1v359E3H+Lm2j<^zGLq>kaZp|_%Y$Zu zgF1M{vP+PW4N?CQ9{&P9^*8kGPLuU!Fy)VwUbM4-InXrX)c?zdg`D))#aO4TdcrC4 z8OqSgu+QRW{4nBfA18>QS}iPMY|>eAq4jqbW_sNh^oSrWni;-R( z)vm@P{4l{RF>JSl3Gb!*Dn7ChsBdzRj?8W{B~PIl@F@`wf5U8z_L6q?t5j9ebsgq-aXMZB4PlIKeEtCOP$iB}=VFO2nD^w7xYM_HsuYBq zVVkM-JK)^OF4gNZJV0KWBuItqT%Ca51xp4!R{y&)k`aOg%&QIOrhw;E2`OQ zlSz*dFz}!j8_4)XP?!;_5rtwC$0mvc%C9`1^6Nx80inKB9}! znOWLYD||=6Hn1)Vicw-6V+D+KYwecbt*vc`3Md@yxD^UT`)j71GLv8$OP_{6q|r7a z14h2PM+pwrg;16lJpJ`EDEkb_*a{HIoQROzR2gz`A_+9I@xSi|+Cy}KAG~|!5O~w7 zR9JH6ssGryTze(w8rLiS+2rSZD{^%v*4)0ouP9#XU+u4${^-OYHf;m0Cjf0~j|vjx z+Gpe`49!y!Vi9a%q1tp*X*w&q?Fuo$uQRlWs-@?YrWD-3OAq?8El^=D z#yr#;-yr(fZH3l935(OjYA{cw{c>mNr|yX;A3Q=LvQGN!_G)gi<@1)fl|*?@py!M; z=~{sDZ@zfVy=bo4NRY?0`xss7V6}clj{JKfkX9QWDBIKYQzScUKeh`fer9xj&%Z7e zr!Q}_P=g;$LEcU0?WW`lsy?;R>$wHUFR2N@818(-$xKF;J98KRI)Iqk{VlYqAF)OY zHYFG7hjj(W?Q!w0sPKaLB5>Y)Xcclji3w=&<=LYNKD`IXhc@Rv)fw+hsxDmBMN=yA zof1+(dxuR;i>hDk;1{{}K8?aFvYkVwtZl*|NkIZ&X+Z`!O1RHb48Vlx6^|x zHd%CQkx*3|lO%|GV|~`PjG|SRRfDV$BpN8{cS8#T=0~p@>GZZLx)m5IjgK|SC4<~3;4TnBidc07hS(^|w=fGVDeL)}m7wRn0 zd9k5|W*pk6EX3fc?N1aQeS9lle#D%^)2YUAY+pA9I^QvHqGkCr2h3idJrt)!-!IOQ zW)Dknf6=A7jsYFz56&+5)Sh}EYm`$S5gK|{g!ntQEuEkBVHKd`5^q?vGq_Lvk9zpJE-6RmLZlq`Nyb#!LF_AmgwRC-+ea3%;~wK+wyUmUY!NEBM2 z{0v!|!a5?Wcek@bBO( zKzrbNk171Zd6|YNid7jAT7+*pW%^^px^xVj_$3l{lxJ+M&^ed#101>T2E&yLeB=|S zsZXim+jBIyTIvuA56FTsVT1|C*{O8vTFroTu|}cwutFpfQS%l_ z=B-c{#GCal@pPex8yS}6-KMeB?9)^&P!J@}w_9Y0#goB1^IqCfr4s~eUyHhqxGQXCDQ!^fHQlI*%Z&O)p>891cQTMH7rrCLFlIii3yOb|zt zhBxFF>39&U|KdlCk;_{PqlTeesF)xjVeNctlo zYfbNOW^^{&L9jWv@81?B#R=V?di||4q^h@Pib?^0O?c! z`!g9kRA;`Ybx+rxi`Pyej1!vhgKQXh>4l9D$lH52n`VcFTR@PsQiI*|HiWIH`%H$x zf)`W^axZ;;|B68Ifk3YEP8hNsaY{>cQM=z)`@m_}VzBdk3}Hsm30{zsTFGV`>O2CY z7~>8Hhe=6M%BH^fPI950(Uv8nL-!#65%N*!njKXv&;z0k*!1PDCJYTxpwYS95p+E4 z@I#uevW`jayT7IsYsGUJ>WkzX4%sxztlwqqCvvZmMWaYO)m0OWyg23VXWY9ytvwn!`F{==5u zO5&?!`pJCAsU{7phDuJJUw_R$kd*eK`8-0*$0X2DMiTmksWp-Agg<86Yy^mx{U%@2 z+BcQtdmwpTb|?-NgzxKr&50_%#JmoIWZW3Z88eamH8Md$rv5b4BP${$^q1QiLziwd zv^a5G(c=a3fNnXI&xOeR4dhe6?a_ojTz=nTi50O^?=dC|KZh__bKI9WOUI;!lGKWf zcO|ZPf}Fr0>#?e`)-Xb&+F5b#ndL+pnxiIVxA^_^meGpjd=J|~_0{sy5}U42{o&Set2W-hRb}9w!#=&`FxEceyd>>0p&$DOXu0$8h;wQ_sgs)oHqjw|EkLT$xu)-} zXOrq#Rj!S^x{4Y&S|}93C1#8xRbJDk!NR^gXDcwm-klirTvV5}pIp8Vo8lC1OC|($ z#VWGXw-)lmBt6P-{_)b;ugE<%630nRq-WDv!0)eW&yXovU@6U^L;z2T19n#Z^_z^c z^fs4aYq}GSH2IJ40hrIw9BT?ahb+ngz&W!=+O&bB!HmYjO{RwL!S)1g9!lZk#ngs8 zeOk70x#`Ias9gy7T6XE@*K*hkRQ_p|M zwUWoMX!mqe=IIBPiBA5b*<}eh@o2=?A1eRCP3j$w`B$eFI4b#9iFSE18Qi#n6d)TP zM@kccS$GhqfpzUW2XW0OO$;V4O-xkji4Rirq7#2*sIxH@V3O}HdS~snFze2NX#7lG zu%yj8gpjLLMjAxOqKEge^n?ln@RU)Kk0By7z-6T)L4r zEJybVTV0sx(XJ;c%b51JWi1DU^js)3O|iBWmf)}Fp=(f|m38=9E1bu{CfnBk6A}b{ zcI<9K)!klj>i_A#tj%5$ij~b*bJOpHtac1l&5&ouv*7urFCANWMi2uk zHyz15DUnRAtL(W`T!(+;GlT3^2cV^+uBrVj!qdSfBA|YpNh~Hr^QU$Pkk$%|Cq*8_ zW-14ddtr2uS@#!j#FDDY5`KwgZzCK^^87Rkt+)$A{E^4?$;782C4%CXd|t5R=70)9 zhN`=@uV6w9C?cWBQ93HaE0g zvwgWUqC_;`EPMPKW$9h=WH4#cceS>}XyiCu1YYwa(gXUYN>_#QoRUVc#zNyPlybg*=hzXKr79s;4dOh8 zz8rI4n4hnB7A#2S0x_;Vw}z%WnZlpKj*HHdD6rc=l=#Yb6{x`XzY$>|&nG_QY7 z0-nZH@CJI^HdZ7OdQbkwJxk_JZ(1w<4v3QHw(HT(tf`V+RvZ-EQ9^t3k%4@r)o(#3 zA(Af>pQ09(=sBeZXB(Z66Sqi_%Ww+9gtMp~1-$;r%07LA!LIs;zYl9Z4qf#QkNN?y zg#RS}ZPdW4qpL;K5~^Tv)WFkM!7%&v%${r}a_Ms!B&sMN#(+&U_|Ayx!F4(0~E@=aM6rXHy)nchj&fA93l*s>&GP-poJG1yIt> zjQ7q$0G}Z3P9#|xK9~f7VCN-IvJNN(h(=sNmjCvkAH;qMRaDZ=?nhksslpZ>0Ul(K zkZ_EOD`C3u?J66@qZ178NrTjF9Le~{?M=;q0CbI20rCFo6f=YDDhyb{J9El>Bp&TH zv|Zb@=uB@0!_cer;bm4tQ)X~)k9)@oQ3k|G)~7)T*cs6|Gw*IxV`I#|LBEIM50`&4(|*OHKxPTB+;&H01|negb+{C=rs}- zm!}Y;{KU48@IBf5z)m`ceGT+oZ{%fg*Q_}P(<%rO@GhrwI|d=*ap?5~K*f!``2uqF zSaaM4A;o01!BSc+O2YL(!uONix}hR%{S;NlPYv zY~LaPR?FSk1Ir>*f##3%Mu`9S?i&B06dyog!DnbSOjqbwWY+CUE+pDkK3pTnjME=o z^+&vY*XaYe2q~|fhGc+x8TVUnh@#_^uXoU#6L@+jQFYnj+eic6$E!J26@7R}Tr!L@ zjmA9w?y8a`NEVm%I3|=<%z!gv0v@S_^Iz_dK{~a_@Zg}NlCZbVu*sU@&$m;) z_Xor-Q?439BqXJ;6tQEIf@tW{2F}KV=6^{q$wQF3OE4kbe;hz&l#o8?e@9Oz4S;n4 ziM~+F+;|Iwy2F~?FMGP%(bPt;K>tuQKudOh`MDIWya_#V-CdOBi@FFP$B{T+Vg~kQ z-Td^I?U+bnGFSeFRN3%E7;}G~?kF==<6d~juhh_u5VKx=#n75WHYn;?@b8JInzu4; zKyOr0ntv-Wy>sewo<{na2|-re*?b$%= zxIwC_gkr{%u48dOzw4*}Q*vsd=bucuWRBLEBXV=?c(GD}L=E(D1&#* z3*8>}AU7eWLT+uevb=&jSY(6U>|*`Pmq1)Wr|7jKjBlhfZy>Vp7J{%zYsw{(`FkoSV{$)Z5)KJ)2bx( zQ_dy48n4Zx%mlxI80zYy|A(Q5<ahwT*9i|cpCvjx3Ti@VQhiFg;4%Q)OI^8iO?ylYmP2{`;qnZ%1Y`maJ zS9K2`7yYFcWeG-?y?Y{sEi1~upsr%&Jl(wV{T4D1O(g}^l3LCa{F5n+FcY^^QbwoC zwE3J)x-&7E=&K;*@W1(|KtB%`Z2)JZw)}C>@GBWYHaOJ2xpLk z*%{P{t&!rFf;IY^Kn*^5D?+?gA(3;H@eACRwFUqTh|^G{S$p5O6p{b+3#6d@%^2x@ zh3BYC^!0d8Bh&BO&{}A{lWkPq&iL^N`&0gXfCRhAYMDjgzm6j)cFL9pllvxyUyOF7uE0%}D5c@*%Q>nx5vRU2hYTqKg@)<0MjppCLkkD79 zXx!uam6j2bkb6=^Ji+aZ_SwV&HMzzEx8BDNh>em|`@xJwz+hmBqAbXZEI%+xn<*9p zqTN2i(EzYq0qZ-ZQd@H2tH`;l-9CWBadpqyWN}(MO(xqB8?ncu=nLjQqp1n0{ED@` z3ReMsXCkx~$;XSP`PA0P#x|lJ*Wa+r`M8Ftt>nIitpARVH@Wngz>D}7SxcBk+GPDK zl}NZ+RWQ_5)UJ`FT!5jEba&NIJ>e_w?^?T$^37+BWvMZ!%2OiOi(bKrfHh^I0xLdU z+R*Raw4XAdEm4_b7LCh)6yN;k^?PgcTcoJ@{*pvXS`6}-rRhXMMicN(ocRk9p&*Va zv_1i`WW~44VgHIVCHXN<1RPH`DZb2WR}-i+P@s{%+Oe&bkQV7j+lCzQK!iBOikbg~ zO{Xb&V;69KBjd`zLPg>Q_0Vgh-|Sf#;<$Ayt5|x<k-zmwxU9C|XVqoM;`=~2k|;5% zUq)dqd?_qsY-Vsb)ZJG^z>Lr4G*?o8(43FQk+!nMo8;n*!RJ5=^6hDS^63LI)NZ|{ zHySC8z!-^NAk&4^K(xA@8Ta7hJ2Fquj}3k5G2Osr7|z}4ZChf^3WMp=4}lwgIs=7D-~a{$gTGYs zn@abxTMTP_tNlSn$Z?a}8e$9h$wZ#g%(+B+BOk$~AWNq(tQgg|q3Eo$p~Wf%v3e0` zowMb)!#Z*x@$({K0XF)bBM)>tsaZi zR{RO!o|Pw~B%Qloh#Op?%mVPfw1)J981(d@PGBAvde_Pq-Lb z%B{|eyndZ~^S8~`!))6Rw#-xhkP3*U@u92Zkftfm?$>6LMsh9&B1@zRak09eOk51Y zTHAC2>in4NFFsM>qhNfX|m>bF1$H0p}5bZ`_t{FDQ3 z^20?P<*^x8UcJVy#%@5y7q;k}&WIZ0!q>f$$xspqFLl=IN#xDgUb|-$JhU)LBUn0U z+2Gil!08G%>E$D|UMF#8-x*RqHU+B zoAL9lNK9a`SemOKN&Dv{XTTrFCVFzqAhpkCGkhy zGJ;M7&nFnr&H(BJSpI=9Q;!Mo%kGs9TpZ5gz~-j$0E5q5-!2YC``{(l)QoCZE+rbO zX`Tly+~}D-ka(6qP|>%d>Snd|J%YdFb8Q}$mGIJkn?EndOqNkT!E~&9Wf#8ZdLtw3 zYzH#Gh*4Phh{^4&YBlm-V;t9%eYl^fvj90En=dgGC0J#!}0c^r`#+wQUC0g z$|t76z*a(5^r8_PwBmzF5@Nv{-`bS#a}Nx~SLe_n$Tb__%Yo4Ch*J^JK_t7&`J0q{&# zo8G3=n|C4HHBsODL>|GKwR7e|QDshDtjtJ1pH>1$DH@v0apCvx3B^dbY8=$QLK-sYWOd+Aj`1|LU!6mzGs8*oxM4k4{<&{H zf3aC=IzVY8%NR(CL_=NhV&I+ z4&4gXzTBRTRxtGZD{Ut-pBXx>8czsyiWui@Lnf7)2s4nj2;ZetYx2aO_5s6hOi%6sh3j zoZT)-pN~6jGc=3m!n!5|EnUr2Og=N}Z+9m~W1h!_e@90-w?oxz2V)gtg`OcR2|OPL z>pea4yl16kQ<&fsV&(j3!aC(|}=;8M_-f#d3{6~n(F;`8cP1{msui7KFn zWFWvT2+1kumunqWx!gP(BDJ;FM8jy&7_M+a%N9v1l_7YM3atW0Qo`NwF8c0%JgTY_ z_QmYrb7fH4^D)A+Xsc;0x56M%_=bHCmaIAKkNv+IJL{;pw&l;?5D4z>?ykXI8x8L6 z?lkV&Xh?t%AP^+M-Q6u{g1fuB&*9$p=FOY+o3-Z8UTnJ0-e*hgs&9SDdg=L1O&xJZ zSxYjn-@S6l2vgURM3zKe(Kfnv7Di%j1svZuF4}uWqw+pjRHLa5la{u6pBeZ=ky%XX#<65;?5QtOtR;R}(;^GcVxBe((?ohfxqQuN z1#w-F)RAD-<4ok^UKtaY0aitH%z?YqVofTEv>a>-Zr#%suGY#KOyjyjo-L{p3@$ZCAO&S_2jqd$z?MTKE&+-~2HYY}UB~>dI8v@TQ zpuSXJi8Q#a4o6M-yUVABf8Uvo2BIbYTsuh5H82?hPQi{N+=J`@P$42Jw{I zh$c~)NARf0$AHBNtgN{2?`^8*;E~2&F}2kI;3LR4RTELngdTduh!j1vXxiis{RP2_ zb~6rvJH^lxQyswVF6xD+an0K$~G3@>Y>c};pr_u7DGf(Oi(-iish>XPgfim#uWSdisN zFkT;?AAgn)fmknM_56(s9JhW`6+X5Ok@REL4JLbZhGkM;`7h0|ksmnCj^+IKn1>l7 z9t)|vXgK6?Q4^;9KiXeHyEqUbV3`l}*)#CtKUdMnr}_I0{HK*!+|wmBR-+oV>j@9WW+V`scehxLMZ-b_3ZR1u%{uGu*41 z9TdDOXC^v7v&~C-Nge!T5mx+rVE&tb0?#{Hi;d9Ob>+nNVd%5{zQGHxks_d-a&-v_ zn*5G&D3BEWWE0LNptC5G$M};Jj~{u2a=ud5If|!e<1C5v_Hj{$?{%~hF?eZt%0WGZ z7dK>lGhrIVb*A&{xvS81Aqio&6#CRlq0wBbbvarIzOUD!0T?*m1_uGlW7b}^o7kRD zCd8+QVaKzzjPSIF$mK_pP=v3(_&Cr-)#+KxZ#>~L()oyH$W7rVgn|gfS57ccmtJvpMzeT`zT>* z9t}$QiTiTF^VJs)FmgbFh;_pr9H|nyl9#LUP7)2$hbTJ9$I_yyBiqluGW5JI->E7b zto5KInftvB@cD+w8JnKr-P2#6KIF(IJ^;2=FU(UmA=c&V4p8(tB^mWcc7I?NLBiO# zmcClQ>mKOKgbz6q)x31Q(5BTo6;I9xDfV-a^3>zr?L5D0YYZePreLg&HitJvFrL#i zVPN@y0Ccn>8JX611fM5s=|G&7oJ-LpOM2quo}!Zpql<&UtVB;fyP?kc%+zc&xiN z^+t?on4X~&#YFeUvd4r~)HaKRvgvj#+_9$_=)-1b%dR3XhrISzv~}hqjCB(hmd>?U z)il@Y)4Rt~YaVMf5+aPTY2HY$*|QC&7vt5q?ulVE{j0(dogiv7>dhSXbw^ZxQTVgE z@ok6GJ>pgucPEUth4k>?4It|}nphR43?LCm zc^$5aa|=duf|c?hAq{KEY^$06>pwANU8ouX{1hA@4JbxHUOEmW4VMatZ z0hP@A;I%snhKZo!Wc&NcCOXcCDg(WK%O&9yO2H6P3s2tm$8G@4?{XLHcY3N(w$&}u zx6#M4c4#;`3p~YO&|Yz~i9EfzgSaf)IvC|O0th4%r&%i#wtFyZMc7t zl=Od7-`D@9^ETu7Q~!)?kSfo)oRKK1YG1Ry_hp!^#Fw!r%(wgpODF*p?^#L>xPLHK z+#EM$ZKI#2Vt~)m7vIv!8$bC$Fk16Nrp{$o3ae54-tQlj?CgLqK-qIo$FKg0xrNP< zM;!qfa<2sR{*AZ`3mOUGUt^yW<;UzjjF*dA%Ouwo`_;DKCfEQ&lvnKmhMWmh6ou3KiO>qk?@e0kc_5N0?x#p%tcSYp{pK27MU(^`^$SA-!cX+|1abE^M%yCwpJ-)} z2wi-AN{%brx9MSAk^0-DFcvnXA;4MQRx+5UX??ajZWD!SHv;Sv3WP^KOV6s|X~5?4 zFV<8cXTJEm)|Bn7S0^kSxySYGl>>W4v9F;!M$79Kv9GWnJm3#rN(1tj)|Qo%oQ6!3 zR*kDtn+)Ig3+VLJzqWyjSO3gYx(lsPhXK4ji)qu0ArWZ9Q$zSKO@Q0$-#Dv%`<)32 zEW)SUJlOb8It%eZ(hSDbP z^x$rpGLcT=!wu=Ej&k}xOeK%&jeRaW=4W0@%NFu!_M`vqZlbP>GnBBtGZdgv#nmxAF`I8O13zv&nNs9r{S%rk#Et{S zFR<8k>N@)KOJ+N?1OqYKS3X!_t(ZaXc)rHZgF4O9j#f-Pv7E%l`P7e^7NRdHX6P1h zWxCoQBD7=eMc@${W2F2buMaY9Zd#VcMX~JRD>-Ie6Z^Z-(a6^I3sJ7XVw8%av&M!7 zV-4V^Czv_7f&D)hSr4jAxDdQpki&}6DuhEPNI#Z|ieUcvJru$VU9Cb$Lh?qE*5MN5 zfN1>$d==Qwc9l{W1(~e+=@o|0#bzy}=4N+yzLh5I&yRtLsT0Qa2L=N!VX_fbq6|4$ z@H;-@$MjPR?^T4~XxAP31M169s_3f8)s4O2^9u{O-*(zP0z^uDpFkwbf?$g}e&>_h zwgw+rD;ygsI~JSI@fKK@ohgVe}%lf->U#6jjEgir+6l zp32xmrP<6i&5IF5*5-&Z=|}e*G#ZRHIWW*m+&O?|(80=JF_KI(m#O)9@@?jyQ#NNj z12Xnr2-v#d$@ALskblk@ZiA>gdyZ>5*V$CrrGSYZ9QU7v1x(&TER~UGBmXOt+<5J(#`4Ayqa2?#RmVIA8qjC`DEJB!iH-HiA ze6vubW&GY#bYNsme2xmouyy`tjyVCHZhY8ULF(G#B?42wQndgOWNhYf>U}tHb9wpH z+0ddazsOgBIQ{H~xAzB`9C!5$y32|=-1|fhugtgQJUlu;31|7uJ)>igN3Y&0lP3;! z-WlhXN5JlqGZIx&5w!bx=Z}@|a@k@v8#ThroQsJGJ<`|+GQGUhq zQ^~;&!3?XV*jZs7v(#X7_lL`6D&Ar3Q;JW-ETM}T2)-a32-@T=^>Z~LiUd|)J)#wQ zc0~`g*rK4dpjyPxhB*hHPdy%d>-{6L;zq%z;)&urxYtQe;*S%;hdR2bPKVJ1oCl+n zE8^B}0+(+iyHXnPT3X>!+Koa@2RO&CnsLv^@6FXo;tjsNEjg{t4n{4<;z+MhFu+XWyTc zch6Xc-$+oFC_ql~Owj(*PH#F-wmFS|ymJT>lJ+b4;yP#i(+Hr_L;#K4Iylg@1wPZ` zG*(Q;VlCd!`8b1+#iw>KqLv&Z1q=8B0(^+vh0{(XI2@keU0P#mnK73%0ZBYrJ9E$< z?=eAb-`;k=4U||aiC79?A++G65w(6|5~p4HaOVYL>RpYMD*MWz0k^rvUb{dhkdR}3 z@1Uaeh#&DLv-1aYq|4EVu*ECE%}Ly#9r91+L01lJIJlpbC^E}rtG^if@p0G{tLLU4R;@|2NS2y6 zV}O;X^B1DZfJGD>qEu9YBP<({C!qog32MfA38=N+BiHEL zCdx}iTLa0-;yh>$iy^Gw9dDP&N~kPgT>M2VNEd!V;r8c0(qXl6>sMHOs*>eC926hC z3=khIUx{uH?ZsVFxD4vAvuJ z-3p%1`YUzT-AQydJnD4@xJDX3AAQJi_qZw_ZadS-Iqe<{sjjJOpBZ`0MZY`u46fJH z?rG<{PzzrlbHZwvJii&ZuvaQRU1@j1Js;xG4g)WIDG^`mdmmt03R$T2PG)nt#~>g3 zRT*_VH^4`!;BzLhc9VR3@|P7S{L4c?=IK)eW>?k*ewQ!j3tHQ<+GbNZ57*gGSUXD);>MB6+%$WJFZ;i-GM{qyaXR~Vg;8Psr#^c*@i#*@vwC;QF z-d}mFc#(9fLVe*!;RqD+m5qXRQ9{wJ#TYv8R$ssITDms>nO@%=R(E_NdcTJ`t3wb% zC{r)95ut@ms?CZE>7w2?Yu!QCUhH+GBIM#$3Pl61FF-wgt5qS|@_Ph==K2!=d^@ z^z*QqBkTM7-b=Z)9$1#v9L!SDraT*&tXXZ#!FNn4`Zxo-Jn^^+!WF zlH7f`4ihw?`jfWT;(zFwicbh__)p~?%e zmp*VHgK*U1I>gK5AuZ;dI1YcKpb^pHGtJ;JxFz5i~h>bxtdd}`+ zg>^59UnM&stB?=3G?&2p8E;4(v9sAXl@=Z|_U)r#fPN{MA({;r>bNQC+#DFv?S5=3 zxD_;mel~=G7g3#jw!aPMZOcbtXb9u~@_d{odDw|}oDORslEMF2P@=wp2QB$#IyPHqCU9T(CzS{wn$Xcv8$m0vYI3O-Rd8^e9 zul#5ccrS1!1fb@UT8>J`ucIO?u1)nO~8vmt4SAmxSbyMwIRuwt)Euy6ec&=fDI$MEx%hAaNacr`>n)K^OKP ziw%;D<-#qt^nc2yw##O znsYYg;6u{6AW2N6#oIfbwkQ`DanEZu82t$I+v!_ZQVUd{QG2B0d0>yehPUiD&ON;} z%=(4|Kfai?U;Kz3sc()23^G;`lZ~I(_y&AhnXaRJs(H!HFVYpYJ-`3*_%mb8BoXsP z=7r}9@0wXyc*Z_F)YguhAxoJILlN&%O*pu&6)MGHMPgOEg~l{QB_jHO#a~VVnzcJc zKPFL58IqZF-b%jL6vwvr4FoMu({PAfaVU(q^a(X5rpU)qxd6kL2_K>_GE;NwKOi84 z=+JEKSv~;9OS!VkZae10lQz=4YQho^(RR1SyLP;Tb$Y2FF=7#j9j1-#PHbp$;Q0lE zpH}b8$YU|G@``%a3OG$XdM_d%&rw7q`ZbrnJanh4am`H6U;`lLYQeucZ(!UTtAb zu$eTZ8_H^kq_XvG%bXjg?OcW>(by6Mv+-edm6+Ecanm+cX9@cqoDJO`!{Kekv`X`J zUudNMp?7a&y$kSd02yOIHgZ=#^5Xfiyf`EJtX`{U!O7?HE&Va|TEbKHsp5K|N!4fL z&Mf>}lO5kg$dZ$|?1l?qlI%OCicK}$7uWeqRiu;s-pwy1humgyUf`7o1c=Q(t%w(# zpM21tX1*{sdviy1Tu<4f!NW zl3s~NjM<__R-Ms{@h-DY;Sr_D3$2A*s2tkMXrg`|nj&f;xMm%*L}jNDLc*HIPK8$J zE)RsHYhn-8y?u`;xoOdoiVLQsB2^4@Lq=jhvO0?2CI?2fCeHgiv*5Gm4F2)>dA2H% zGP2TH^(>E9_JqpiPh0uZ>PW#L6vkH=$Cvc|>O2W*72O#PWF%O^swJ@Y)>kBNW8&Z| zMaGEnE_3NR1sbl7N7EZfchsDoVc)CXg{Po)0uMvhQHQ^6;Tb23Q~;47{N#YyfGzlz zveQ!hEEYa0t%<^zXL^~6n1X?0CnOa?ar%u>3gXVXS9XgH!6yIMweZj!ALI5>TD;!9 za3q@p7Q`@3J-t~tzbGG~q)2UJ3R~-O52&n>a$EC8&DOh*h(Ldy-Awh0DhP_A zvD%4@roB&!N=}hdMv16sR3mTsP!21}Gk!<#2+^ALY&J5j2hs)KMzm8WrkB4xQ)`(5 z%pgwHRR%Qdssvo4V5nEXQA?>zNWQYQH2CwYxAcGwCvW~~MTjH}NUn6ut znDRe`Gte-c^Co=A)1BM#hJ5G?u^7%gWNt@9(O>-$L6io9;GK@t39b@s2y`T!zuMya zTV%Q|`yxcrS8oF4X(MmlnZu#ZQ{Zg7=&AHpBWYIE~FGf4Xt z*^E0Z2Dao^EsrpO*8#81nA5Y|50S@!wx|9evMcY#WqmwAvYs$LO`BWE9{iSCi2+Vd z_SG%TMqT(7oqd%PiP(j5UAHEpRZIA@?e5Q_V z-gcRBusb%v;>EI_E`t;^r`k;%rQb!B>ZU~B2C3H?JI%jKJH-9Q(ac`y#&f8+vipy7 zBN1`+gDpt9ZKQABQAtL)Dz5Ee9R#xl2#UAQ;EVpCkK~wz`uw7n=`7aE_D+WL5-pST z`!R8vpqARRUlH{!c;$ofs~aFadlOSRpOvDsn%VVMPwX z#Ctdb5Pe7~U zs_bHaYV3vYQV-;T0Cgh9VE@6lo<23m+NCzLiom&QeK#5(itWhV)({_7ekuA*(32Tz zBHAks0^pBXopvviPBd;%{=*-;-5Gah;;kBOU7AJ_ z5Tla-b>jEP+?bH!@nLb`gWZHn%Id5JEL>+a>M)y^Aa=fjDjOU{PgiAEQy^yPy02kB zWCBB?Gf#qsB=CDawxUXtLo4dmS)fZR7%KVVMTpd5ocX+D)ukoM1CgvkigGr1T}8XF z!a36h4T$G4~iX& z(gG3QA6qs*7afC^v;oKC3ocU$OzocJk^%lII9CU_Kj#;1Bt>-AQ_^_rk>Af3?LuZb z=`HKw{}DMmtoL`AFGIL;(KCby?M5@iFI2EO;$T)ARH;Z_NFr!vOw_A+4$LTygnO$r+qcSW}Z&i|EcL&sR#z! zBFb&52Z4qlU-QuAeXeOtParqsTgZctQN&SyTL)`@s+s7^%-OF28#>S(g2>6syCUtg z=Yaq4$%v#E1yL!TvLMf)eGozh&(t1oxG&yE;}mr*E4>@`>v~ap_e4zzV1&0$_jn)p z$B03c+{vSXO8_^nC%o1?X%($7q1QijB@g)NglOOQN9ju;&F@-57JL^I;_p@HxX&8u z#)=!NSKRP0R$%u3N)uVxKv3$yn8&JG|4mmZpIc!L2RR7CY!1NjwYQY4BY}2cYhx9r zaFDbh5xl{s%Tv962C!@7wHDS9Z2jv=IqSFuI?Kpsdi_#d54UH_hB*SR{wes_ms*>1Ai2VRFw8@*m#-d)L0G zp~>vr+=y~^Ll2$fKt=^`I(eOiU~?pxuwS5Ze^wa~+g8guCl;{+0ytQu(}ZP~*Z=B$ zBc%?T!N)~NTOSzQu=abLB9?`vMcrNr5yi?;MlO*$k;9H1#nK+;)tLrw&QP;^#c>}G zh9{0_3)3tmSV*)KXL*g6z4~}ZzHrLSR+zlSVbWercbZxGPg2h-UxJ_|qlFl_N#fD? z;$vZ>j>D40@919$wX+?||HS$%KVYPwvxd)<$v5C(I`tJ-hj^7XC;aiR?=BHif(dTA z6HWbbcL zWc)6cwIl%=nH(htf+F84%n32(=RQB8f!Y1^UzYBXgsuGNh6EcVg$czMw*gsh9ol|4 zA!%VPdq?4sE;=U#c$4%IQY{jt$e}DX^mccoAhNbKCwOS!qRL?r({a&)1_v(wK<8(b z*Ri0$B;GPI$QOox@D1&`c^##5zoNq0*=H}1F){Kxo^Zfe$`CWBwY zH9K&;(*4)Xd+1;1q_Nliacg}u=UX61)wzpMA{weJCqkDXbx1<2x9}^}FqP&>}mIByKju+k)IiU6U+Aqn< zo1|a5qHXk;0%}^-IOJ{Q)QNz!F)|J9CeNxn!dC3Qp z(z&MGsy3IJZS-&bkn@@IcJR%tx6{umGLow)l@%hmlXz4nB5pyn7x`H;2N$>|k_pqh zD{|>QMI3t`vgJQ0l1riU>nH$v+B{iUjJy=}G}MgTdb;Fpy66v%iWI#ZzMT*uiM^|D zB2~;|X&*Kon4*zwKI}O$wYE>1frGpM=tQ3GXJ(>ddmdYXPBiSTAt2v=lrw)8w3jUI z&nqf}a}W9^3uc%itl{fB1vJMue3~s9^+y_JywS%1Fd|X+!R97XmQ}bHAetqRhD9$O zqdPT<$V~R_JrKX_fZ(|CC$4Rguv1+vrYU|c+{FV*>?&CcRg8wp88d6b- zEf17uHyR=sip-9@C-|4?#aNz=n~VP1e!Gn9CK~M~zl$!x=CcByKk5M-;9G~A1h^@i z>={jBOD1ZSJ@%b88K`H$Tah!bBden#UCEkmK;2->OdpJJYv}jH_o~d+IN7Fjwn$yj zk)N+{OKZH70*G@sg|_Hc{+b$^=KXtbT|_f@^g%~{6u z6^Vxbue+1#mLYy@+&w1)KrECuCi&X@N{%Eh+#T%pJnF^Tpyo;s1im&Y4wX9bJj5J_BLKR|NxZ&Ifw-pYl+X=OE)+fe$LS z5xv_TqU_SLY=A*6%M9?%aE{WYDsh~rg5sm#a?X#a8i3!(!%+vpI1m`GDd8`M;@<7|eLbXY0rXLH{r(@H&Y&f*tMF}sj; zt<{&AsmTp~@>(b;PoImlmG$Y7s>jV-0P9Ygyto2MtzAEuKG&B_VX*_Y>g|szV>f5; zSql0;Hx_B7FKMC}KTL`kgKx!HwLxU+qnXf6zd}^kj&A~RU1MUIFd8(E6mXV}D!$Y{ zCYKL4&d8Y2?KoBvhEK@^;_xmHQnX3E7M`IJmBn>6T)vyOWktIBA5t{mY>M~NvvLDx zGa3e2p10BV;{_>0fh-WOb39b~YWI-?_z?9Cqk-eG$&a>fQ~&n6Wb;HU^UGoX*Ol`A zuYt#XgWuD4K;(JsfRQ%fJxU@FT0TSC6{0ctlRjKq9)bO+ScBIZsrxM^tntq#ymFRV z__FQ0FK{;Qz&Hd^&W@Dz>P=IVNGIQ-<>h6HMdvq3z<54BUig?x*LOap)=P;i(3Kr7 zVcgME{cIuz;~9|5TD{7YLBv)M+0j&!ddx_>c?!pkIOZA%fp5Xk;uW^O*(X2 zhXPs1U(4EbEHE|DYZPhn9jRO0Ev69yN%lsoG29tJLd{Cw<@z;uCkkNJv3Qu4GUJ8- zFC`&n!gsmf;L9jJE!d(41p?i|pr(EM>MOF)CGzZ_zt%4Dwe^XV-VokwZOpNvtk*il zlhNnL3Mpe8LY1d|?~Va(`X5K#W0+q8+jw(Z7cX98%1CdOWW zkjHe+?tR;;f6{O~-fcMDvI`Mae`|zt(Y@B;+%_H38%F!>MMi>;wGkg3y)$s^ zf4hQiJn?k{eWK%JV!f0;ty%DGj?W97x;EK$7ue-TX@+63Gxg^a!()I}!djV8#ihl) zTxj-`n`YWUjTUW8|GDi52wul=3gBC;i0n5pqee$Gjd9Z*yE8vuTsXG84BP_L@tUXR z(?2wf0scPFdGS48%pT4*TPVTDX9sLNoll&c3Dj$4#W76AYEOH3!-_Sq%YfklXmB&W&hg)a;(!Af;T<^6RyRgQ67hR-hcPcwqAm zMB>d)Bol7Q*RSVMQOIC)0fhn)zD!t`mkeUTXHrCcmpZ&PLjAZ@ulaDNwkNgQ^>0i# z=+J5bDb5Fh+X!l@nK*%=3uTI5!z;^KFjPx+igM-^V7fM({1{s*4>#KnxWhy=Hfrbj z*0Np}m|KKSemYYu(eY*;YspT5qr;TGCkN$5IYysG1_V71S&^eAU{WHV@O_Q*p7oiJ zFflOIfgYEUHqp+&p+(6M_kgy&K5Ytw^Cgl?4exxgW&~CtJ@%NdaJFMG<@e}W)lSF|yJ2)~L}K2!EDMC^Bc(>exFoXV3=m0#T%!xt(Poy` zn4gl54<62@9+PZg6JjI4_ILV&zVugi`zEE8cgnxN{A=W+Qi~!Q^e0_LB(gJh;9+*L zxAQ6Z`a?YIyw*ilN(*aR9u^Eg&3fA}`r=G}CK2*=7w7a9wq#yOFv4}}+|e64oi|I( z)e0wX1!P&ldO(y*MtD--+WRGt2Obkdw)oR02|%sxBWD0;!NM zB?-YiPOc1cua)WfE@83T6Vcy>rapwreUTKn!-^@>q)Oy!iu(pj<$iqe4bjh0A>*P! zs!U>-k+T&4TU_z~fH}&GPIpmk`ssvQq}^{JEC%SXh7r~<5u{yb4us5YcR=c60k*x{ z#aDqsoD@_QAd23SUGXPG`Q;YYvY4xf{q*W&k*Gd)Z%`M2Wa$_EI?SrO=|sI=TZ(KZ zr_o1lm1ZzId4c650Ag`4BsNlm*7#HK93pfXfc^$ht{N>J1_#T{^WN&+p;gCqSAQdw zbWB3V5rB;Yy(!g=MwpB$2I36P76IK9$WIhOgeR5d*#NuWt%dVEkgB5zzhq1gh* z2NwnXoD|q|dgXTW3?4Y0+Vrb1Lz}5q@6io_0uyw>;D5_BWpUM?pQjywCIuraK_m|A zKpBNqC-^Wx{s1UT{)dWlZ^T)WqaihO*9jL>uc8?$Q7(9!VAu$SP*9HPzqxJ8w*A2y zR+aWQazq1M_k*?Lol*q!gqRqtORlhfv4AD8`5j?$@)W2XV`fo>?gBy~eR0Cqw9=bx zrTTk7^%F~}P|KfBQTYvpp;`5pu;2Q4b;T{!YFC{@6a7Jkw|)I(ZcN_CT6<0etcivu z8K&n_4ptyhA-(ehRAU}d1~M#1ohaI)*)bt78i@ot2p1J}g-A~kH{xnXjT^_k(vSnN z;8a;d4Z;ZDt-ziyulLQb#I{%No@(x&Pt|l0D5>?R!Apacy)o5%IAJnU;(NW!QToSW z6JLTa*VU$2s8jsvw)t=(;qj+R7?~~nNG{hjcFMu<=(;tlj z`hWN;$Xa6{cz9P;=(rShgq*!s?m`UB)fvB7emRh5_jU*u)APt>jppI%m5^A85HB-F zH`bZ-n*G)mUdzDwRZ%6q>vWpf2qN^uZK3I#S76w07`YOSsi2e%z*hlf1eiK3A_yTLbKjSN`$d5y&?ISQO%IGMC zBLJdaUluFZ@>yyF>QE#~_f*68gzP!J#BjO@v0vl}n~L->Ukr#X-YH@%-<^!$5f%J( z;%vOrX2xr9h;_5goERy+as7cf$7%K2eE#vOB+c|aj!ha+1aU3X>lt;S&cO_QQpFt( zf~G1|4D&-#@VhSe-Ft7@I%1HRwYb_PyX;p~lwzFHJ_0xB0{v-1)YyZlP?b;_mpk|ETD@?KkJKxT)Ed^I{h)2w@6%o_)a?8M>h zaT%bnV5HC0BHS72r4>z>x^l6!T0Rod$tV?+Q~yV5-w)G8LL?+aJI91{wYw|g;iM@f zqM$GCC5G>6hi>|}2lD|0JMgey-e@7)MF@SpSv8k_qsg=!D^gXP=5m6zJM0#w5PeWa z8uwvM3Va6GA0G1q$7G#;<9{QDp#iTL;bNDGAi3>WT%49b1v^%?`uO-mXrxZUYCc@8WsXlT=GV< zk=N?5)It*W1dbZ3Q#VkF@b4!y6#u^NJhr(8KCTl3Ul!yvSjaN&3VF}HJM(@0a%2f9 z>`)xQ(et|J11L%x$vj Date: Wed, 9 Jun 2021 17:47:24 +0800 Subject: [PATCH 22/45] asdf --- examples/benchmarks/TCTS/TCTS.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index e8cbc8005..aa707b26c 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -9,10 +9,12 @@ Sequence learning has attracted much research attention from the machine learnin ### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. + + ![The optimization workflow of one episode.](workflow.png) -At step $$s$$, with training data $$(x_s,y_s)$$, the scheduler $$\varphi(\cdots;\omega)$$ chooses a suitable task $$T_{i_s}$$ (green solid lines) to update the model $$f(\cdots;\theta)$$ (blue solid lines). After $$S$$ steps, we evaluate the model $$f$$ on the validation set $$\Ddev$$ and update the scheduler $$\varphi$$ (green dashed lines). +At step $s$, with training data $(x_s,y_s)$, the scheduler $\varphi(\cdots;\omega)$ chooses a suitable task $T_{i_s}$ (green solid lines) to update the model $f(\cdots;\theta)$ (blue solid lines). After $S$ steps, we evaluate the model $f$ on the validation set $\Ddev$ and update the scheduler $\varphi$ (green dashed lines). ### DataSet * We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. @@ -20,31 +22,31 @@ At step $$s$$, with training data $$(x_s,y_s)$$, the scheduler $$\varphi(\cdots; ### Experiments #### Task Description -* The main tasks $$T_k$$ ($$task_k$$ in Figure1) refers to forecasting return of stock $$i$$ as following, +* The main tasks $T_k$ ($task_k$ in Figure1) refers to forecasting return of stock $i$ as following, ```math -$$ +$ r_{i}^k = \frac{\price_i^{t+k}}{\price_i^{t+k-1}} - 1 -$$ +$ ``` -* Temporally correlated task sets $$\domT_k = {T_1, T_2, ... , T_k}$$, in this paper, $$\domT_3$$, $$\domT_5$$ and $$\domT_10$$ are used. +* Temporally correlated task sets $\domT_k = {T_1, T_2, ... , T_k}$, in this paper, $\domT_3$, $\domT_5$ and $\domT_10$ are used. #### Baselines * GRU/MLP/LightGBM (LGB)/Graph Attention Networks (GAT) * Multi-task learning (MTL): In multi-task learning, multiple tasks are jointly trained and mutually boosted. Each task is treated equally, while in our setting, we focus on the main task. * Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task T_1, then T_2, and gradually move to the last one. #### Result -| Methods | $$T_1$$ | $$T_2$$ | $$T_3$$ | +| Methods | $T_1$ | $T_2$ | $T_3$ | | :----: | :----: | :----: | :----: | | GRU | 0.049 / 1.903 | 0.018 / 1.972 | 0.014 / 1.989 | | MLP | 0.023 / 1.961 | 0.022 / 1.962 | 0.015 / 1.978 | | LGB | 0.038 / 1.883 | 0.023 / 1.952 | 0.007 / 1.987 | | GAT | 0.052 / 1.898 | 0.024 / 1.954 | 0.015 / 1.973 | -| MTL($$\domT_3$$) | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | -| CL($$\domT_3$$) | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | -| Ours($$\domT_3$$) | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | -| MTL($$\domT_5$$) | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | -| CL($$\domT_5$$) | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | -| Ours($$\domT_5$$) | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | -| MTL($$\domT_{10}$$) | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | -| CL($$\domT_{10}$$) | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | -| Ours($$\domT_{10}$$) | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file +| MTL($\domT_3$) | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | +| CL($\domT_3$) | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | +| Ours($\domT_3$) | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | +| MTL($\domT_5$) | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | +| CL($\domT_5$) | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | +| Ours($\domT_5$) | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | +| MTL($\domT_{10}$) | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | +| CL($\domT_{10}$) | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | +| Ours($\domT_{10}$) | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file From 5a3dde93a893f91c69e54f495a67cebb3f2f36ab Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:15:06 +0800 Subject: [PATCH 23/45] update --- examples/benchmarks/TCTS/TCTS.md | 40 +++++++++++++++----------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index aa707b26c..b0389dc9b 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -4,17 +4,17 @@ We provide the code for reproducing the stock trend forecasting experiments in [ ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -![Temporally Correlated Tasks.](task_description.png) +
![Temporally Correlated Tasks.](task_description.png)
### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. -![The optimization workflow of one episode.](workflow.png) +
![The optimization workflow of one episode.](workflow.png)
-At step $s$, with training data $(x_s,y_s)$, the scheduler $\varphi(\cdots;\omega)$ chooses a suitable task $T_{i_s}$ (green solid lines) to update the model $f(\cdots;\theta)$ (blue solid lines). After $S$ steps, we evaluate the model $f$ on the validation set $\Ddev$ and update the scheduler $\varphi$ (green dashed lines). +At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). ### DataSet * We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. @@ -22,31 +22,29 @@ At step $s$, with training data $(x_s,y_s)$, the scheduler $\varphi(\cdots;\omeg ### Experiments #### Task Description -* The main tasks $T_k$ ($task_k$ in Figure1) refers to forecasting return of stock $i$ as following, +* The main tasks ( in Figure1) refers to forecasting return of stock as following, +
+ +
-```math -$ -r_{i}^k = \frac{\price_i^{t+k}}{\price_i^{t+k-1}} - 1 -$ -``` -* Temporally correlated task sets $\domT_k = {T_1, T_2, ... , T_k}$, in this paper, $\domT_3$, $\domT_5$ and $\domT_10$ are used. +* Temporally correlated task sets , in this paper, , and are used. #### Baselines * GRU/MLP/LightGBM (LGB)/Graph Attention Networks (GAT) * Multi-task learning (MTL): In multi-task learning, multiple tasks are jointly trained and mutually boosted. Each task is treated equally, while in our setting, we focus on the main task. -* Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task T_1, then T_2, and gradually move to the last one. +* Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task , then , and gradually move to the last one. #### Result -| Methods | $T_1$ | $T_2$ | $T_3$ | +| Methods | | | | | :----: | :----: | :----: | :----: | | GRU | 0.049 / 1.903 | 0.018 / 1.972 | 0.014 / 1.989 | | MLP | 0.023 / 1.961 | 0.022 / 1.962 | 0.015 / 1.978 | | LGB | 0.038 / 1.883 | 0.023 / 1.952 | 0.007 / 1.987 | | GAT | 0.052 / 1.898 | 0.024 / 1.954 | 0.015 / 1.973 | -| MTL($\domT_3$) | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | -| CL($\domT_3$) | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | -| Ours($\domT_3$) | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | -| MTL($\domT_5$) | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | -| CL($\domT_5$) | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | -| Ours($\domT_5$) | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | -| MTL($\domT_{10}$) | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | -| CL($\domT_{10}$) | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | -| Ours($\domT_{10}$) | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file +| MTL() | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | +| CL() | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | +| Ours() | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | +| MTL() | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | +| CL() | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | +| Ours() | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | +| MTL() | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | +| CL() | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | +| Ours() | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file From 02d0eedd68e75b437e38c32c3f52022a73a62ccc Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:21:16 +0800 Subject: [PATCH 24/45] update --- examples/benchmarks/TCTS/TCTS.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index b0389dc9b..f500fc155 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -1,17 +1,18 @@ # Temporally Correlated Task Scheduling for Sequence Learning -We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://www.overleaf.com/project/5eb8efb42dcf710001d781d6). +We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_nn.py). ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -
![Temporally Correlated Tasks.](task_description.png)
+ +

![Temporally Correlated Tasks.](task_description.png)

### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. -
![The optimization workflow of one episode.](workflow.png)
+

![The optimization workflow of one episode.](workflow.png)

At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). From 38c7b7303adbb1d5061e5f556870aed24d10e159 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:26:50 +0800 Subject: [PATCH 25/45] dsaf --- examples/benchmarks/TCTS/TCTS.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index f500fc155..d450cfff4 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -5,14 +5,16 @@ We provide the code for reproducing the stock trend forecasting experiments in [ Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -

![Temporally Correlated Tasks.](task_description.png)

+
+ +
### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. -

![The optimization workflow of one episode.](workflow.png)

+
![The optimization workflow of one episode.](workflow.png)
At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). From 4c4e77b11ffe3c6e36f6c3b7c642b5b17efd0329 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:28:31 +0800 Subject: [PATCH 26/45] asdf --- examples/benchmarks/TCTS/TCTS.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index d450cfff4..f3113f56b 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -4,9 +4,12 @@ We provide the code for reproducing the stock trend forecasting experiments in [ ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -
- + +
+ +
+
From bb6c1572cadc111af94e012eae2756aa0980e433 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:29:55 +0800 Subject: [PATCH 27/45] asdf --- examples/benchmarks/TCTS/TCTS.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index f3113f56b..eb644ff4a 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -4,13 +4,14 @@ We provide the code for reproducing the stock trend forecasting experiments in [ ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -
+

-

+

+ +

+ +

-
- -
### Method From 89d53853e5630fa339b3c463f6010d6bce69f953 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:30:42 +0800 Subject: [PATCH 28/45] asdf --- examples/benchmarks/TCTS/TCTS.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index eb644ff4a..eab8199e4 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -8,18 +8,13 @@ Sequence learning has attracted much research attention from the machine learnin

-

- -

- - ### Method Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. - -
![The optimization workflow of one episode.](workflow.png)
- +

+ +

At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). From 073fe4668e1cd970a0b3bd89711012823359b623 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:34:31 +0800 Subject: [PATCH 29/45] asdf --- examples/benchmarks/TCTS/TCTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index eab8199e4..581d34f90 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -5,7 +5,7 @@ We provide the code for reproducing the stock trend forecasting experiments in [ Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other.

- +

@@ -13,7 +13,7 @@ Sequence learning has attracted much research attention from the machine learnin Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2.

- +

At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). From d199256d3489368fc03054987b1f3453ef95a0ba Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:35:14 +0800 Subject: [PATCH 30/45] asdf --- examples/benchmarks/TCTS/TCTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index 581d34f90..66c569734 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -5,7 +5,7 @@ We provide the code for reproducing the stock trend forecasting experiments in [ Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other.

- +

From 65ddca133f3d3ae28fe6051134eb1e850a5290d1 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:36:12 +0800 Subject: [PATCH 31/45] asdf --- examples/benchmarks/TCTS/TCTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index 66c569734..5579c42a8 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -1,5 +1,5 @@ # Temporally Correlated Task Scheduling for Sequence Learning -We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_nn.py). +We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py). ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. From 567e42840c5d1a3bf0565ab016a20ea2bf6f3a6e Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 9 Jun 2021 18:37:25 +0800 Subject: [PATCH 32/45] asdf --- examples/benchmarks/TCTS/TCTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarks/TCTS/TCTS.md b/examples/benchmarks/TCTS/TCTS.md index 5579c42a8..ee67ffbeb 100644 --- a/examples/benchmarks/TCTS/TCTS.md +++ b/examples/benchmarks/TCTS/TCTS.md @@ -1,5 +1,5 @@ # Temporally Correlated Task Scheduling for Sequence Learning -We provide the code for reproducing the stock trend forecasting experiments in [Temporally Correlated Task Scheduling for Sequence Learning](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py). +We provide the [code](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py) for reproducing the stock trend forecasting experiments. ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. From d4b36bdab448859f04b163b89843594660b3323f Mon Sep 17 00:00:00 2001 From: Young Date: Fri, 11 Jun 2021 01:58:04 +0000 Subject: [PATCH 33/45] Online fix - Skip duplicated qlib.auto_init() - Fix TSDatasetH flt_col bug! - Resolve qlib log attribute confliction - Trainer API enhancement - More docs and user-friendly warning --- qlib/__init__.py | 13 ++++++-- qlib/data/dataset/__init__.py | 30 +++++++++++++---- qlib/log.py | 8 +++-- qlib/model/trainer.py | 56 +++++++++++++++++++++----------- qlib/utils/__init__.py | 22 +++++++++++++ qlib/workflow/exp.py | 11 +++++-- qlib/workflow/expm.py | 9 +++++ qlib/workflow/online/manager.py | 15 ++++++--- qlib/workflow/online/strategy.py | 6 ++++ qlib/workflow/task/collect.py | 10 +++++- qlib/workflow/task/gen.py | 4 ++- qlib/workflow/task/manage.py | 10 ++++-- 12 files changed, 150 insertions(+), 44 deletions(-) diff --git a/qlib/__init__.py b/qlib/__init__.py index 4fd48f8c1..5f45f4557 100644 --- a/qlib/__init__.py +++ b/qlib/__init__.py @@ -20,11 +20,17 @@ def init(default_conf="client", **kwargs): from .config import C from .data.cache import H - H.clear() - # FIXME: this logger ignored the level in config logger = get_module_logger("Initialization", level=logging.INFO) + skip_if_reg = kwargs.pop("skip_if_reg", False) + if skip_if_reg and C.registered: + # if we reinitialize Qlib during running an experiment `R.start`. + # it will result in loss of the recorder + logger.warning("Skip initialization because `skip_if_reg is True`") + return + + H.clear() C.set(default_conf, **kwargs) # check path if server/local @@ -197,14 +203,15 @@ def auto_init(**kwargs): - Find the project configuration and init qlib - The parsing process will be affected by the `conf_type` of the configuration file - Init qlib with default config + - Skip initialization if already initialized """ + kwargs["skip_if_reg"] = kwargs.get("skip_if_reg", True) try: pp = get_project_path(cur_path=kwargs.pop("cur_path", None)) except FileNotFoundError: init(**kwargs) else: - conf_pp = pp / "config.yaml" with conf_pp.open() as f: conf = yaml.safe_load(f) diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index 8d7786368..fe641be35 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -1,6 +1,6 @@ from ...utils.serial import Serializable from typing import Union, List, Tuple, Dict, Text, Optional -from ...utils import init_instance_by_config, np_ffill +from ...utils import init_instance_by_config, np_ffill, time_to_slc_point from ...log import get_module_logger from .handler import DataHandler, DataHandlerLP from copy import deepcopy @@ -243,6 +243,8 @@ class TSDataSampler: It works like `torch.data.utils.Dataset`, it provides a very convenient interface for constructing time-series dataset based on tabular data. + - On time step dimension, the smaller index indicates the historical data and the larger index indicates the future + data. If user have further requirements for processing data, user could process them based on `TSDataSampler` or create more powerful subclasses. @@ -309,11 +311,19 @@ class TSDataSampler: self.data_index = deepcopy(self.data.index) if flt_data is not None: - self.flt_data = np.array(flt_data.reindex(self.data_index)).reshape(-1) + if isinstance(flt_data, pd.DataFrame): + assert len(flt_data.columns) == 1 + flt_data = flt_data.iloc[:, 0] + # NOTE: bool(np.nan) is True !!!!!!!! + # make sure reindex comes first. Otherwise extra NaN may appear. + flt_data = flt_data.reindex(self.data_index).fillna(False).astype(np.bool) + self.flt_data = flt_data.values self.idx_map = self.flt_idx_map(self.flt_data, self.idx_map) self.data_index = self.data_index[np.where(self.flt_data == True)[0]] - self.start_idx, self.end_idx = self.data_index.slice_locs(start=pd.Timestamp(start), end=pd.Timestamp(end)) + self.start_idx, self.end_idx = self.data_index.slice_locs( + start=time_to_slc_point(start), end=time_to_slc_point(end) + ) self.idx_arr = np.array(self.idx_df.values, dtype=np.float64) # for better performance del self.data # save memory @@ -341,7 +351,7 @@ class TSDataSampler: setattr(self, k, v) @staticmethod - def build_index(data: pd.DataFrame) -> dict: + def build_index(data: pd.DataFrame) -> Tuple[pd.DataFrame, dict]: """ The relation of the data @@ -352,9 +362,15 @@ class TSDataSampler: Returns ------- - dict: - {: } - # get the previous index of a line given index + Tuple[pd.DataFrame, dict]: + 1) the first element: reshape the original index into a 2D dataframe + instrument SH600000 SH600004 SH600006 SH600007 SH600008 SH600009 ... + datetime + 2021-01-11 0 1 2 3 4 5 ... + 2021-01-12 4146 4147 4148 4149 4150 4151 ... + 2021-01-13 8293 8294 8295 8296 8297 8298 ... + 2021-01-14 12441 12442 12443 12444 12445 12446 ... + 2) the second element: {: } """ # object incase of pandas converting int to flaot idx_df = pd.Series(range(data.shape[0]), index=data.index, dtype=object) diff --git a/qlib/log.py b/qlib/log.py index 379544392..ad55e2200 100644 --- a/qlib/log.py +++ b/qlib/log.py @@ -28,16 +28,18 @@ class QlibLogger(metaclass=MetaLogger): def __init__(self, module_name): self.module_name = module_name - self.level = 0 + # this feature name conflicts with the attribute with Logger + # rename it to avoid some corner cases that result in comparing `str` and `int` + self.__level = 0 @property def logger(self): logger = logging.getLogger(self.module_name) - logger.setLevel(self.level) + logger.setLevel(self.__level) return logger def setLevel(self, level): - self.level = level + self.__level = level def __getattr__(self, name): # During unpickling, python will call __getattr__. Use this line to avoid maximum recursion error. diff --git a/qlib/model/trainer.py b/qlib/model/trainer.py index 28d854477..a534a7a3b 100644 --- a/qlib/model/trainer.py +++ b/qlib/model/trainer.py @@ -8,7 +8,7 @@ There are two steps in each Trainer including ``train``(make model recorder) and This is a concept called ``DelayTrainer``, which can be used in online simulating for parallel training. In ``DelayTrainer``, the first step is only to save some necessary info to model recorders, and the second step which will be finished in the end can do some concurrent and time-consuming operations such as model fitting. -``Qlib`` offer two kinds of Trainer, ``TrainerR`` is the simplest way and ``TrainerRM`` is based on TaskManager to help manager tasks lifecycle automatically. +``Qlib`` offer two kinds of Trainer, ``TrainerR`` is the simplest way and ``TrainerRM`` is based on TaskManager to help manager tasks lifecycle automatically. """ import socket @@ -153,6 +153,9 @@ class Trainer: """ return self.delay + def __call__(self, *args, **kwargs) -> list: + return self.end_train(self.train(*args, **kwargs)) + class TrainerR(Trainer): """ @@ -286,7 +289,9 @@ class TrainerRM(Trainer): # This tag is the _id in TaskManager to distinguish tasks. TM_ID = "_id in TaskManager" - def __init__(self, experiment_name: str = None, task_pool: str = None, train_func=task_train): + def __init__( + self, experiment_name: str = None, task_pool: str = None, train_func=task_train, skip_run_task: bool = False + ): """ Init TrainerR. @@ -294,11 +299,16 @@ class TrainerRM(Trainer): experiment_name (str): the default name of experiment. task_pool (str): task pool name in TaskManager. None for use same name as experiment_name. train_func (Callable, optional): default training method. Defaults to `task_train`. + skip_run_task (bool): + If skip_run_task == True: + Only run_task in the worker. Otherwise skip run_task. """ + super().__init__() self.experiment_name = experiment_name self.task_pool = task_pool self.train_func = train_func + self.skip_run_task = skip_run_task def train( self, @@ -340,15 +350,16 @@ class TrainerRM(Trainer): tm = TaskManager(task_pool=task_pool) _id_list = tm.create_task(tasks) # all tasks will be saved to MongoDB query = {"_id": {"$in": _id_list}} - run_task( - train_func, - task_pool, - query=query, # only train these tasks - experiment_name=experiment_name, - before_status=before_status, - after_status=after_status, - **kwargs, - ) + if not self.skip_run_task: + run_task( + train_func, + task_pool, + query=query, # only train these tasks + experiment_name=experiment_name, + before_status=before_status, + after_status=after_status, + **kwargs, + ) if not self.is_delay(): tm.wait(query=query) @@ -411,6 +422,7 @@ class DelayTrainerRM(TrainerRM): task_pool: str = None, train_func=begin_task_train, end_train_func=end_task_train, + skip_run_task: bool = False, ): """ Init DelayTrainerRM. @@ -420,10 +432,15 @@ class DelayTrainerRM(TrainerRM): task_pool (str): task pool name in TaskManager. None for use same name as experiment_name. train_func (Callable, optional): default train method. Defaults to `begin_task_train`. end_train_func (Callable, optional): default end_train method. Defaults to `end_task_train`. + skip_run_task (bool): + If skip_run_task == True: + Only run_task in the worker. Otherwise skip run_task. + E.g. Starting trainer on a CPU VM and then waiting tasks to be finished on GPU VMs. """ super().__init__(experiment_name, task_pool, train_func) self.end_train_func = end_train_func self.delay = True + self.skip_run_task = skip_run_task def train(self, tasks: list, train_func=None, experiment_name: str = None, **kwargs) -> List[Recorder]: """ @@ -477,14 +494,15 @@ class DelayTrainerRM(TrainerRM): _id_list.append(rec.list_tags()[self.TM_ID]) query = {"_id": {"$in": _id_list}} - run_task( - end_train_func, - task_pool, - query=query, # only train these tasks - experiment_name=experiment_name, - before_status=TaskManager.STATUS_PART_DONE, - **kwargs, - ) + if not self.skip_run_task: + run_task( + end_train_func, + task_pool, + query=query, # only train these tasks + experiment_name=experiment_name, + before_status=TaskManager.STATUS_PART_DONE, + **kwargs, + ) TaskManager(task_pool=task_pool).wait(query=query) diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 1e8ee2e48..778d0e17a 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -642,6 +642,28 @@ def split_pred(pred, number=None, split_date=None): return pred_left, pred_right +def time_to_slc_point(t: Union[None, str, pd.Timestamp]) -> Union[None, pd.Timestamp]: + """ + Time slicing in Qlib or Pandas is a frequently-used action. + However, user often input all kinds of data format to represent time. + This function will help user to convert these inputs into a uniform format which is friendly to time slicing. + + Parameters + ---------- + t : Union[None, str, pd.Timestamp] + original time + + Returns + ------- + Union[None, pd.Timestamp]: + """ + if t is None: + # None represents unbounded in Qlib or Pandas(e.g. df.loc[slice(None, "20210303")]). + return t + else: + return pd.Timestamp(t) + + def can_use_cache(): res = True r = get_redis_connection() diff --git a/qlib/workflow/exp.py b/qlib/workflow/exp.py index 467c7c3f4..08f429eb3 100644 --- a/qlib/workflow/exp.py +++ b/qlib/workflow/exp.py @@ -213,11 +213,15 @@ class Experiment: """ raise NotImplementedError(f"Please implement the `_get_recorder` method") - def list_recorders(self): + def list_recorders(self, **flt_kwargs): """ List all the existing recorders of this experiment. Please first get the experiment instance before calling this method. If user want to use the method `R.list_recorders()`, please refer to the related API document in `QlibRecorder`. + flt_kwargs : dict + filter recorders by conditions + e.g. list_recorders(status=Recorder.STATUS_FI) + Returns ------- A dictionary (id -> recorder) of recorder information that being stored. @@ -320,11 +324,14 @@ class MLflowExperiment(Experiment): UNLIMITED = 50000 # FIXME: Mlflow can only list 50000 records at most!!!!!!! - def list_recorders(self, max_results=UNLIMITED): + def list_recorders(self, max_results=UNLIMITED, status=None): runs = self._client.search_runs(self.id, run_view_type=ViewType.ACTIVE_ONLY, max_results=max_results) recorders = dict() for i in range(len(runs)): recorder = MLflowRecorder(self.id, self._uri, mlflow_run=runs[i]) + if status is not None: + if recorder.status != status: + continue recorders[runs[i].info.run_id] = recorder return recorders diff --git a/qlib/workflow/expm.py b/qlib/workflow/expm.py index 04cc3bcb7..751459d81 100644 --- a/qlib/workflow/expm.py +++ b/qlib/workflow/expm.py @@ -351,6 +351,15 @@ class MLflowExpManager(ExpManager): experiment_id is not None or experiment_name is not None ), "Please input at least one of experiment/recorder id or name before retrieving experiment/recorder." if experiment_id is not None: + try: + experiment_id = int(experiment_id) + except ValueError as e: + msg = "The `experiment_id` for mlflow backend must be `int`" + logger.error(msg) + # We have to raise type error here + # - The error looks like type error + # - Value Error will be catched + raise TypeError(msg) try: exp = self.client.get_experiment(experiment_id) if exp.lifecycle_stage.upper() == "DELETED": diff --git a/qlib/workflow/online/manager.py b/qlib/workflow/online/manager.py index d3cc0cbf8..b4b509483 100644 --- a/qlib/workflow/online/manager.py +++ b/qlib/workflow/online/manager.py @@ -6,7 +6,7 @@ OnlineManager can manage a set of `Online Strategy <#Online Strategy>`_ and run With the change of time, the decisive models will be also changed. In this module, we call those contributing models `online` models. In every routine(such as every day or every minute), the `online` models may be changed and the prediction of them needs to be updated. -So this module provides a series of methods to control this process. +So this module provides a series of methods to control this process. This module also provides a method to simulate `Online Strategy <#Online Strategy>`_ in history. Which means you can verify your strategy or find a better one. @@ -31,7 +31,7 @@ Simulation + Trainer When your models have some temporal dependence on the Simulation + DelayTrainer When your models don't have any temporal dependence, you can use DelayTrainer for the ability to multitasking. It means all tasks in all routines - can be REAL trained at the end of simulating. The signals will be prepared well at + can be REAL trained at the end of simulating. The signals will be prepared well at different time segments (based on whether or not any new model is online). ========================= =================================================================================== """ @@ -113,6 +113,8 @@ class OnlineManager(Serializable): models = self.trainer.train(tasks, experiment_name=strategy.name_id) models_list.append(models) self.logger.info(f"Finished training {len(models)} models.") + # FIXME: Traing multiple online models at `first_train` will result in getting too much online models at the + # start. online_models = strategy.prepare_online_models(models, **model_kwargs) self.history.setdefault(self.cur_time, {})[strategy] = online_models @@ -148,8 +150,6 @@ class OnlineManager(Serializable): models_list = [] for strategy in self.strategies: self.logger.info(f"Strategy `{strategy.name_id}` begins routine...") - if self.status == self.STATUS_NORMAL: - strategy.tool.update_online_pred() tasks = strategy.prepare_tasks(self.cur_time, **task_kwargs) models = self.trainer.train(tasks, experiment_name=strategy.name_id) @@ -158,6 +158,11 @@ class OnlineManager(Serializable): online_models = strategy.prepare_online_models(models, **model_kwargs) self.history.setdefault(self.cur_time, {})[strategy] = online_models + # The online model may changes in the above processes + # So updating the predictions of online models should be the last step + if self.status == self.STATUS_NORMAL: + strategy.tool.update_online_pred() + if not self.status == self.STATUS_SIMULATING or not self.trainer.is_delay(): for strategy, models in zip(self.strategies, models_list): models = self.trainer.end_train(models, experiment_name=strategy.name_id) @@ -236,7 +241,7 @@ class OnlineManager(Serializable): SIM_LOG_NAME = "SIMULATE_INFO" def simulate( - self, end_time, frequency="day", task_kwargs={}, model_kwargs={}, signal_kwargs={} + self, end_time=None, frequency="day", task_kwargs={}, model_kwargs={}, signal_kwargs={} ) -> Union[pd.Series, pd.DataFrame]: """ Starting from the current time, this method will simulate every routine in OnlineManager until the end time. diff --git a/qlib/workflow/online/strategy.py b/qlib/workflow/online/strategy.py index a54eb32bf..1e8e85c0f 100644 --- a/qlib/workflow/online/strategy.py +++ b/qlib/workflow/online/strategy.py @@ -52,6 +52,12 @@ class OnlineStrategy: NOTE: Reset all online models to trained models. If there are no trained models, then do nothing. + **NOTE**: + Current implementation is very naive. Here is a more complex situation which is more closer to the + practical scenarios. + 1. Train new models at the day before `test_start` (at time stamp `T`) + 2. Switch models at the `test_start` (at time timestamp `T + 1` typically) + Args: models (list): a list of models. cur_time (pd.Dataframe): current time from OnlineManger. None for the latest. diff --git a/qlib/workflow/task/collect.py b/qlib/workflow/task/collect.py index 9410c2b9c..36ccf434d 100644 --- a/qlib/workflow/task/collect.py +++ b/qlib/workflow/task/collect.py @@ -6,6 +6,7 @@ Collector module can collect objects from everywhere and process them such as me """ from typing import Callable, Dict, List +from qlib.log import get_module_logger from qlib.utils.serial import Serializable from qlib.workflow import R @@ -192,6 +193,7 @@ class RecorderCollector(Collector): if rec_filter_func is None or rec_filter_func(rec): recs_flt[rid] = rec + logger = get_module_logger("RecorderCollector") for _, rec in recs_flt.items(): rec_key = self.rec_key_func(rec) for key in artifacts_key: @@ -205,7 +207,13 @@ class RecorderCollector(Collector): # only collect existing artifact continue raise e - collect_dict.setdefault(key, {})[rec_key] = artifact + # give user some warning if the values are overridden + cdd = collect_dict.setdefault(key, {}) + if rec_key in cdd: + logger.warning( + f"key '{rec_key}' is duplicated. Previous value will be overrides. Please check you `rec_key_func`" + ) + cdd[rec_key] = artifact return collect_dict diff --git a/qlib/workflow/task/gen.py b/qlib/workflow/task/gen.py index cdebf5049..ca7b8ae7f 100644 --- a/qlib/workflow/task/gen.py +++ b/qlib/workflow/task/gen.py @@ -6,6 +6,8 @@ TaskGenerator module can generate many tasks based on TaskGen and some task temp import abc import copy from typing import List, Union, Callable + +from qlib.utils import transform_end_date from .utils import TimeAdjuster @@ -199,7 +201,7 @@ class RollingGen(TaskGen): # First rolling # 1) prepare the end point segments: dict = copy.deepcopy(self.ta.align_seg(t["dataset"]["kwargs"]["segments"])) - test_end = self.ta.max() if segments[self.test_key][1] is None else segments[self.test_key][1] + test_end = transform_end_date(segments[self.test_key][1]) # 2) and init test segments test_start_idx = self.ta.align_idx(segments[self.test_key][0]) segments[self.test_key] = (self.ta.get(test_start_idx), self.ta.get(test_start_idx + self.step - 1)) diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 7a85036da..01f79b1b4 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -275,7 +275,7 @@ class TaskManager: except Exception: if task is not None: self.logger.info("Returning task before raising error") - self.return_task(task) + self.return_task(task, status=status) # return task as the original status self.logger.info("Task returned") raise @@ -411,7 +411,11 @@ class TaskManager: self.task_pool.update_one({"_id": task["_id"]}, update_dict) def _get_undone_n(self, task_stat): - return task_stat.get(self.STATUS_WAITING, 0) + task_stat.get(self.STATUS_RUNNING, 0) + return ( + task_stat.get(self.STATUS_WAITING, 0) + + task_stat.get(self.STATUS_RUNNING, 0) + + task_stat.get(self.STATUS_PART_DONE, 0) + ) def _get_total(self, task_stat): return sum(task_stat.values()) @@ -429,7 +433,7 @@ class TaskManager: last_undone_n = self._get_undone_n(task_stat) if last_undone_n == 0: return - self.logger.warn(f"Waiting for {last_undone_n} undone tasks. Please make sure they are running.") + self.logger.warning(f"Waiting for {last_undone_n} undone tasks. Please make sure they are running.") with tqdm(total=total, initial=total - last_undone_n) as pbar: while True: time.sleep(10) From 5850490b245a78279adbb9dd10aeeec0d3a17f21 Mon Sep 17 00:00:00 2001 From: Young Date: Fri, 11 Jun 2021 08:23:57 +0000 Subject: [PATCH 34/45] simplify the code and add docs --- qlib/workflow/exp.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/qlib/workflow/exp.py b/qlib/workflow/exp.py index 08f429eb3..627b5ff82 100644 --- a/qlib/workflow/exp.py +++ b/qlib/workflow/exp.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +from typing import Union import mlflow, logging from mlflow.entities import ViewType from mlflow.exceptions import MlflowException @@ -324,14 +325,21 @@ class MLflowExperiment(Experiment): UNLIMITED = 50000 # FIXME: Mlflow can only list 50000 records at most!!!!!!! - def list_recorders(self, max_results=UNLIMITED, status=None): + def list_recorders(self, max_results: int = UNLIMITED, status: Union[str, None] = None): + """ + Parameters + ---------- + max_results : int + the number limitation of the results + status : str + the criteria based on status to filter results. + `None` indicates no filtering. + """ runs = self._client.search_runs(self.id, run_view_type=ViewType.ACTIVE_ONLY, max_results=max_results) recorders = dict() for i in range(len(runs)): recorder = MLflowRecorder(self.id, self._uri, mlflow_run=runs[i]) - if status is not None: - if recorder.status != status: - continue - recorders[runs[i].info.run_id] = recorder + if status is None or recorder.status == status: + recorders[runs[i].info.run_id] = recorder return recorders From 730f6258d6ec6b88ed3a8e42d2f8d70b3ddc12b7 Mon Sep 17 00:00:00 2001 From: Young Date: Fri, 11 Jun 2021 10:40:56 +0000 Subject: [PATCH 35/45] add warning and * --- qlib/workflow/__init__.py | 10 ++++++---- qlib/workflow/expm.py | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/qlib/workflow/__init__.py b/qlib/workflow/__init__.py index 2b2535edc..63f63fb56 100644 --- a/qlib/workflow/__init__.py +++ b/qlib/workflow/__init__.py @@ -215,9 +215,9 @@ class QlibRecorder: ------- A dictionary (id -> recorder) of recorder information that being stored. """ - return self.get_exp(experiment_id, experiment_name).list_recorders() + return self.get_exp(experiment_id=experiment_id, experiment_name=experiment_name).list_recorders() - def get_exp(self, experiment_id=None, experiment_name=None, create: bool = True) -> Experiment: + def get_exp(self, *, experiment_id=None, experiment_name=None, create: bool = True) -> Experiment: """ Method for retrieving an experiment with given id or name. Once the `create` argument is set to True, if no valid experiment is found, this method will create one for you. Otherwise, it will @@ -262,7 +262,7 @@ class QlibRecorder: # Case 2 with R.start('test'): - exp = R.get_exp('test1') + exp = R.get_exp(experiment_name='test1') # Case 3 exp = R.get_exp() -> a default experiment. @@ -287,7 +287,9 @@ class QlibRecorder: ------- An experiment instance with given id or name. """ - return self.exp_manager.get_exp(experiment_id, experiment_name, create, start=False) + return self.exp_manager.get_exp( + experiment_id=experiment_id, experiment_name=experiment_name, create=create, start=False + ) def delete_exp(self, experiment_id=None, experiment_name=None): """ diff --git a/qlib/workflow/expm.py b/qlib/workflow/expm.py index 751459d81..7e39d3a32 100644 --- a/qlib/workflow/expm.py +++ b/qlib/workflow/expm.py @@ -109,7 +109,7 @@ class ExpManager: """ raise NotImplementedError(f"Please implement the `search_records` method.") - def get_exp(self, experiment_id=None, experiment_name=None, create: bool = True, start: bool = False): + def get_exp(self, *, experiment_id=None, experiment_name=None, create: bool = True, start: bool = False): """ Retrieve an experiment. This method includes getting an active experiment, and get_or_create a specific experiment. @@ -190,7 +190,7 @@ class ExpManager: except ValueError: if experiment_name is None: experiment_name = self._default_exp_name - logger.info(f"No valid experiment found. Create a new experiment with name {experiment_name}.") + logger.warning(f"No valid experiment found. Create a new experiment with name {experiment_name}.") return self.create_exp(experiment_name), True def _get_exp(self, experiment_id=None, experiment_name=None) -> Experiment: From 973c4137e442a2ad2a73fd94230a1c7abea75733 Mon Sep 17 00:00:00 2001 From: Young Date: Sat, 12 Jun 2021 13:54:26 +0000 Subject: [PATCH 36/45] fix mlflow & task bug --- qlib/model/base.py | 2 +- qlib/workflow/__init__.py | 6 ++++-- qlib/workflow/expm.py | 11 ++--------- qlib/workflow/task/manage.py | 2 +- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/qlib/model/base.py b/qlib/model/base.py index 12caf5f73..493981133 100644 --- a/qlib/model/base.py +++ b/qlib/model/base.py @@ -97,7 +97,7 @@ class ModelFT(Model): # Finetune model based on previous trained model with R.start(experiment_name="finetune model"): - recorder = R.get_recorder(rid, experiment_name="init models") + recorder = R.get_recorder(recorder_id=rid, experiment_name="init models") model = recorder.load_object("init_model") model.finetune(dataset, num_boost_round=10) diff --git a/qlib/workflow/__init__.py b/qlib/workflow/__init__.py index 63f63fb56..a14f60c01 100644 --- a/qlib/workflow/__init__.py +++ b/qlib/workflow/__init__.py @@ -333,7 +333,9 @@ class QlibRecorder: """ self.exp_manager.set_uri(uri) - def get_recorder(self, recorder_id=None, recorder_name=None, experiment_name=None) -> Recorder: + def get_recorder( + self, *, recorder_id=None, recorder_name=None, experiment_id=None, experiment_name=None + ) -> Recorder: """ Method for retrieving a recorder. @@ -386,7 +388,7 @@ class QlibRecorder: ------- A recorder instance. """ - return self.get_exp(experiment_name=experiment_name, create=False).get_recorder( + return self.get_exp(experiment_name=experiment_name, experiment_id=experiment_id, create=False).get_recorder( recorder_id, recorder_name, create=False, start=False ) diff --git a/qlib/workflow/expm.py b/qlib/workflow/expm.py index 7e39d3a32..84cc6a13a 100644 --- a/qlib/workflow/expm.py +++ b/qlib/workflow/expm.py @@ -352,15 +352,8 @@ class MLflowExpManager(ExpManager): ), "Please input at least one of experiment/recorder id or name before retrieving experiment/recorder." if experiment_id is not None: try: - experiment_id = int(experiment_id) - except ValueError as e: - msg = "The `experiment_id` for mlflow backend must be `int`" - logger.error(msg) - # We have to raise type error here - # - The error looks like type error - # - Value Error will be catched - raise TypeError(msg) - try: + # NOTE: the mlflow's experiment_id must be str type... + # https://www.mlflow.org/docs/latest/python_api/mlflow.tracking.html#mlflow.tracking.MlflowClient.get_experiment exp = self.client.get_experiment(experiment_id) if exp.lifecycle_stage.upper() == "DELETED": raise MlflowException("No valid experiment has been found.") diff --git a/qlib/workflow/task/manage.py b/qlib/workflow/task/manage.py index 01f79b1b4..41e243b43 100644 --- a/qlib/workflow/task/manage.py +++ b/qlib/workflow/task/manage.py @@ -272,7 +272,7 @@ class TaskManager: task = self.fetch_task(query=query, status=status) try: yield task - except Exception: + except (Exception, KeyboardInterrupt): # KeyboardInterrupt is not a subclass of Exception if task is not None: self.logger.info("Returning task before raising error") self.return_task(task, status=status) # return task as the original status From 9e0e2ff7362989b2267702d20dfc013bc753f78b Mon Sep 17 00:00:00 2001 From: Jactus Date: Tue, 15 Jun 2021 14:46:31 +0800 Subject: [PATCH 37/45] Update QlibRecorder wrapper --- qlib/workflow/__init__.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/qlib/workflow/__init__.py b/qlib/workflow/__init__.py index 2b2535edc..7652ff709 100644 --- a/qlib/workflow/__init__.py +++ b/qlib/workflow/__init__.py @@ -525,14 +525,33 @@ class QlibRecorder: self.get_exp().get_recorder().set_tags(**kwargs) +# error type for reinitialization when starting an experiment +class RecorderInitializationError(Exception): + def __init__(self, message): + super(RecorderInitializationError, self).__init__(message) + + +class RecorderWrapper(Wrapper): + """ + Wrapper class for QlibRecorder, which detects whether users reinitialize qlib when already starting an experiment. + """ + + def register(self, provider): + if self._provider is not None: + raise RecorderInitializationError( + "Please don't reinitialize Qlib if QlibRecorder is already acivated. Otherwise, the experiment stored location will be modified." + ) + self._provider = provider + + import sys if sys.version_info >= (3, 9): from typing import Annotated - QlibRecorderWrapper = Annotated[QlibRecorder, Wrapper] + QlibRecorderWrapper = Annotated[QlibRecorder, RecorderWrapper] else: QlibRecorderWrapper = QlibRecorder # global record -R: QlibRecorderWrapper = Wrapper() +R: QlibRecorderWrapper = RecorderWrapper() From 64582e9d4622a7f1f6b1a16a67099e289c03e1b9 Mon Sep 17 00:00:00 2001 From: Jactus Date: Tue, 15 Jun 2021 15:02:11 +0800 Subject: [PATCH 38/45] Add QlibException --- qlib/utils/exceptions.py | 12 ++++++++++++ qlib/workflow/__init__.py | 7 +------ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 qlib/utils/exceptions.py diff --git a/qlib/utils/exceptions.py b/qlib/utils/exceptions.py new file mode 100644 index 000000000..69712172b --- /dev/null +++ b/qlib/utils/exceptions.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Base exception class +class QlibException(Exception): + def __init__(self, message): + super(QlibException, self).__init__(message) + + +# Error type for reinitialization when starting an experiment +class RecorderInitializationError(QlibException): + pass diff --git a/qlib/workflow/__init__.py b/qlib/workflow/__init__.py index 7652ff709..e5cdbb71c 100644 --- a/qlib/workflow/__init__.py +++ b/qlib/workflow/__init__.py @@ -7,6 +7,7 @@ from .expm import MLflowExpManager from .exp import Experiment from .recorder import Recorder from ..utils import Wrapper +from ..utils.exceptions import RecorderInitializationError class QlibRecorder: @@ -525,12 +526,6 @@ class QlibRecorder: self.get_exp().get_recorder().set_tags(**kwargs) -# error type for reinitialization when starting an experiment -class RecorderInitializationError(Exception): - def __init__(self, message): - super(RecorderInitializationError, self).__init__(message) - - class RecorderWrapper(Wrapper): """ Wrapper class for QlibRecorder, which detects whether users reinitialize qlib when already starting an experiment. From 5331ab93f883a10f951a07c3f6d87ad33d88e2e5 Mon Sep 17 00:00:00 2001 From: lewwang Date: Wed, 16 Jun 2021 12:18:16 +0800 Subject: [PATCH 39/45] Update TCTS README. --- examples/benchmarks/TCTS/README.md | 52 ++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 examples/benchmarks/TCTS/README.md diff --git a/examples/benchmarks/TCTS/README.md b/examples/benchmarks/TCTS/README.md new file mode 100644 index 000000000..ee67ffbeb --- /dev/null +++ b/examples/benchmarks/TCTS/README.md @@ -0,0 +1,52 @@ +# Temporally Correlated Task Scheduling for Sequence Learning +We provide the [code](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py) for reproducing the stock trend forecasting experiments. + +### Background +Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. + +

+ +

+ + +### Method +Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. + +

+ +

+ +At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). + +### DataSet +* We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. +* We split the data into training (01/01/2008-12/31/2013), validation (01/01/2014-12/31/2015), and test sets (01/01/2016-08/01/2020) based on the transaction time. + +### Experiments +#### Task Description +* The main tasks ( in Figure1) refers to forecasting return of stock as following, +
+ +
+ +* Temporally correlated task sets , in this paper, , and are used. +#### Baselines +* GRU/MLP/LightGBM (LGB)/Graph Attention Networks (GAT) +* Multi-task learning (MTL): In multi-task learning, multiple tasks are jointly trained and mutually boosted. Each task is treated equally, while in our setting, we focus on the main task. +* Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task , then , and gradually move to the last one. +#### Result +| Methods | | | | +| :----: | :----: | :----: | :----: | +| GRU | 0.049 / 1.903 | 0.018 / 1.972 | 0.014 / 1.989 | +| MLP | 0.023 / 1.961 | 0.022 / 1.962 | 0.015 / 1.978 | +| LGB | 0.038 / 1.883 | 0.023 / 1.952 | 0.007 / 1.987 | +| GAT | 0.052 / 1.898 | 0.024 / 1.954 | 0.015 / 1.973 | +| MTL() | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | +| CL() | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | +| Ours() | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | +| MTL() | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | +| CL() | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | +| Ours() | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | +| MTL() | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | +| CL() | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | +| Ours() | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file From 0fe8b281ba7d4e1269d8d273196a853a1aa686a2 Mon Sep 17 00:00:00 2001 From: Jactus Date: Wed, 16 Jun 2021 12:28:20 +0800 Subject: [PATCH 40/45] Update R wrapper logic --- qlib/workflow/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/qlib/workflow/__init__.py b/qlib/workflow/__init__.py index e5cdbb71c..98b2c9925 100644 --- a/qlib/workflow/__init__.py +++ b/qlib/workflow/__init__.py @@ -533,9 +533,11 @@ class RecorderWrapper(Wrapper): def register(self, provider): if self._provider is not None: - raise RecorderInitializationError( - "Please don't reinitialize Qlib if QlibRecorder is already acivated. Otherwise, the experiment stored location will be modified." - ) + expm = getattr(self._provider, "exp_manager") + if expm.active_experiment is not None: + raise RecorderInitializationError( + "Please don't reinitialize Qlib if QlibRecorder is already acivated. Otherwise, the experiment stored location will be modified." + ) self._provider = provider From 9c8d423a86ec4c2ef306e6ac80fb67ba340cb4dc Mon Sep 17 00:00:00 2001 From: Young Date: Wed, 16 Jun 2021 14:10:51 +0000 Subject: [PATCH 41/45] fix ModelUpdater --- qlib/workflow/online/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qlib/workflow/online/update.py b/qlib/workflow/online/update.py index 561f7e18a..96cbf4d65 100644 --- a/qlib/workflow/online/update.py +++ b/qlib/workflow/online/update.py @@ -136,7 +136,7 @@ class PredUpdater(RecordUpdater): # https://github.com/pytorch/pytorch/issues/16797 start_time = get_date_by_shift(self.last_end, 1, freq=self.freq) - if start_time >= self.to_date: + if start_time > self.to_date: self.logger.info( f"The prediction in {self.record.info['id']} are latest ({start_time}). No need to update to {self.to_date}." ) From a3679e67586b312eedc21b120aed58b65f66859f Mon Sep 17 00:00:00 2001 From: Young Date: Sun, 20 Jun 2021 08:03:00 +0000 Subject: [PATCH 42/45] simplify the code and prevent float when shifting --- qlib/data/cache.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qlib/data/cache.py b/qlib/data/cache.py index 2d0b8a7cd..3e0680e74 100644 --- a/qlib/data/cache.py +++ b/qlib/data/cache.py @@ -237,7 +237,7 @@ class CacheUtils: lock.acquire() except redis_lock.AlreadyAcquired: raise QlibCacheException( - f"""It sees the key(lock:{repr(lock_name)[1:-1]}-wlock) of the redis lock has existed in your redis db now. + f"""It sees the key(lock:{repr(lock_name)[1:-1]}-wlock) of the redis lock has existed in your redis db now. You can use the following command to clear your redis keys and rerun your commands: $ redis-cli > select {C.redis_task_db} @@ -784,10 +784,10 @@ class DiskDatasetCache(DatasetCache): def build_index_from_data(data, start_index=0): if data.empty: return pd.DataFrame() - line_data = data.iloc[:, 0].fillna(0).groupby("datetime").count() + line_data = data.groupby("datetime").size() line_data.sort_index(inplace=True) index_end = line_data.cumsum() - index_start = index_end.shift(1).fillna(0) + index_start = index_end.shift(1, fill_value=0) index_data = pd.DataFrame() index_data["start"] = index_start From d0f54343c7eed7427103c689595a0c151c62006d Mon Sep 17 00:00:00 2001 From: Young Date: Sun, 20 Jun 2021 12:00:24 +0000 Subject: [PATCH 43/45] support subclass of TSDatasetH --- qlib/data/dataset/__init__.py | 4 +++- qlib/workflow/online/utils.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/qlib/data/dataset/__init__.py b/qlib/data/dataset/__init__.py index fe641be35..2acaa77fe 100644 --- a/qlib/data/dataset/__init__.py +++ b/qlib/data/dataset/__init__.py @@ -507,7 +507,9 @@ class TSDatasetH(DatasetH): - The dimension of a batch of data """ - def __init__(self, step_len=30, **kwargs): + DEFAULT_STEP_LEN = 30 + + def __init__(self, step_len=DEFAULT_STEP_LEN, **kwargs): self.step_len = step_len super().__init__(**kwargs) diff --git a/qlib/workflow/online/utils.py b/qlib/workflow/online/utils.py index f3ef13aa9..f5e3a2bd0 100644 --- a/qlib/workflow/online/utils.py +++ b/qlib/workflow/online/utils.py @@ -8,8 +8,10 @@ This allows us to use efficient submodels as the market-style changing. """ from typing import List, Union +from qlib.data.dataset import TSDatasetH from qlib.log import get_module_logger +from qlib.utils import get_cls_kwargs from qlib.workflow.online.update import PredUpdater from qlib.workflow.recorder import Recorder from qlib.workflow.task.utils import list_recorders @@ -161,8 +163,9 @@ class OnlineToolR(OnlineTool): hist_ref = 0 task = rec.load_object("task") # Special treatment of historical dependencies - if task["dataset"]["class"] == "TSDatasetH": - hist_ref = task["dataset"]["kwargs"]["step_len"] + cls, kwargs = get_cls_kwargs(task["dataset"], default_module="qlib.data.dataset") + if issubclass(cls, TSDatasetH): + hist_ref = kwargs.get("step_len", TSDatasetH.DEFAULT_STEP_LEN) PredUpdater(rec, to_date=to_date, hist_ref=hist_ref).update() self.logger.info(f"Finished updating {len(online_models)} online model predictions of {self.exp_name}.") From 21eb71d4a9313745badc572222497bd1b6949c23 Mon Sep 17 00:00:00 2001 From: Young Date: Wed, 23 Jun 2021 02:05:38 +0000 Subject: [PATCH 44/45] update framework for online serving --- docs/_static/img/framework.png | Bin 277491 -> 213555 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/_static/img/framework.png b/docs/_static/img/framework.png index d8242f7c192081aa1ebe3f3c266fbb76173ba7df..c3db3235b5409958b1d302da1f2471b821d6562c 100644 GIT binary patch literal 213555 zcmagF1ys}h|3186qeGDpkWxUpyN0NA2nYzHq+@iK0~Az3L{dOd=@>P-Mt4XJMz?^3 zgw%~b~P+%xUSdry54VewN=PT7)SsB0J)m#Lp=Zh5(WT( zsEDBWPq2EQ=kYHN76@fTH9257^^Y zU;_aB$yIx(WZ-Z916XC^39aXbMRgdxOJlWYM~)3uyzqqdQ}6PG^=n;sxc`v zj4o|dEj4)?)eRA?0 zDiOyTTl0aQ-y8VZf*OIOSq}^MIC3~QZv=%rj`I)rFOO}h9Ko-{|9twN=fajcF5~)! zV}q(Ve}(wo-%d;U&j;(L??~l;+0b1=Oc~bP^v_eNV-a$a6dZ7&A?j6f76lfEUDDv=NY^WGd<#n**wVY6hB ze|Bm&A+lQ5X5iz9&C&L*o<*~k1ZtXRCk64op?|2-b&IpmOi$36)-7?8RI<0IQl?PA zjEh%&s{gRQT2DME)_@k3{PtO~O<#=DGr5pgeEk*zqJdzyS*2C`HvLNPz%^}$1*--MkDB8ZYHi*d2$!8+V z>uHB=7@hU2-{ZFM*5KQ-o74fPuL=*f%ih^eRI&7#>gXiYdWOJXTgoklguad0y?oR3 zXH4vz)h$vp*fu#>EJt@p%zno~W31P+sJ&YsRp>df1 z>s;-8WsMtB`2QwAlK^u7QHD|$grze0>|nVg>YYoh3w1mRtU7l?bf(U;81xxdvVHDD z5@<)VWMum!7W4%(3d!R8aL)2nz3(N5i2YyBA&U0R`kYxU;`xKZ{>DT zCv|a|>8JOR7c-Ibl_6IGGc~UHViOFkipGt{cY#L{Esm)cRSpr`qRxS~Oy7H^_8v6* zavAo{{an{OHW#vQ!`Y%%c06wG2&Z~&d=ke5GF{ERvUrEt^0>+?tR4wDilQq%QEoKm zBF!^YJC8|f%fR;b2kaQIY)M*I`0Q$nnbU{R?2-W)x?srLF}jyswW+% zmS0}v(j5L8&_$qWrdI}ln@!q1qn&82d0yrDLXT_D8aapJ52DG?7Z}ZqGPu&Ze86>| zVq;ZByBD7(wnkS9K4qd7pKb_O(#E&9<>Ei@ACA3TGjK1}iu%PA``aG-sg!Vi3DxmU zr_kxQoMz$D*m8)9?ditVVba~9_5}pt*FU+@fqu{sg}?)U?|V<1Mt*;pS>QsuU9!#W z`|qohyp|^?{E{{1bBx(^YFd@?+R#iDxl6CI5z+GAF#O!=(**pEJX;ovWAd&~tf%GX zawk0GTd?{N*Ou6XOwshsbmy_BFAI4J_NPCuZZmJ=U^UejJ`q?yQ<^;&)2!?4U42f* zd=?lGU|S|fFwDv-opMdVNFFINA~n51oJD zoov-0HRMhCv7%$dsV6dcyVjUw;TdZeb@F{bgxnplc7I{n(3_TR3qE}d%HN7EH5V7H zzD%uu#=Mi>hQ06zjNGEnd$~pb$uZ7G9B(WQHIT#%#5cTEc1 zmDK{-B;^>xl1Zjqoi1my2`e?>P{O+?n!bq_L+^DKQj0GX5f9&}Yqbftd)rVaiav&0 z^X7kg7d4!`<(6zb@N~W{b+%u%I(s{G{)wOa$h(L`O`j-%nfm_1Vtr?!tmCT)U{;Z( zrh9^DK2O)<0|g(=#OD#hwl1yY_UGR>74*E+(p7<9^q-W|Zb=H*jOoT3+AQRdRJYJU zpB!4fQh!wG6q*?*{}TY>5?e~cJ{t-rFAB}LKs+6M@{>@bq%}I^@3@fs9u6e>v7R)4 zNF@~^Mo=@kd!674!RX0eJ0cg?#8(;>9s|JQqg$IBVVPQ$ewS>;vbUn{?Wr@NB|I{B znxWdJOofC@aTBXb^Fa}ZoKeLms%t+wfllM0xLx<$mEO)F92#qWT9HTH?-(TVT{I3W zs8i8~D|C;jwyt~NH&BB!bBd@`4vH(#8kokojQj$V#FafW5ZvLdEtp`;fwP`~3Z*1a z4)Gsf?b!C^PwL6PFgy4oL@8|X)508HJf_H(TB)hd^|UCFs%eCs5^pr+D@m5a2m>Q60@xyO{>DaCT#UMVR)4UB!q z^OVA3Ea@IIZIcD6ds4G1Y@qS^2G6szKF{@E(8>fu5LT)@$(x$@Ef%_2s!!pmrze!P z_Obh~f^V8WF@!Tnk#x$loWdPgWi|iv>rg-9YUV<|U)+Uyy$R~_e5m3v<-R=a z9W|Ph{kb|L>9_#1s>iq@e6+3vu8ZhhH1{fKWGu-5{bFS^gF~#ycVAc z3qFEEndd?tQv7gAxO+K15LVVGv0G&x{VN12=bI08Aluuya8le|DL#e4EmOJsfi_&> zFA$);y#-I!pC>{36B<_%nR21(ZPg>`KP(v?n)BP{1uvRjsOTT-7Y%=;u-X@%6x6$w zvsEk<8$tRp%Sj<*%7C=Ub-$q$=f;kkwr9j~q{dS&W#B@AsGCZzVQGK);J>a{Z*2seJ1NY2eTtnWailrzxfJ zPWO39_u#q$1&+MaXB}k64BwwMXpCquLpeT`EFXX3$Eo7vk}O-`*F1or+55Ox#{N2$@Bq6VD{ZPJq=QYU7;u-9|%!I#z z5}eT=3IHeTFv&XlD1lJQQ8{4M$9tM|@dmEOb16!H5V#2CX&|}Bv|aeXn}$2uv!fv{ zgh4QE$h|v`;t%S6L~a+&Q`jH9u}<4D+=aTxG8JM}vvaft*{}(qw!FNTRnpC_OTxBu zi=0z;b~+)B;_?m$?1>4kAyp~>!EY$V}NOh`m+=stv6ri zNqPxacb)u*^mca*<0T4albnDlg4U7L)3*S0u;>r-Btxz2cY4|EWvDuv12U&?4W9s= zFNiz+g#|qF1C)0uP8v9MJUQ!vg$Mplest0S)${=Giq-W5y*k%aEaV1MNkYtCTa6nm#4SgY#@j>#Vc*+`-cI? z;?H-faGa3Ea8E|Dwa_%r-R%Z7Gs_R2B_NNH94z?{ece&uJrA2=VV~e?i^d`ZEa4V$ z+O~GxyF1#Z06yJsNfr{p^p-DJ-b5GrrUD4DtoKxjH3xQvijTi`%KCv5<=Eq{O7e)NrKAC?W zR~!L*y9EA@_BX?JWlO=bA1P6o+y9h8@L6v+9^;w{0~^D=^N?*?)B@V~Tmem~zOjs`!!>VtW>!`t zhr7|%#qg#nu=rxcxAU6%mVXe~f#VTAcattBfsn9cxP}v1x|ot|DV%vQ@L0SW#x#cp<^%ae_0BXoJ*2Wg$NPo)&mfAY#-%Sia;Z>l)OE{ zRH+i3cQW9l?gZ+M8Fl^V^)q{a5!?g)@^7^UznEjOboA=*49r6V(SG|=;9>Hn;ZK`b z@_UojN{yN-=w#0hSCZ~IAbHZ=$@lCaTg@n{637Nld?6>mJq)p~(5=Qe_=bBUsy=(9 zhLD(qiqQ5YLf~m5lh0jv&)n>WyUJ&0T;^q}cz5%VXXpkSi(3I#{yaX?OiV}g*SiRv{~Z&~Fnb0L#O|wC03i~L-2;u?7IeKoHDCZp z_TID61jdbBK~2Eo%PP}Z(@xDW>hNRkySIVOu-W@VaS&Ld9Cs&RM!1E97) zWX_$H-xewpvK>=2o3<&be!~i!AoETzq?UM;zQJnOI@Dq0hW-3~{nkWfse)PYg*E+Q zf|{psmKq-km$*(jh9bzBd0|Y6UU5TR8q(LXRY0B(ik44ul3SS-(xRiWd`1GtmRH}n zjCkI!s^}Cs_emqD_sp<%h{5-P2h7^O8k0ML{mxZizy9xsFt5Fo zqij&mT(w<42Q>k11WQMZ8`842ob3Ak|2C4tT9xwXV` z0j!-1%wt`_H|F-9nZ)}lCF9Oup$)rPK>37P-A2MO*I*RsFL3K^QO1l7r3ZD|Kb)!R znX_5-65aO?b&LwICu?np)6=9V44w%%@Gj$}4WnNJ`1$UTYWi>5Yh!K0K_EcEcJK*t zIXU8D0JVWMEo)XIx}m{$A4Q~90#2P%^)Te+{v!mzUq96Wtdabgiu8Em7_*rIAptK+ zjX*U#qV3el0cYUydZyy0Kfv;toN?&1M5{7Pxw{KLVlT#QiWqNq2*s~?qjt9eBKpbyRwc1i{_9OXB(%E= zl{!~3N3^G9;bxK1H_xCsm2Hv`-Vso_`zDBKNNWfg4&d2sm~aHGoC3^&!xRuJFE^fp zTHDs~E>Z(%3;t*(NYAR;w3#w_5Xf)#{MjPe6A&6l$gz*1(i{ablVunaOTHte01x+J?hjRiYtWu9_HME-yeNDAn}?rIz2I=Z*#GLz&+$x#y467_e_Iw83>lInP?P=c2$ayOP&1M%jf?!e^h4U5F+QF znfm(JLhpwhA~u6BE1s)vmdDn{CekE(;<+dMtoJcdQ&EX_+ZVI+jtmq)!!mJ5n1~+g zIn%Pi4C0f0_z2L5*YWbDqr(DWiXn=utgbz0*aM)~1m!I>BC$)}XU}-{N*NXm3sYg# zoQ%gpf4Q@Ou`b-CRIv1nT7gqx-T95+$7mX%O;ExhWEU!cdst68wv9hF-)}4zwq7gj zdNl6M&2g8Vtsuvp%b^w~t5G>gG;VPl3Kr7R%37vWLK=MVn zboSs$HqzOk-Zt~VoF)Ek<1}=LCca1f4%{u>`C)6pB4_4{X>qL8#oLf6pWY5)(dLI+ zYeHN?`#raL=lX_ves~2`^#3;o@>NUa+_+u`xFjSH;$Ji|JA9ekas53L#mXBXEZ=}#kcLOmg2k%==o zn5?eM7(gk;iYZw_Du2o3szuCCL?(F23nDs{(QAT%Z9{&~j6DPxivn#>K0vjM0*rME zDIKd?7Q*XM)>GG2OSp@S&*1UuBepkc9fcm}kS!G=b65iPV-Se`3 z(GJh!J5+X5Bpxi`4g(?4PIPX1Fa99nNxyevrx8@NuZc^%b8jv2p4Z*=F!~5`>Y9(< z-lR|M93rut1Pnv>0RsZ`Z&i-~F4alkAF5F>fv5WD^w9bRY9RNii)_J#9XKe4&~gOA zSBQNoxx2hjGqgh$&!kc~L>}FK>qgYrT*+%|cr@=M=U$(`9jH^lVOZ*=`wYcpnsWQb zl-*ufXI(Y^G^MXVy*Pq3f~5vbwJ*^uGr=hHRK5vUw~XEH4fNuP5Hd)=7NJSiazm{@|9FTn%A-<=DRh6?93BZ~N_aL(uE;5clB{ zMq@g^m!4T?BN^|UkH5E#JiRkzKu)MkzYTK0$nGH`a)Wo7>!AK(ko%Cm28p__2#or6 zO_)h{-&4uwe84!`$Hh2xf%#R6RN|cc5f6W6Is|m3!LxNG(pCYXw?!51+nBvfw>Dnd z^ZsL~TldS$1eNK;$g?@@b~nWorFBu8@rc!N1O|Cv37&@ks~9u{xiR;{K`2Ri2=&TeY^Dy>)6bVXQkE5@Um ze+Ha?qrC4V7^|YxRMvlo7;qX%s_!zIDUSxO3USwF{^5_j21InIjlK5L3CXSZ`1yk3 zF%t%+F}_=5mPP1SZx)GETuKk%hiR?{$Lig8&pOz=cONPLVdHkvN+$)S21pr*ue`;_RE*bd|&#w7i_ z;P8`lz^fSIkXmV!a&O~efE!(j4Rc-O&&pg`z=d9@|Kg@0J;{>2fH7fBaD54Q@6HP( z72Q#MUv9CUCP9eA{!qhnxuGa(k0<%Khr=uJ*yozhX7kV;DM59e{zXxc|1-a;Zs=gd`vaqVvljH$LOWcjm5^AD!EiyLv>#M&3q-x-RHya z(~pYAhIKlBSb0_zDH6=TStZTy$6U&C&1QPN`m4^`C?K$x?^y+>@2A_O_I(HbFY5R9 z%te9|I^^XGjr`3=_YV7S-kv^ps!z;{@Mzetzgbvqw%XD2=;jPjm?X-f{DEZ)EoB`T zljdp!Az-AiA;9oxDZ9tvgZ1|Gw#;U+x_Fqf*v2tr5a<>bkta*dP2cc}sl!)FMcdf3 zqXE}*1&EU1PH_bbn90IPKU;km_bJ-;trG?M@thWSy(hWf1N;Gs!-s6UHcau52TWte zeXk&82a#U-ge3$d)d2sR#?*=Wc|FAojZX=%@~o00tHTUXR^*_`AwuHUo|zPFQ-XT@ z07o-!_M48{WK_i&eQo{~?-_QZ6*ERxPTDRM;+MrSb<-IQ6Zv{M#!Kb2r}hd63jgl6 zau^Gfek#fvdB2^tL7>KrL%PPlMGF;@C^1%c1aY1`pGF6o+~ zQZLi@$X61m>Na_;*IVd2wccpBzHfun>=fG^&l^pbee+PGyKSBkm^nlY|Kz9PU-EUf zzGURhX}<5=*F))#ZneP(+n`CP7dAay4`*o--BF3Q?>*oR6YjTC+ z;L$e!x+(fG#;%!8pR@(%57ZwCK{ax-gTmbR;N=$i-DmhQl9(v?IQ}ZE>%WiI?`r5z zRzDHlDO}v=di1kWt+{=5_&OHGUDfk8{=d8LSq`yq`f=AWbUN5W144$9V&$TCL-QTug zuO{|8keKu(uhxQ@D&`u20DsQHtVBY7*OU)BoW9Ev#h@ZFog8L12lO+uS8sfFC_0A%Mmu7}3J)z{wZR-v!Pd4Vx^br-5mZMDTH#uG>Xkm{9x4fb<+T zSve_)M~EIh1ohy~Cxh8vlE_i0=@~QCB0{m=4}4v{JH)fu7(W0XB2K4#X0w<3vbC{L z?|hwa`N~D}621Mxb47T0`M!Z8glq|wyL9yUGSB9!y1jk4h0lfn`A_>vVt{ z-0rC})%RImrKH|7?9cJ((Os^IAZ_ZE?P?|66mHxX=7$8%ZBX?j4M}=I9A0OkC&ccP zcFQlgO>ts{qUuucgH&|A(b0{VGz99#(O~dwb|Z8fFIA4g7B|c{s=MF~hmVdftl+@W ztfu!bF0Y2?H(<8gD!lXMdkE?S6{$0=a3r*kLPLIXGSp2QU%N>$CUnx>I6XFALy-C*)%l zM?&0MOged=>W+us|JvbnRCU1|(G#@n#}eUx=-XpP zpMH#t1bdbX-)VW0uoo?~|)OuF_Y%e3|sW8eq`!o62%5*mkVn8~&MQVAW zmz=xDSGTr!5$S=W=#Ey&h$jC1TJ!LDc+#}_$*(k-7C zG~Qqp9!GYUFFJFx>-TfD@yrWNY~(#^Z+bzzfo!TkP0+;LT~&*-PXEpeNR<}Oh1rj8 zj-=~d4vJ$QZ?sE%VmM${^b~=EH%vfl`_c>wFUzIcZd|c(L^*5HQ(o8x>1H!^w^fro z=$T@ZGDKvu7MW+$Q#Cj#{NC*zHg;eeLoRlZ9|G$S(Noo9`(Fni)hB z;9h+Py_KT(umdf+KktSJfzKQ(te6Dw@JqKudeD7I$w&ps2DZWXG+-Q9+){dYQ(JYv znagX}1d_J(oJrJEY6hn#eyetp8P;-PI0>LSFjAsCMwJlYp0l4foFV-c_i)w>HKutw zmHx4Qj02!N$?22)Gbg6nz4Ja84Q4xu4BcQ@QQ1Ha;Q*yI^(*v+s_i>pBZP%%2dfa8 zQdwF!dTP_RMC92gCgUsc<~=D_XSy$gN$ig_V(V@BzirN^t13T_>WQJTR!Loj(vOIx z$c#v=?%6+;fUOphj<8+|>RgKr5*kdj$xIsDJb1sNzGVWPvQj$m)tz{M0&6*Y0-zACk`b zaVnYDX{P$ejH&zHIVMx#o}Pp!!nN@HO<92jUeP##(S8Q0I$kEX*%LXbC1$*qg0oeC zYkQ!qV>|bUI!HH&kXj747@Ho;0RX*TaT+-yKv=pHg*jLXDcWY;P2D2}!y4rIbOvtX zl$BEB_J||w`rCDqi?P3^d}Q(Zp~k<+LuFH87~<1qThF5p`nIpa8>`kF8DZR6Jo0xX zv#1|d3;B1P`{3pFcM>#jyovb9y&~$$z+tt0$Dh7+yYrypS^R$RQ7YhcpcHAWAC8K1 zLg(spBXoti`;Qnkh@T`fHh7AzFPHMC@^pAL_6-@O!_Gvu&elo@{`y5V-Qy>Y7YX(Y zC^IpP$%xpv9Fpws;krbAlEF`Z&{{^q`PN;Y7wK9-vp6)kNSCh8oreMJm<>`j z-n7I&=vvQKDG{8CK#VJ1kQ)1zKFLOh&$eZxBZ6B0c&CZFRsUMehu9oH*z6|B2i= z%3-jddQT3}@Ah|pF{=#iOif^3Ouw2YtC-n25OE< zc$Es}SziXTA8{HMV11Q$;|g*R|JIpEsOr?AeguiB!%C6_3+B>F%sD?2%pDmY{7+D* z8R9m2Bks;q%m^ho((38+WGE7RS`cj#7>`Il{4M!%GlQBSPW~Y~$HO&WZ5`Vi6Mxi= zw$8?j*MsEkbD4A^q5T!VIp)UmcpD)9L}HPHD5g=bjh{5;A%iMT;sA8E&D2mQ1+?<8 z=NXN*hVhveL6A(`q?*{mOhp*d7fE=McF+EWYo*<_tCBEkjboBRp{}QpD-xWmhHzW(*e|JR zXxvdea)*Mz+SZHHc~j7b$EZ%21x-UJ5I-gH4v*rWVXY9Ms<``-EJ6+yzW^g7x++&= zOCLR!{>;rv*kO!Nt87u^W9)Hw!-Hh%hY2jex&uFt=y>S>s)lY1chF3wmX6#f*jdw2 zNRZe=`fuDIg8%YJ!dHcPvpmm#R0W8@w{3xpO3M1#z_l*4n4~J>VY(5o^@(NCdlv+= zOd+lJEIN(>{HT9mIo9W$^Vj8-O*t-mS7b|CAY`?-lhZ9lj^{&4e)20kXI9f~QH4I< zvJagT{jK%N-=NUre4$#hcW9&wQ1uTqh!Sv0meV5NmQ~C$H4q^qt;Jt9err>ex+`9g z!;G<;8!A9r9w5CP53EO%*#z~}2dvZ!;$1h*kF zxQx#L=kBiL0Xj)OpIIb(`3++=C-NT2OJ$?YnqxG>Yv0u+NC9plm@l;~d*mni=?bXa zGbGQGsgsM3mA0oad;!YT-8JMt%9)+REq@wg{k=i!4jp-9{lYKSrGfF~NPA=`W1XF* z!QM)?tLaB*|LuLL>Ck~60c9p^urc@vAk?PxiLkn@aU#SNbr9_+kD{dIR#JuZYON?C* zK`wM+C*|>8gw-E^4;ut&{5qd+HvMMbbRT;f^qZ>gS1fAz%1huw<3);UX=tgDTjJQ0 ztF>Re?SUOqDnuF62YVv*i4r}B;QWU^qxnRK<`>cCBK1hM$9PVs*AT z6b*M9G-VW|^xpPcV`+4f6yAv4ti2vAckqnvTk_p>wu89X*ll|xzkBqlgv6@@esAfI z7Ed>e&jI&4129GYd(VamgP+t6G`|u3Q1-1+j{y)`sJzI9`oQ&o-pN%Vs?IOy!^bm+cs_QfKQCies92 z-QwPi-HZ*C4RW+UWojgCHz_UyjR>imL=s@3Y5OJ+pzs8XE#BLD8toU&-Rs6+3v|eO@VbztAobBp$doqg%?*NksOoF zn~|6|85qAfAK2&QSJxbu`iX+!o_<+wa!$DFWA-R_cA9qV487}nvfr=jiB0)X-}%5Q z;X@PKdLpl0#RafFs zCLqAHk!pD9>e?%ZeRmE+W+~H^HJb&-j`?2Lqh2tJbhaJN>gXgF&4_PTH{+es>vOeX zo6*nyt#)SU1%U3TU5;^T3OPao0cLK$M^s!s<3(PS`1L_B9XqkAR_)a@_V<)Vty3)& z1QgG;Vl_iQr_Y6!`VO0O8eO4_<>YOWON=|Dao^JKo;JTHDB=&psEvjw7cpXK|3~Q* z{zjoE?VU{M)ua9-uc;NUO}e8RWr(Xp%{TwSd$O?F(%C``ZFw&zD~+oGtMHPj0Dy+| z!;7#eE16dY@uUW_=*`!lClV=waeAz&L$9La#P1j(z`kAcT|}Grvu_b0sCS{+GdZ8x zusC>W;%YfM?593gMNJ-v?s5?4}M&Qo0&%;u2eHR!Zpl1E?6nodhyE|?EQa4eH zma)=PW#(U{h63KXw$#uI9{CJ72R`yBz3_8XiZ$UyX#HDo!Z5SN8f#nk_c`M*3SVja z-u4rZi9&U^d5diV7rv^|EuLn*2Cg2FUT=DO6H?_hT{li2^gs5+Z=|Ib@rD$5W1V@J zA|n%6@29vIEFp~nU`%`lUf31`Y(yFcbFggYgSwL8ja+f!#X+-{Tzc`+b&^&QFOZts zHEJR9t;@*m2<|u_nPY2xJ>8d#-+<{s58z-u9Q~T9jcuYvS*e>Sl?d}yKp)`MZ>B3u z@M5p23Z}fC>?0LF6CZnlU^N8#fCIs*TfBcvGUwXf0nG9F{?cnmc=JlcJ^3_E!d?tJ zCb-ctB{0Q`5XGOU*K4p;sJx{`heJ2sUP{Sk+*3hkjF@Q8G&ZkHDvzv3hL*yMOg-nGk}gg`@VL zttOb}=!9Mv)r^Q=58!uYALOH6%f7}NxjcU34SoLS8P$#lhNRyK-O)+{o*zJNli10K zQ%cSp2nx91etQ-muq36gZMqwtLt$D3#X(c6a#H|J@6FqjEK*P1U}8olpnjM0H}!Pf z%r1MD-YdOolbm!DP~=KG#6iS0&s{5O6|u;q8HPymo1)+amga=H4{=@PhSUVjB4w5m zS}JFl52ll&BsFh%z=wprY(FXAKFuAR64-0I05;6(S;U8i|MZOIE>H1z1GZZ?j@w9< z;jXH$^w`ilHQ(qsj0Y{dEYy3HnT&V{M*@~Kx9A(seA}s^iodU$KrPprCFMB>!epQcwUHit&0^o z=QR0$ohWi6@+OsZZCrcfrQvsN0kp>tv#!RC`e9#uXwtvjEd;@qUy1{7>A5@o*(7-h zJas{mrK+V3H50?}{@8u^%FY#1WcDga)N4GHcMVdQo)3 zi}F%Z+UxQMFd=m1nD!&4ezRX>aPRYB!6*;>qNW)=;_C)UAvfCUTg|^kZ}%I3uDR6X z!r*TUfJz@XfHpw(q9?PsT)o!>A)GF&H;T&2|`4%Wwh296A_?Tc6x=k{6koYVn)MYI`@>s7Msfs<*U_;dM3rWjAUl;eX?NR zQn2}W^kGWSwKb5ErIhL)Z=eYsXvf(6Zte%p-UGS|wJiAa)EH(X_Vpx-#Vf|^-g@Rx z{MjpVr|w6FMDr2eK)BIshvzr_rQhD?037RXAz;)-_v5wP2<$*&7+PeKNILkYB)^aT z-c8FJghk}puZ9Oh+Cs+L7W$|y3Pg|vT4)ulbA&@0jAu7@zNQSTPp2F2;X!OkWT40# zlf~W5IPJJ2_{EH8zvONg0sbqfPm@F8d$r2!ft5TWOjKA_7r)s=8=zHo-<=>=4kslIxUN$CPA!S7m49w}cdWQkHH%UnfTwv6 zu3eYp1B_qpen#p@(TDlvAx%k0pnr2??p$$r9zO{tCvAiI_L|fD`4;`LF63A-bi(Z zy-di7|E#&>>uOk*&U=;6Ox}{+l6?pC zayxdx#nog}W}BK$a1co<;0U0zBZ z2b@zA4YOw2I-m4<*l0%aoK?|L0KTqT+Lo=LA9EK`wf^}r)22!H99$6QG$vKo{V>+F zd{3>^%9TLn&xA~Q30r4V1x2dt$w$;2-}Q98E^1ihnc{W!Z_b3=RFe?cvY*;5{?m;*7=XPatr@148Q$G@igb)h+;$SFRW! zqkf6OqS(kH;+@34NNz?(+a=XDAg~K!IT=5|NrzU(s4pzKQy|20VI|;1(7gM%1jZ>& zOLj-vUO{K-RkYuwlJiHR?y#Sn4qv&~e~|4Vyln1v6P#(bB96L*vnAI#v!Ml59_GRN z3fZmD*5lcYwGEjmLhH!8lP@=LkH|7|MzYx(;&w=Qmas$A)~mTAE8_#JoRx4W@jeKveEIT&Vl;KF;8`wA%&4~%i}9$ zOvqd;s?65aO$wJzuCE`sPzy= z^&LMC+FQIhud+Fa2wbf*wK@24>@|FCyZGBJt}uotYk#Z5aQGoc}nCU zR4>dNBe?XD{4Zd&yOIP;ts4f?3$_8ayI3Rsv2-SE4DZ8D7Tk%M{kS-^`?yNliMJv1qO z(Kcmwd{F$&2g*j>2@MF{Os`b?sf1FdieX?La=b2%SxP`Q;>02s&+<(lJfDa@lb|IIX4W{AYh z+8io>?l|qK=ScPxO+YPcuzcbKdT5ug-7-<1+Li4Qv`Lir5c4yj$=k!P4osiP$x5`ysJBQIh- zIaN{5wTJw#t57`~UN@_2@6+FQ&75ohQ#WFnQ}6o9m!*SO437{E(2}teVhpTD`bqe$ z({YJzq9GD1dIs53o)iEwjLYS=g08?grzuPeuU6@VIVS{-x$d%0uGk;4H?}BH9t^Be zK?TNLw`+iE3$i>t->oiBip&53q`E5opPL+Cm^4-+lkwYC_^o3JWMJylvcq%zN~xC< zcq358oPK`9>uB`NWq&TSdXMF|P|TPc0L8ug#Yg_WCF)rO`rSp?D$F`Wp^yczQ=6T0|(x2tV>mlE1saH0DgA7{4Bq<^GF^;G!>>1cqEf#cb7ivtb@o>`lO-L z$A#Wu7*e)1R9rg{6(BmXUs`kjUr8V@Z0Xq{T77?1BhPdI;h})diTWFHu5Ua9Uau`K zXMm6xT5U5|A6K5!wO>MA5ksG&k#K;@xLUNq!pzVfJdenqXu2xQ1X90!`b@G~k1-Eo z+ru}kam!C|79es8R{yxCoy~&qU0;U4^!7Z7tmh^LEEwYnauR;E`^cZ>hR*tb`7rlF ze{^5@*Ev8{gr(*pEowfUGAG9>`wu`HfA0!~V9jdqZ!I3D1+=Yx(TO?@L2Xym+fK@m zbc1^x!JTRNfyFshV=Q#(;bSz{g35ZH55T7TR`xbS)p-8M{qJtW(yq4Eq5RJJ8g0Ay z>yI$q*81W0p9^JxYAuEqQI@9e6_h(Jnp$4L$U}H|0>S8JJ}yowai7cG6mq>TJ-~C5 zf9?jA8G3{Kl9!M0Syqt2N*Ty?&1>+O@iX=zKDn-BDY-2T#DlcF?{P!^^1Wkd!Qs~? zO3sIHHF^oYnYwy)ctM1hRwgz~|(&aQ6tS!}3J(p5mpYBltdam`IsKkZr z0-RW*7PhH%IT56oavv-(i@Z1bF1S`8N_e+l>=RmNezEd1!-@=`906B7#Z-G>3nGh= znhSqs^i+hBF=k6Opi(FxFAJj$v5gh+hSCY_d>6gU3wByp+YU! zIcNMi!eBVN6MZnP0v<0m_ynftU0t$kH3kW@Y3-|tM<)HTjb21YNSI=_<&TwB9QxfY z%F9x2QK}=&`eL1?yp)l`B<>D7jc-Au|GDrn93l?(`op6U5SZ9H_AE2&D1L>)rNG(s8J}UY zOSxL;s8bQ7v?&gF(IKzpCxQTsdX^4JlWBJIe}Cym!PWQR$1~ z+`TT_1LGD&QB)Xv@|r>}isjpqbg;Y_1v}3#UTzy*2B|&R=FpDdF{^$`>XbST&3ZKo#cZzFN@i)Tt4p2ITf5YRV%|92c zoFI6i3-5!8*LWHjmDL;JW&>^GeJszWXt-(4Dc^&r`ZPY)dmLHI@lW4V2*x|l?-D4W zH{HK8WTcaX0rk^bKE3$Moq&&jSo>ua9bK-oeihY(1h=Y-3`bbz#IEQ0-s0^%lXBvG zl&23*T^hwV_tF(r2)w)1j&k%XKhiq=GC?A9XmZe4SA|K}=^JfGGykH%s#2EOYAgVQ zI(3-nYgU`LnsNhgyBPGGjSCo&5cU^P*A{zTG8_Tr@)R)^SopHP;ifTDF(7b?D5P>- z!BF}|M2+~aJnbOX3bIkXW9(*<&xYo=1=%9utTl+i8%PFs*9R}AQv1_gELw^Xb9DYL z;R{vr7~T385ei<+*hqn$gU5qlY=gOxy=hd*-9x&gkDOcw$L~2rtPLCua5evWOU;Rf zs3~OraloJ70}uS%s8Y;BqUFxMv))#4D$-=8>nwOLB6RwserJ1H+Al7NW~#x2`Cq^_ zwJbB8ie3mpFZ%-!gi&gFyB7`zDB(AKKiIO_k>vEw18Jmdp1Ib#KC*{7D@8flOL>SR zK?vmO`RBx2HzptkUq{eGoG75c>|XQ(73XQ!Bc`$2ViHUS*z`Di0y47Cwev%73+Q@E zb^{5%qpMjQ96&xBuP%BO2ZD@Eizq`VMw5;Pz3;F+_FLG20is=}d(7PH&U`)@*7%&g%lN#joq=UBtbFL|RUE?EZy;G* z)@Hm2QE9bK`>eLae2ULZusFFSi6+`9$4RN0kKasSog=BzVQ)-=COVqewy%k&kO*h- ztVS=H-%_)M{=x&w8a%_ zJO;Y2JI5M~C`OFH5at=p_#8OV%e&(ldRQHN^7L0WbGmlO&)f#u)LhB4w;COW=}6xR zEk-S%Iq+qXIe75%XaJLp?a)p=1(L)7y|skOZdyiCJFO}h#!c-;jlnws3UOdr#O{=| ztw%#c=cOj*3%M`*#@k;rI5&{bE~?AXNn;9Mg(v5V_APo{ygXGX94GuNKJD%rU=lY-XerNbHv-eti-S_=l1Rq+3 zM$rM?3|tU_WD#PtU3gV~9AU zqBlE+zW==Uk;0d4N#y^YXjYCz5i9-uRPSqirE9E6ZsR&>?pBqzr9Uy4qB+`VLRlV~ zQ?ftNBC1B}RSt4i`QH_3F%tZKelIyw8s?itZ}9=V@n0r{Gu8oCGck%w=QmKg&|p(U z!_)E8_CU;aU`Ua7VvEpe$BjW%0~x&$x$C26srMc=pm>^un1Yxr;??Ux)i4&U9f^=v=o1AkK6tHFn;Wc}_ey-Ic2Avtc%bpZhe6orDZmtIKipT&!uWel1z(aD? z&2bmuWLGrxg$>_?(OP;#fg+5Z(0SAoD=8XnZ=3lJy`1#U4afkF-|@zQ1>NPZ~^X>{v!Q z05oHA|M!rSmxvAy$VOk27*E4e*#Sk#zIend0@pRE5c{8Xp+q}Phxnz|*#*{4i z5hGU`Q9Zmr##)eJSLm;K-*^5EKj_fLzse$%Qq0nlkN#}*;Vh_V%BsNWKVL0HB?h0d zh;L#`tmQFFCuq{J!kNlB@S7DsH%^tcFgQOch0`+ z5|g7v_c8mB!Y%%a0nDuJ@(IXeg>4P_)|&(658jkRCo*#S$U|{;8#K(w2Wk$R`oAN2 zS9;XYNOyi&%Z9Uh%|hv5ImSkVzg6GAe_voYcZ{?E7KEPVcsR`+ISYU~)c0PH)jW+% zcx7PjUhVyC&#E)|{@#{+DwP?Jqi%QNMMH*>AfC0Accf~o>^O!t?|h9HDS?3N-=LfV z8F`1SdZKTd`rJmW3-OM64smg_HUbv|DgsO(*{lrRv8}E7YofV|Adc+~dfqEs@=_WI z|KY%xz-FY%ViSgE)@aN&>nJPzuE~U5kT$~^2-f#AM||2?+QuW~9ZbQI^5UYjN>2U# zq3{9TA)9jdcORo(E)!kuxyy=)rb`E8@OQYW(y~o;r8pTSe56N;XSb(;8tKYFi2#&f z=m0sD9BHVATV1j0wFAFmg1G^ZOnC~FUO;QxV`G?-WyNvL!~98vAwYnpAtTq>Kyxww zL(ZXQxfFO*EFnHV@eBawS&9wo0gIAkyp*N4oYn(HkJJaXeh7gC>o+EbO@V*PXYub- z75PgFV{9t~>d~zk8b@^;aE6h+$08Qtxqs6|G(0E5!!a3Y%oOhlA`h?sx5eMv{S8!jrPOEhrT^Gd*Z7ugN_{*j?>mxA0nx1+QgrwPcT z`6qi`4a3D^YxnH90e4+yL{5~pjvi27vDUCjH4|cVsu4WjTiGI6mr!Zk^QM2-dqUKB zg7bwgdrFu4jlQG*Se|x{MGx3mzt=1AXCA+fzCnF!mZNg_ZvmFieX!^wZhM=Xnl{zV z+Ba3E|Ne{N`#x3MNjB_ZYn^d-20k9=u;9mA$QPvXZ*oM7!%`P%dGPk+Eu9sDa(HK; zeJ51lQOX5G2kEV^n&P9p^L25n@CPhk==tY%=&uEiYzkrV!x-x@s<7106CNnBcw~RE z$g;Cjm^MCtk4cJtl!K}`>tX;GGgGPg;5q)%l5tyu&@k6+1U1FI^y^QAp$_)^HdOa@ zXeWJ&ZiQZ~nz);9^mgSM*qvQ=l_Ftxy+j{d?KRuJRO#ey&Hgxsad`IfWTK#ncq+kC zEknz8z!cid@~qIOs6$0<$aBYn&Y}|@l%pKSdf|Qi7+%7ybi#H&)UHB=w)mGEEV#rR z0J4j2Hz08X7`%j{^DB*4xu}fwJ9c%oXiC!>anKWS$0fCK{~xi42-Scm?Uz({`9A6t zMXz;NUHuDWw#9s-9N7p- z&C&hMLhjr+glB^-syWWu-{0Gq@_lzuBq_!&C+8n=sG!_3nVLT5yRLGF{MH~9Bb#^9 zYMmL*yLfxtX`OQ2;ZR#6?-@D(AB=Azwt9$@tOGM_WgKN?sCl0q;KGUJ#9QEsh2ogC zQ022ykAVu3I4Hzj%C*`5cTTkmZr`AkAf?}5sk$id$SMUGYF;#wfRCTE6a4TbWVdrM zWlf4|T*#C+3?5n!^Dzdu&gA5r4q|*meU3ch5HagHzFCY#&9~GWo2OnX^42C)n3km; z2NFdup@atq5$kMTHOm|EDE8hUjq^H@C*0vH!x?FA3`Soec0Zhi9Xp-na6B3o?ezA5s zj&CeIrqh~%PiRyqX}edPvs3ZF6~M*fa{f4yl(D4W7qZKOkvuVxRh}8Oo2w_DWMm?K zx|8<1Y*FAe293hdkrS&F)g8!VYRhf$yyZ(E!~~hxb2&2anpv+uiko5V8QyfV{kb=p zf2N)&r%FJ+wBAuI*S|adma}$O7K#GL8qHd^j*I=Tn>M@vz$6kam^o%2JM~x3r@kZ) zyg)(zj*`vS5=fin@X0PhbI{;QuXlyeSVtM)miw1hBTR$avOFTHo6y8;a+PzHV)_IZ z~ECP|jaz$?vM(O81GGdkBWS$VRu1~ph*EEFZ~TE$i2 z1<(hIs!jGsxTv!}vK8v_n`7Ved_7fMaOif|+f}I4aI6rusiRs>xM6Os z8Pa&tin7%sl4KP2I)x^7+OezNqnR|+@N`b@A5vpPl;*#sGz?VW{#qKKd!zg0{b9PO zK1(t+3z7Mc@AU$1tLT)7H!_F*Pu!PM-s@VW)1`^pu2sjTNJ2w{s}Az>`5JXN;V56= z_dTzK^^OJJEG^p{+i1T3PEzVYBg)NA`BT7hZ}r=Ac2BV8CE+wr9w(ma3*(9K3GjZj zp|mNjBhWP^A_s$Gy3t4x`pBKF$EHV^(k0nJ=-b47gYhm~fE;Y2ZabCD%n9s(UIk$I z?N4&+1q)4(|1_yJo!fJdSOt}?8q!>^=_fJec2Yl%6Fc7BDT*M2R^={{V5j-$iCDQ1eX2w$VYW36s|gUr_4niN0Ho~Az=5bvD{(2 zYpOE6ba7!lIW1r#crSj=<+aqcfrjEL@4L4NeVh+DgI}|E-p|r>etS@A+8CvbZ;=<9 z8}^sgC%;t$OPvxKCg62A{BX=95~o7jo6Tx_>7^dyMxpHb#&Z!<)+D(KPuU}X*>sw% zoIZo!z?RO8+mhFLhQ}r($#u%s5%|I}Sz3Ul9*Zo246yXM<2vS!Z=auM*M;y%bf1gH z)wjT*x>UWpcl*iU6sGZ@ON9gUGn)~ip4Vr8DblHKz0VE?cLs2sQi7NKL_oP@~>XRs`nVHjTtK* z_-JJ)-$j5tcit#HWQ0H$`4i2BQvoc~VLAL;mW>7@MU${bz^CSf8JJI}*O+|*=hr~A z_(_iyMc1*VwTOB0=J{3l)o%Zy=aeYYKYZMfG{V@3`1<SkFgl&X|rGq?r>jOIf8{PZ22LBP&|sag`@}u4tbMQ5KNnaZFw2+}qQfJzV6OEjfGVf4AU%ITnHG2(lN2YQ>A&o3Fqq@#vwvTbKy>TzbS${c zO7>27Q6guj?j(?JZwi!ztVyF11X0j%=5ss;dE4D4zl{Ai|3$G>N$Y`k#LhkQt{K5gdbtpZa!G~Yy>E=8+fKRRj)OBV~& z$DaANIPCNGKRKg%pc1rF-*%*OM0`gMA+u3WZ8?Yr#e@{!(KNbnY#RFjjFSXe#EemX z)w1r%O@~jBIH`RI|MavDMYL%gv`~->67jQ>`i-XGL~B3V<;DLFJMDX*PAvZYHNIy#pW3XBU+Yy&&Jg<{lrk@oL_IxGA|F%gC9{8V zwCB6crPI#*=f`oJ5))wdzcf!4oK2{xSZB%R2QkQ03ie5Bfo zqU39lsnga9%+!jr2i=}1mPC`cZEnq89JebHa|dQ zvMCYSd5`_^=h#2#+C8KnR(^&e>Ej&RW-XT)kuOs5r@G8h?`w=+*5eOZ!}=0u+Z@8o zXvU8AI{aH)K%4V$bULM-z`GT$xm|!Kq&IvoBelLCv(mBp$a=U!FrWHtlbDCA@pa*` z{^z?dzhjdMrO;hE0g9P#g>m*vIbk*T4#DugYtz!)VP(6U0_KDf79kFRZMFR&b|y$V}V{={x&(T2~%uoXZZN@YWZyl@JlOUFoY7X+@AHsRfx_PYC5H}~rGGm(jm zuSEJ8xd_kO8va`OzdF)!aeuDL@s7%LmaO7vC?q%?p!1QBMFuTZ@~vg6uXi{4UU0enmoa2Vzn zcs_gPAAc(BWC`01TNSi8O?!-KNcB1CQbivVw4}Ip; z`0S&1^=^RlYNYp5vu1q1W58VXi8hkEI*ep<+&h;KcKn;@JnP?=ys6U)p*Qu2T!sBW zXE&pW%$RU2^_pksJgc~uTOsqQiStCigg0Q+1OBaM$QOvWNUI2X zNbo`Kq5uPrf0$X49s_HV0rkuvidqh4R-4D=X4ChtMr}SrikYaWBeg~^-Yjcl6if=x z#;Z<5`b_I3nK$ESYZQLUt)uZvi1a&N_9Ex$_pT3|6A@V`am_<}l?<#QeJ+Ss=bw&K z@43rbz9v?5yj*WHLjm;!6w!pMiI-#aCH@C`RGFUzbcC{Jp}FoFYB$efj}M!1vRVN( zKUN1rLT_?%;PJKE=# zJ~+o3$*)n1#I2Z6WgUBu_ULHQIBY|QTK_!X=fn#7gQ949tyC7vxBS`9xp8XXPIU^a z;qd;*SsVFe?tzGI7{P|Ph{CPDOWr1sJHH4y)QX&r&dl$f513~ zSF;3zTke;ms#!tQL))&Qj7t2ehY+E4qEIQ%%;Yo0{}y{P^T3-rz1LpE;PrwdqH~gA zdMG~hmgfI$XbHlV`VcJOlW9&LCw^ZkM%9)_v9uP~RE)*bxoBj&j`o_r$~xS7Xr^%a7|YbeG*isOxxE zUAvHn7%qO02>Sibd4Jgday)c-dTwTlMV9GZs;7FYLpY49R0_;a8DkAL>1!*@lKAL; zc8&moUJ1HI*}5sAE`>FUU@;e^T|lMBS_pfi&`HE(DAn$*Cr+v{9ut!N2e97PZxs~` zReThWCCJ8`$5bBPZ1X{Q$k5E$F5{MlIr5oTfDC!yJVg6NYZGJWBp0LvZ5hjLwhoP$yAr@k zlhGt``RA{Wx9;@W1U&9ACseK(shJsa4`u#3r_9KR!tP&B?Glbn!pY9R$2pPO-|FO{k0F<6 z3kc}dO^}*Dm&3*k2olNP&{l|?#S{JmcsE(p0mAPhpX6^wSm7!*nfRxp^kwv_g^x=Y zzXwX0LTh;rGjBAZaVXay=F|%56#t1squPVY2ZYX5y{GJ$H&wr^?V3sIwVP1dolEliwlfc5WYNU@F zc^lD5eN-Rk&#Nr`*oa~t@z~el7hYg%q_H5uULdGdrc-H=e>%lTiwaoE(gGs8n#TIc zr59KuD%+M58tPn}4f=rJSzl4@czG<2+IK0;P4$aqIf$3Jn6fhTs=H9e+B{CpT0cd_ zk*Up&B(_l0(+ni|{;K`0ZL7jBXKqtJ6z-J8@9T5~v`6dqOhj@>!7pn1;%0^Zw7mYD zyAR8nxC{0?tn)YyjRN^@5mSJb2shz^1q7kI@XC0vPGSFoXdnGg{x7|LoD!XCn6Gz; z6~Z_Hwj=?lgMiF7FBF0^FKyV=p@fR`1Qb`#Vwwo)xhh5F2oIss*C&d*WK^P_M5#q0 zz&hq;?K#rob~zB+tWAWmJ-f1BAJX|zJ`K~>N#vv2>|rA1%VfHg$>X-Mg5Zjs(fT)c z)G(Zg42oDsN-b@nj@70Bq*`ZTT6sN!A^83PAyvIRULMggwe9i|G0l%O8toPG8@zjL zSu-LaWjzT$e5Zfx3+y%n7(+#mV{b?yW0r&{;D>e*RUMsl%N`e=DT00WiW+q|SEqme zatiMvYb(lHIef7-@qiWnVb$3`1m&c*pOiK56~>=P;X%{iMXwtUcZVDoj;WBY1?05j z{orbi=zhL@Kn(i15G-D$oakL{!hhynsv|WeLeC8z6lkb5#xwb?Jn^k%Mp%G1$KD<%A&WbExp{m!@gTm(trLPB!d*G?&8n)8o@88 zQqR^F)uNMYRKd`1tHi_>0sD>TKwod9byV=aB;e_DLo26~Nd)-ARF69UOJ|bIvzBWE z3arv^?nL1x<>=^r)sZ%J-f4seQO5$T6_M9$s#q=L>Zy2RUMtMB{syZ1BLNdL{qiXl#=N#*DEi+6pAi5>9FIu%@0q{J?qlmub!M}qhu!uYXDnVa%yh+3Fj)kzI)4uI(7eEyQU=0Clky}X2=5h z!;7&iT3%q@6BUnlKb(VwAd~J3)>nh^r%qli+q$I&sC%4O`LTEZ~`qP1gbat7S`#RVqe@EuI)z#VT<~p6gw@SI&2^EmC)OWjO!7o@fRnJ8$@j18bPiH7F zPpx}mKJi&fMH0-T>|qk}c^h`tK3;SelQ;8tV#lJU_lxS^mV<;tL1Br1{s%XTn=@&u zy4>b&10DI?=!off*^$AES3-a8#*ugjG<88FD+QB$@EC7PlJQ)8BgWADWpGIG6}8D$ z8qa>8yNM+2-~b8f&z1t;#U+-v(`(OnU$hO`co;mrwYPuueA~-c)c@VSZ_-wus0fSn zhTT1vOs}07i_9qZm!y1JR3e&LH3=%1+F&cP^j%uRC5|FHeaz%l(>8;p(36lpG9*m- zU*=mbgXq`TapZv*R}PvOW~|P3(@!lT6Jlo^*qKT=Rs}DJ|gE3E@Kw4A_bq5XX#~l)_F)SL;|1 zC_SZueA6{|J7)eix7PmqiJ__+1#7#ZA-=#>FZhi|FzHFCfAZ=bbo=r$SFqHi%Y(;QwRRRoFXm`(NRb;q9q{-#sxPRHa@@FK~`U*(W#>U_v<%Q1t@UFh9Y z==hA@ueUhDDnmt@giGYZyP>y+4(+tMN$&w4Dsji+$IpTr`VTV}z9mQw+EZ5_f~Msp z|8AZNDRC6JPA{S{rf8Kibr_jjyknYsLd3ty(@=PQC?!){QswjQQMzuGPRf?z45eK& zL*|M1q4R@R%D$a&wRhT*a$TpcN6BT+kVex^q3;-|%#`Hqb$iBluJK+;$-gC0M1uZll6cVDp?^?r=lS*Hnq%HTs3o{|(=>vo%Jx>IiL4_ltQWda-Mb$?^YkaEej<#I<>5XhaR_onB z8uiSYln9w(0V}%n{sTHBX>qR{$$+X*;!az}URneKN6)dm0uo#<`!FnR<1(fffu`KP zQbOvA`vw?iI)C<8UiNjaPHJhUXs<;dC29+m=01vX2+E|h4u9$Ri{mfjAqSamNF(bA zAko|{vJi=JLH#%aA_x+}rWq~+9WbZI3kz+qAPry7S_KI+O4tfh!sh=gj#EHyWtF2nBU8fRr9PRj7*3thpuyA47X?z0r9b@(Y(Rhd9Z(bzp7I z;}zi |Gg4T^eZRMh5hT~jB*#q2$O4P3W&_j&q(4e@pm$>W}y_aLh3Kv27e_N^iI zo*crHZzlyFatfnw(0`rAm@)gFzoV28Gw6B+h3;Nn3o)2HZ!3@t zQNTr9gnql^ESnItdKDT{mGOK|Q5i-mMzMFQQ7!Ig@r#k;LG*4xMsWgZI4HAsY|VI~ zks)c?Uj2uTvipza%Z-{@eF9fr;JzK5u6Tsm7UyG!p7Wg-NoSm=OU5(KSw?}g$t6qt zjskThw~E!973_kok0)82@((^h!lQkfP96%^S&Wp~t--&UmxuF40(W)5i0{}Fo*|?O zC-J-!UMg_MHAJ zk&cPf&?k#0x^+$p$V!(CtNaEftJc#?E=SQy2}6t0cS~&Br-aGOs1TT-yvKqdsAI;OpVV7B=N;X!t=_eOh5htsD!2-53uD2R^z(u>^eGd;ip% zq;+ZXoN$+Yp;?-o?TBm5;I%@M;`P1vMne$@qnpHBIk?4pD=}O5>@@k#jmT-t$^H#% zDRP7LTvn3!>ayZ0xfh{dd6^3Dy>}Qi%XegD*VdZvC!Q7|W@V7|Opg&J-q9MS^~`;V zC>tOs@-Cc zqRwq)-5*ds%#P)t46-t8pPbL60A>W30K#egi`5%Ruo{(8q-|QZG(?b0ay3{q|4wa+w-KvJU!v>1# z2E?SOrEwAQt#|5|<74YBo#NM(YecWr&Tb)()ST8I&%nzo^F+oY43a`!(efwE$v2k>_~ixL7bZyya`S z3bg)_2+zM8q1lbqTNYB3sM!!-W43^=Ri>C9*-e~{%GM~sr9CdqdCWGD2ho$++s`pv zk}lZ$Nd}9>lLBpOG2zzCZ*3aEa%WN!kxjTrK9N3-K{Hr*se^cdqPK81G=VU=pIpw_ zt6%MeYTVI+Vt6Mre`9Iu7`zI1^1bsb36>XB$WN%kd)`1DY@j>6ZNKpnN0eNxoItt5 z!rSx-Vpy2YhqxpMR@3yvh7L@wSKAyU|L_=Yth$H|*Z`igIj8u#5KTXfJq5kWeLl6X ztU++uPr_IWf}bS*>VKyp82I@_+r4x3dS3T;&%Q#-p-G*3LKyG4PV4dctO5rWM$Ut^Z9uKkgIJoF;kiU0B)XB`y>r24_EW_5=n*_}_Ohecb&|Fu3xXZ7Ss z>6PQyJWT=$^$_hwhpgh`A(dV|?Me&U49Hy_)|kk#tWwbz)pTW;-F`dxo#4`CzmbbD z`B@d~8>~xr%DKSmT08`%-ix}dU(+F_<0`bjvi`Nb8*JzypQ_=_`$3nR9!aSYTS3VF zcFhh+b3EXJJd-majqQ2W%FUfL&XFTo3S5QF)sV;vGV^rG(b1(9%rfQ_FwI4~Bl$xZ zalCj}j7lTvS07wX3lc7SOn!LIuKw!pkfez?n_r!=iviOwhk_FLHIfK5zeE&yF)$i*%6U8yqqP0?*ZvS>!+(g4 zvgaP+dg!i%uBJ1k)umhCQxbEnPgn^x&8KcKgUmM&($%E&{Qb}?LG)+FF3nQH%N(P1 z3O`9^L)dE{{igcLHky7_P!76kCxHc9@E%#FnYbaZmb)xzFgUt6f*aald|!cAIUZ!( z6Usx&x7NShL9Q|kdHeG;)x%#rLwDI=>|l{7Y?ZK#=JMmsZnre8dg_un>;cu3rQ9VX zvF^co(Ot}C6uYPzP^9*Q-ioRr;}drHKu~qae%W=P)y(wlmcW7w;O27nIXj|5Jxjr& zwY1?nUYF$lt6$~gu-t}|Ra1Gj=GsbI=W6uPBu6{nHV*z@;*MQZ)ahIbg!$`5uQ$&1 zyUX+VtQ>?J!x1Q#OgF|5H&J8@xnK3|6H~h_d$9E5ny1-ypzYJtM}KoOAYpj&qE@;K zFxm!kqXEyYrKx*p9}NU0o<4)qAui{EO%10y{<>x`rsFeOSthsI$Fg%{no1mDERZW1 zX|HKlZ?kw#`KDL;JXvDODJYiCZTFGhf2ZW;gOU32#^YNwA9Qt@g5aVAr>YV`2Cq{|Hi#*FZ3Q z?aQRk$(Fl%?Od0Dz24l?Ev}{7hIOKSDEkfMFL%v`yRRkwF;aXWwcxRvQ~@7Kzsx7B zs`OmpS{iqzZqd~ZRQ`?b2(AWW-we#R9#1S(c@jtNfigh{<&Mea0lmIj+^bRctP&oR z@0(E)c_KOE`B@BkD3ONl(2Ce;7F41}sWzD!v*ntw~7guBfRPA2_<@Rt%jc z3=a&Ukdb-OcBf0dIr?9V& zxDGL!-bc!FEi)(|-`LNe%Po~_o+;3OLXAo)@;R)+aw|6VPV~ChaPpK?lE4;4CtS(H zW&3^3{OCsV1ZbFcg;luuUVm=WOpyIRyn%yVYo-rj_S8HbB^qIH=CX3@fX^&pJ?&s^ z@z9qh*j9R~V!78lf})5!Iwb;bXXJLd5;##}=IBUGDOy?3r>Oaq%|lPT9Vz>5?J))d zqctHQbxkp|FaMlxfP}e4Odi=ZzeI39xJr& zw@HcXZg*+-Kia}+oiP9gT=axA347?c9q_=I0(>gAn&**S zKb?GSb1L|Z<|M1|wzuh%CHiOxOzyT@8f8slB3t8S%wNE_IO`&btozsaV7;s1hJj-% zOG=dT-|BUDOFoqF<8$MXX_S$_XOem9YR??wU!Z~leqK@1YS?XON;MP3#PRPmdh)rEBjQXTD_SiHR% zWMGA3eiosRcMWQn4PX@!c#skB>pYy{R^LgdZ9GuZ8`-q{)AF2Wi+)GNh|dT7+e>c( zOs_rilm*B z(Fw|u9j|zRg&VMHuGLh+p=XW<7QYNyLxrrPzUn}0Lio~_N}WUdydWoAu6y4xjl}vu zUU$jX#_@SIUCQe%U4vhLgPPe5v9L<-u6nwKMvUc#W5i$~ z_JSXpt+J02zCyXrQPx@_H@vF)T3e>Wkr`^AC2tAs$L{^DUfZxtmWk<{nWowKY7|1; z5wmQ~qOp$}KJ9dvr6ryh#3f!lYvxC!4r4)C?CD zxyx0ZT&~YlglAv@cP7z7iHp5P7tk@BMu-xIi=09)bL->|PBv#f1KR)^7vxO41=%cW z8Ti=gw`@w20q3Z7+aumk3Lc(3?&fl`@#uxu)(_V`q_@4u;7idoS@l_%p=D#v2nEs- zEFPkc1>t9zcwtbH>5Rn4>dUUoiteHT=IFtq-Ur*VmU*ieo*x5sJZ2~-j22r*Jx}7!h-zkl@pp$!y!!jl*O9T# zE;zS8hLjz`f(z@?h)!{*H*3yicco0It?w;gaNRqE>Ilg2ZF?1o@uC)r&xW{tW5UU=rywqy>#$&PbcmrZvX*3Moiq(zI^9C5D=n%NB{! z04eSlpF<6e5)eN?&$M-nU#%_C91OMeF;LL9Alu zPJmKc3~7>eQ2VxTug{;OAmVd2y&iXbx6Bx>Ar{FB@a4XMv+i@%{gyOO< zCD#yVy3KLT*p&%`UcY#51yNRD$hus?14Av4jh8!<$o!a73$Y2Dr2|k8YJftN3upwC z92SL;UZT}+X}VY!mmWDYuxFiboy~nCSKXg1b{2j$m+@rf`oUWac9M=mS(4#3$~WBZ z&cwo20pl8w4Q9ABTLy#DRL=c@YoF!BTx8g&wG!%pz`T2)c)@K>>fBhz<$moOIEuH*J5!pY_l? zG$oDY%-OwpXC!TW8du*OET~deywqZ`8z+nbz&JWt3*e)ei<*619kX$KF*(#is=gMY zEYkC^Qdze)OV^baVFf=3EV)EzEA*}CsKE9QdB;hr#7=`CgmsxrfrDq?B)n3xK7IaK z`MY7c%wt``tJEcJwWR(|>&|Y0eL-n4;RRp%_xN_sPImKxpNVurr8{hkM1qliUgLDa z4PsU|8^w7_c-GrlXt={&*SmH|&fj}l3FS`VZ6kKqNMl$ze>6S3`f|tn7P8L1 zee@^3C;4t?O)4Wv|V0WIKHeuVlX?6_2JbD@0ir)7h9+*xgtzv z|6xKi=Pzq{Op$x@Y39>>uI}4g6j2ifH$L)1BDyZ+Gfn7!G+`@(IcKke>U&s7Ck|WwcnFh8_2CzC~F(pi?+biO1<8N@C^wLj%d**fV zu13_7v1h)zLsD#x^6DIqZ4L?tta#P=L_cx@``h3D-I#TBDh|4QiP}XS zGIWIW*W@o;$iO|5JXl@vhLg5)_xp}rAT=uOXKk(Br8|=&15Fh&84(8b%Aio7LD>xI zN;tT;Akf^oAckp$OQvwt0bA0;{930g%nBF4xo_oR#oxz%IrL=?cbq+}JQQnpN7Z}> zwePX2bCx*f`*MPbqSWfNN*v?SpE$gqS2l74tW0~lJ z5@%P!y=w7;kcR6&e)7aJsb^Id;|8P23)B10Bd6|?wI%yjh{bSPSBZuGqWpC#wp{I$ zvxGAulj>_W2c>VS3^&*`G=5>>=g#RI8=e>2%wmt7EHYwXSL__X%Ng9teb{X%(S82T zm3xMPaoSJYQNnMQL15xr7bc}^jCYQ2pyUBJS?^u@ay{^YoQGJqp=%whx(qpt0m1LM zoALImP5Z0sqqcV&aIbw*q|@7Dswh0yS(6c8nagF2QSR|96w8z zF5i|0(KxS{TdkyxU9#%^{TyVP<3;XjF0EvKIC!x!!HG;LJbsJPHt00ko2(WdsGpAW4#gy3&P3yF z^r~H)xe*)Np#HVrn)jsiIyL~XLWtS63H3iraAs~+Ta5(iAYlez-gjbUceB^K8Xcj> zGhf0Q`JD|&&P%l7h|q-Of-+%v6P{fOT6YiqVsxf=^^b~ctO)K3+bX(sGnWuIzjKLY zUMyc)m2$z@%cYvU_Wr;Dss8Mtxo1M^$Itd!o^<-02$r#GYXe_+XITs zL7c5+R%?%-fyS04wsiW%JEzc>T3P~f>qCU$k<)`EU1Mc5(}3P%nV>fzZet7m`|BCgR_hLr364G53QBZSmx0rYm9qRFamhs*s%0lO7Kpmih8d%W3 z)hB9w&*}GCG_qc4y+@b#q0>Aw5&r<5orHE~>bN(0mFEdB+=WLK3;I2DO?ZK7sHtfsBznkdtTljQ5Xcp+8g2E-saE4uErJwkKL|h7`Wy zY=y4z5Xnd>WA-fxG8>#}MMzM?N<&US;>(+eU z1!K@-XSt0syW`3>iJi1@M$m@{tf3GS^8#jwN2=)A#iC4=k>MWTl zZwmhZ*gDI&D8GJR&kPL`3P_hhH&R23N_TgIbaxG)pduyRpmcY~&>d1k_ke&jNW;1P zKhNI#oM)f+@{t#db+7eX-|uyq(H>?f2;~+Q%8(9-e|mwa%eJIWs^`c*SJ znm7K&F9Tmw&3((14kvh68oI6UCWU)N`UF0C^ZO8^%pA1eQChu8^YdKLb5rL2_I2{5 z-}ZYy_E-{ZJf>Ba8=Ty*WzzB`UY9W*v(2v0lNLG(rRq*2M~ky*HL*8hxAH;i2P!`~ zfWPooTJT#LJ*R@>_bFz4>Oa07K3QVfUdioh70L0Ox`?Z6FRLfA|JpI%d&{`r9dY8| zKap&Qy&6z)EInSguX^;pYDA!2Dk78a>aCoGL60!gCY_I6X${W3hrP-MxhQRv3xm#vwNFwupdY5cacb$fuYty;!6^8_#T1=fr4X`~t zjjwLpJ=ttf{}a{AQU3<{a|Q%u?|lrCPnCDRzo!Z_@61ZB0wl=zLDfuO`Ud^A-t29C zqu6XCp_kt2Bqh=Zh@U^u2mHmRDF_VdE?O-^;(#NyqBFPAPo3+p#ov7Hmh*jn5aD{O zL*F?}%atBzYjy7;=Zl;BCnf0t)+Kpi=_2VH&d`oMpVhniHXW@A97uXa=b=`(S*BPV3wE+ z&27R4gO(g|S1*xO^chk7pQbhTate$iWzimsm6^IH3p>j_ae>$$?uf9&kdVCBQ|FX1 z+69A*LnL3ZaN)o!JZZ64gE2!c z+svNti@qHUr^!74b;LgxgsvG8qXUgX$hBE5tGa%ni#G(UZ0w18*Kq@7>0yNIb2%v? zE7j>gUZK{#XP}yw3KR?Ed8g?}Utj5=%qQKpJU&Ubzu8Vr!4R%VBQ4(HW#mqdJwXN9 zcAA|3kV+0}*)8=|>u~G!QjvY$PZ9R}+L7lpL@kNSdcIj_`NrPO05uQ%5yRsZlk^R; zaEvg}akyc9`M8Q{>jbUDbHa7!OL0@b*!F?4|99LzJl_7CU$%?K3KIB-VhL^{6xSzi zjlw1~w%ry~t}ol6?ri%Uzo z)+P|`)j5azqX^G*-lr=REdjmnI1LMTz~-l?p3o0pD3jKcvvz)KMJCX*C@wFh2ZKwn zX1-KnfYph#d0ucb@wRoi%>W>yA2cm=Yv%5K78A!Em8uTE)d+g|lJ7Tlb{oDIKVp}5 zKggg!Rdi^3*Pv$Rhe9Cm={ie3#Dj;u^%TonVBT%~zemNj5Y(Bg=(6|DC)YT{I#@wh zFabuln_uOA^ll}Jy|n*I^yWA(&i1RoHCcA&cGGZ^4urjO!3yxlOcQ=+ zvid6gQICPXZSczABfRRl7$wTWEy)GmcCCMgI)i9`-6KdoSd#)su+^?`W3rA+f6GtX zqI%)zEL%EIeu#cI5;g6dJ(WUmNuz`u96~{7B4MND#e#k2_I{ho+hB05P!jhAO{1#n z+4;!PJd%RweXTu#6e}xR;)T!Bx^ zH|3wXs|(b3wWJE4s=wi^HFG56a&40}h(RqiiT)PsF|A$}yWVEvAL!43Ri2nw{%xZ1 zB{wv9F{fDC(1_vrL)eB`Y>V}R`jUWaeaT#mKz_iq=0IR=YSW+VW!EQtyuxiPj-fVM z-0p7(=CKx?vs%;y_~kb$yjpJK%@AU=j)}j&omAyw=OxCo#vy-n>38g7#J$|ruCFr0 zwj^ez#YGKRKW2zMH-cT3i5rW@&a02O(b4mjUMuZrInhFsigwr1H>(049Cs4iS)kvT zSEncF+BM5f0xr7S=}>+N&rC5|NK(h^wbOL0dLa6#0;$*uhw2!UNR*@qNCXY|1E9L- z_g9vU^w+7A2Vv{IXbbFncXdSVA64CL@gUZC zSSuS3k(egCV{1H#%-%hGYueSaVe~<94F$-!AI94ZyM*`C-G1~@>hmY0C`D_(U=W=Y zB>&-YLQ^lw%Aus6*z{O0=SM2D`LOE}vMjYN)2$!fY#v#e%^Y~Lt?~sUt!}t@Q0G>e z(@|@Q&1@#jJ(?L6(~hugqgC#mRHILM>}cSxXX*2{v@42bjg8G`)Xnnt*9rY2l)rvu zZ&X#UYFz1zA}5%<4&$y#7>OKo;HA`=9~AiqzF!z6h^;vP@H0YLGH!t`El7qc7De~Y zHhc@$xqGNyeK7JXT(9>2fvhyTmlB}vaNi_hkGRWLwA-z}D0p?q7|&(5oBB&FQvY}( zHD}MrD3PrA;;q&P>{C(4SS1>q-owB<&1G%UAt@>ug#7|Y34B=4KE-P>Ygd@j&W;9I zm_h#Sa4o3_`366t|F~K!L+iIQ%@0hdFJjj=iblU3)%7U0bd{CB&e#Mr;q*rDC*zx${a z%Vg&&$#z%IPZZvHyj~T@TI~r-D*HBsz`gkcs1Ij4E@@*P2cye2nRhD8PHFLn@ZV5< zzb^Zoz~Rg~bVb)~k+{FSd)uvrzpyRy@%n2Vs)t|0uOhyTJvNbpRg~`d3IEG+C#y37 z%I>)*NT_-K-jQwGK)DU%{u_Prs{ZsA-bAaT)AI0mvL797Meyj>rRb!F|F|8ZP4ZI1 zVCWrRUeyx_hAiB0;kJH{+|O8XbV7%&g8IQDeOe$&hdiQ)7=i)KuU&bKNy(U898*3M zS{}5C@bt!XshblO1Kz9xPChdr8?7>VDB&us)ErAhS-+VMM~n(5NV=WGC;Dr|K0#}MMT~In{1Zx! zg%aa~UJcyOC#AK`=87hjUjy$oH4cqd6gw^a%Q4VL;w-|83HvmNTG;C`EW~aW^i-ja zmC%I##@ktQCIjV{Lqo+H8LKHp?yADH@+{eVR+OZU;>Um7n~72HZ&rNi_u4_n1~9ld z2dIKK9lXNYz%5nQuK{9yoMnm~zGPKF zS5H0}vo5f0``lvW8R3Ur$>-K!Sq)a5Y6h>n-|G95kmYZ>L3O=-&)5$nQ^6rnsyF)i z!M>2dIls}x_X>Oc_hvTob`;^f7#HXQ)V5VZUXNF)Xa^d&{$Talzc{%m!bSX+dkfwa zQG3V^$Xqztszx8FbrXJ%7&R^%S?q9L>&|r$33a9ta(jIq?rMkKcOFe1t7ep@c$*oq z^assVqRkos-IyQ8a#+cU48tq|^KewnH?TAQC@mpAur71o-CDLm5e`9qGyO0^NEWzM zSNM?foJB5{fYcLOo?Z9iFS@ot`-g1k!?%?M!HH8zy(>?ka{D)jRr?Gzpc8ng`PD*O zp&C1-Fcf>hzt`fZRdnn6d@KJ)UA2U9Zm%lYv34TrrNWfflyO-4uHQZ!i;8zCM2sAl zDRU{9PjN7Q6uM%#`HE@rO~iF^!@KLSJm>(;xB>K~%h}5XlK^=eUEx>BLmcxpd!6FO zQ!l0uUoI>kg#BhUq*H*7yv13yu6u3#(ya9Zd#zugoF1!H5WJ(Li1#}pee_~4_%Bxe z2Dt0XZp}9rg}8C0sgURb=n~)iJLjuDF_%&c3d=_^df>QrD(1;Y9#5eI?iP| zC~O`r=^rJW?Fg~RhgW#{H6-gHI#hKtuyqk^D+Rj>(Ae`WdpcCWn#f^DYawB0s6l}I zH4+<)DZe3*k@=uo6QdvK-EN*O+b$lk-W0zkD86>p^O9R+5V`OWyLl&eDdqKup+Ypz zbi~Fx-}ff#x2CtrZ$Ik-T!S98)~-_;sy1l}v&9a&7R9{vtR-wvU$3w2z9L!>jqh*A z0bhMszX#?c*fX>bxS`(xz69Kt86CL}ZMjJL4Jf>DUFeLN-4`Yxph_pXnuL6|($^ON&$8@sfn_PSZhiAp|Df z;Oh)>b?gnj>A8_wfmP(CYT!r@iSV_$yI$6J&Ay3q?`-KsJlP92Gf}i`u5nEi%#Lza zNrWtf{igWy6ZmiJwJ7F}9~kX1{gLzL0x6YHf{B+;1MNROVV^`}Kw zGgQgU6xN|(izFy z0jn7-4tu6vEHMJveg)8%&(s}R1M;UF3b>)WiW)HzE z7mvN|uT7DP%XJ>{Up3g*SB46_*Y3c>Xp#@#58Us@Xri@54NH4%b$}P0v#{}L-y6ci z!TArx+V%SmSKCua=C^Mt4fi}Q?!01M_&Asqwq7l(o+bywql8bS*JnR|0-C6kQgMv^ zJ%|hY3FU(vw4-MrB2dEs1TW}=xVD}yD2o)1Cyqw?HH0;CBj77dxLn6lmI7Zz$;Xf< zAWK^kL?b0!KrB9X2I9B@kv^F7fn%!$1z~u66tq?#EtX*MpHW?Pwu&VuOzWZVo`Te& z;98T6DicOmV^JI;*ALbkNR>#Ut*GNtc}aZ}8}O`{;7{sVPl*m`_D%FR@Rk7%{iaE$ z=+*6b_WK`8w5F+vMHuKEEfZ2~#?X!2v>v=2V1qRwBtnVS!AOSlA-F7mN06-aM`ECs zpe($z(;cP|r@^$B1>dduEO-GWc;(1cVK@DtNzT-*U#{zmZ8zyH`cYb7U61Agq4Oxw z>jFr}vDw1Re!@u=c43l;F104-8S77bKhuq>_}J^BI1GeQ;EyxXHhI3j-K92%X1|j~ z$4*-Zf5R_Y3{fQbm;N^Mn9u*t=IMv=ca5rab7L4O|3ojE3o7-D1#*s}_KqaULP2CM(;T;m=c<(P=Ht~FP zUNqLh)@&1kAXpU5TFU0T5Cc(6nkOi+Oi@DB)+E+Mux$2^pu39sl5xN27|;S)8G`d^ zq3B!J!S=78ZG|5jr_>fu{dVTc1aOQgYV`plNgD+{2waw40C1>5BqSd+?P+st$}R(w_AtY9u9s#YB{C%mGv(xAKj_>TCX6_ zNm22BY%XbSa$Kx1@w0G}cXI@=1P1t3d`)iyQi;}a)A4)i%oyJT&iM*cMRcwo&M#cT zp$}Fk;VXA<(OY?ya^pTB^y_TLPMue- z`iPg-^)20EPT5^4e*cdM>{J@p>j`+4lVf`7HBPcS=BVW>R zF#;|GD=HjkBi|B@kmT=vXdFd;&`=%+du~9!L39$Y16p%w~Q`_2@Uq=W0QMh`SZhzWwX!!5eIT~i# z;?L8?e^@|LKo3~fuwL92E7N%~M2h3gC6oO1;D8E{g=?I^pqdRRC8W zuev==%o-ZngsGbBc_cW|bg4#FXjzbXFgLi)Vgzi=OHp5(0U)b^*}=D?e+<#I7|%g- z>kv_tEym{$WlYPHDxUP)4DhJE#xNZ*S-AG3Jw=+V52m=?B2)3IqmvQrJ1+V&WXkeK z+v}lmUVDL$&f^dv$qG`cS2=u7tHqsByYp}3dPcs08isQfwX$g~hnLMV zH(I#3{1GENFAis<99E3wH@=VAz4zFXlgfu!e;R_m$`-9i3$X61W$MLc9PJ{=YWH}| zvn(EP;5c*~u1G0JcD4gMHwl8&{((p&7Ma z{>RGsZW9x46j%R0UW^D;s6&ywXu0-TZN?}7?)e`AET^KXv8lq_^eOc+HSs5;wyy^O z;O}&(Eqvh88S*|H=V^g;!sDshEb~ez!%mI@dRAP-e8XX0%9#T6z1F~Qf|?ofEO(Wi zC-EGJ8TQ%%J!OvQ8XFZ%?-AP8520-=01m((w^oP{39w!)Qo(S+JJXy*(3`#h4Ho)>QKJejhJBcWCD9q0r-hNP$c~&N+ft3mRl#> z4l{z+vUyfM<&3*QxdgE2L9psYl#2Itk7xU8ovi@z1*cERP6J_G>`4$b-qy#_XC8!B z7vf_NSHbbZnG$xTZ%w6)lb_iC zK9{(=Kmz6d9zg1u{_C_6czEoIMo2Pj!0P(ti$zDv+bU~!=1&HiZK`17#p2@)sM9)x zb>m1KBeism7QJ^qtp=#4d>>r19k6fKF=&^z@3jopTxA1!EuDB}Ed6Rqaf+>fX%TNt zpoy^$CU7)pmJ^H!8W%`#7u*oPtJ|wb_wT74=C%7}nGl;#f_P;nd3P;f_9oD#Xjgq< zSkzfbTo{o;D@yH(Ej!gR$9H}MJvIo2!wo>maxl5HFMJFop`P$sBD8$SmeY-mRWZ5_ zOdbv9IlojZ-C>~D%C(mF5(#v6*$=wzKu%lq_Lx%i7VMrcA($?B;yeeN(>A?#WM3%` ze+=_Ml9bA;jr5ouNaE3=Cz1jIVc}#K@o>JM zeDmTopcuHy{q%YR3mf=*lyCh4%~RT4tAh=AT)+hTf`BbJ-I+$M9qYU?qt=ufVH(T3 z_1V2s5aT=a)ti?FVU3|RM4RryC79)o*ya55WhT&zpRSItO?Ndx0;I7GQ?){=Q`*IQ zU(Y{wHg*d7i(9s;(c|Q?%LT!csnhiTqTYc0+WxO$3b^%qW2Q*}2vC2je{r?wQ4i)$ zozV#Uq8`L0)Ce~ry1=&}p9TCT-GEa{-bwxg#W0)3xu#B94M=N(D(-0_6QV>%(a9fE zw;0eR8CwD5ZeW(f)Q6?2rmVQoSkHEcdPF9G%ZfhxZIb_hzBdTR3w^e?d{U({OEo@5 zZkF{g>;RU8=NTP_@_G`jqI;r}{l`uT(Q#3^Oj|k3yT}j8XqypV6yc zPc#V2d#eWS4eazEH^{i2Y^)qe=lp7_M|j5VEzY^2WXKzFQUXFaaM@)9d-8~Pj75E* zn4P2La7y%@KLPa9L2)p})aEDFpB{l~r=!9(o5eO@MBpSa+!-d9X^l@XgLWRRSSjAD zkk6@%&(AkL#>)=ZK~rNBCnQjstCZX^`~J2Ft@mT2ni;`u=3dP@6Or8n41q?Q8tfC4 z;Y|94#nW?~UCRdR4H-PPWqiwz5^MKGV%eULNa9(D^W>;LwWuRYljd(za8hx&~%H)#jxwl=_r324uJr8pE9rG zM_A<0XoN?SChr zIJ(-IWTR(2jjXFhFWN8E+vGx0f-)l{0p2cG-&^ojX<=$e%rlf(X+0hZk@*opnB-Ou zYGDl%;jGzdUHZ9iDZ3Fa%o~62$tu-FVQ9-OMJCD;yH=awF8Wr6s70pM0t(9gnEt|+ zYStYL9sxKFMfHVC&!R2ReJFFx;W8V9BSIpKtt@UX)~%p82Rv&&X`LxXqrfcZ$D zZl(P$##!z3NLsNQT?cfvNdOiVUfX$XK02Ou9G>gd`3)6S6W=4ab_VEY{@=1k8x2f- zBTiN7wqtBGk#CPzZcE>Vt|UB9yniL2ne4NF_1CNPKg8`=MNpFV(Lh?<#%AeD;$YiJ zuEmm%)s{mOABRejAl=0VxaQ=`uN%alzQ`faT;{rQql+hbe{F1{_JSYLMrM|yi0f54 zCCiFER@%=(spr^v3PQjTZ*!5>4yKV{XKSllfo%SYANx5)oI}{t@e@kE>!sMw5YY+7uBxqoSTPK0pqI zXuBBUdOLIVZO=@zv$;@KYwF*bm4M*R#$Qs&!vN1$PFiz^AN9QJ3;WU-!X$T?@BB++ zCItAlG0Gv6e9V~2az6UVc;m4~`@RVp0!3$pk8aoe_Vp!AQ;>r5q9*@=;TC7hs3>BQ z2fX*3+UVBg$oKm0Ef(f)SqztdUHJWK;{Tm3umSXcsvK9&*tfM30Als}+po)rE>pa< zzl!44HiiG%he!RHwK}?QWisg_3nP{-oB4grPdvk)Wg`02rF>j zy9ep>1*1^AiWE|vIjg=fhfIQWDeI~nNgV=-+xrX8-}8xXUM@9L8c>1J3ue6eBb92bN8Pk0Taq=M4r{PJm! zB0o>Ojj)Pz%iOwOJsK%ipF8)kvSt7w?7m1vu3kFxz}5wlYC{@|N_0mS!u_vh zkex{ENPg|DJ^kktbe!Ukq|c7W=M?kljv~r`HD^5&e?nlveDD;xcXYg14U#W&^*^Vs zXNHm1+tI6p?fc^xCzfxy22O$tUF(Z(3nN9J>lf+M%)}gcp_xJ?6%wl;!yUXeYzI#0 z%Mea7DH3>($Z+kgFZ|>@FpIWga?3~anwc~> zWr1waF?bNGi+psY2_E(vo71Lc$t+txKb-55FLDPA9+MN|b0=ZUQ+n$6^$4RZdy{r7 z+Jb1xVi{@Nl%o_%RObEGrLViN#jAo566uzcOp|8$RQ-8R+$j!>M{^Ys?w@S6pP!|l zKS8v}$XQ(XdslhQW0Rc^OZ{(Mbg}p

(12)o?Z%!{u2{1tDY!b;IkJl2prGW)w=i zWS3YCy&$djm?$&gbnyHd6Z*?*511B~hEqfM6J-yLTWf4FFdZ0Lx0}J8{A~C^?cSKT zzj?-(k}j8j9q(E#Hs(-P2%5<3apu1-iyNtSOhBjYEyTvGl~9OOiDa2cumOA8AE$X^ zf9a*_hQE(|tjEekI=hJr4-ZG6saeN*%tH)Egu85uW>`RWt{X*^ovxBr+!YAczU)V@ z^8IY>%tYpKl`)t%pnTZ^VjCd7j!LrA@F8{Vub1?$a@*yY>2QuS6~W-3!(-cjVBaY3 z;|aaIoZVh%82Mkt12V8zsUYRL3tZ>=0@;@YD7m0U5nPNAGmnOkHNdwB?Xfb!xDEC; z>$$7{;>Y5i8$PrwssmY6I7;vOQB5PG3_|=to>S2m|1C@q?++{j_AV;}Cwg~e9aPDe z&uKOoYGp^c^(`eTYl)ASms-Ph2a-EAmK4)PS|II%zAHo=kv*!k=fXL8A#eL}kIa+s zcNlyzy-Ym9TAAU9gwv)|cLsbhz5gng`rYqGH1Cwd0*>I$y+Zm7e`=l%Q$+vygx1|V z>Cfrmyi-Fwg>zIdBL34o6(7~>Qd}sgy>2f$KwEsp{xpd2a>JzB;_U#`J0H>azn1_& z)n+EUmjjLAa#zp>EaC>$b5OE)s{FYFOq&Vml761FPPLEjR5%UmBcnful;iZCLCKn! znQh#?F-zPl0mwkIi299-SQ6uscdPqQqT`U{0o@TlevmJaFb)3f2)MPy836@-j9m0H zNEo~IGdnyJ5ZBK0{>LpjcGgaI&8*&cj~fez^r~JkVw6=Kla*;x7;D07%wt@dQ!cDd z1^9xi?N8n<8_^a5G>)*sa0a<3BYh73^Ni=na4q`P3Xey9OWor7z)sszSarAZ4UO;%I~nwVncw+>&xXl`Sy&T3(i|^@f&hGCa2VI`-)_u zC$=jB{ylU@-Iy=%m$YP{5!4i)SibpE z{@X7&qzc}7lwBacWgIY``yi~rNq$2g?N3T{N}Fn+ABrGuchFj1xCToM2VVn?dlJR? zM~68l6d{ssi{YJm1(=m8XHtc#TytV!Cf_eX^R}9}R3Q|B6~dY*sJ^Lup;(!`w?{20 zep=U*eHoVOOHi+YVD9-WO3ie`J2)|I3D~J6%Gnv$zPeNc#$%?CWS{RaKL~v`{h?{u zD}68le#dfTwy*fd?DJ1Af~kTv(LT_IoWF+22v%8 zCa~<#^gLH$+CHzI^fr)Q7Bwg-&;Ue2+J6U|Jl1d21?j9d4pj~cph!A9q?%Iq0|Tx3 zPlcfOlaBk$76BW0YFf=TuVHo=gZm@|5EuKcoI4Z~>;0tXT>f!`ww)k7O<|n3vrdxBogdCnn_2W-vwIHA%WNtSzir z&|A|epjBIY>ig%-vxwLxRODJ3-zW|9l4IMud1rqBd3kE~Jm>Y8!IzXA2i**}_Sz8r zfyQnPsvJ3|0BbWh*%yQmeucfV08M;8y9sRy#ez@jh%aAIAoI@6xY@ch}*0ymXut;b20e-wR5)JQe97Lr(!w zeS?5g@l!S}Wmm~_%Q;~Yr2|z^(l7RhTl4HVVh;5(-G%chd}&?@7AOAUpQ#zU;^2Aj z@G0J@i0AxmGse20B zF6R@x%y%J*&%rt47NGZ$KaEnK>td`XQuYtArXGS57y<-zMizjmU~YWiKWiTbq$p@f zIH&22^Mfm%=Ch^hubH9_f4owbjP7X%X;aqF3r|CWAqfT*Pu5Bn$rcw9hDh?6&lgrn z_p={0Y(d7!$-nu)6RN;g@6F;YJLN^hzkop*F8;l5^hgPp^%^ILIWs_-*N(Q$=m+ea zqqXdbj;@^TUlWsEXxW$hxcTRia{kU?OnvEspHR5+_@o-1`m5qUGG$*jQy^b$dZ6c| zo+{O^MQ7ZC9_;)kF&2W%O?C7-=#~&pL^PCH_;VWh*2kL7YO{1iVR+DJ+78qy@(ye5 zkO%uFjZ;exL+RG{2sq`%yx|=oj}mbVOY@D}wDY(27!MrS0eg3V)_#Fw+=bzA)ad*F z+yzacpZ+qNFOPwpus!n@R%O)dika!_$3W{8hv3b?5E#b)Jx|LH$zF1KSuc`RQoM;r z%`BO@!;DZ?Ql9AmIgZGk=Aa;In%^~v5~dZ8D!oSFg%p{s3M^si-;pyhyX5rA%lZBR z8UmOZKSz=1$(Q+@Qv1At{kVtIfB8bIuFaUbHoWEJ%;_4YQw3E zKUBf@`b_CGdsH(be`)GQ!vxyf(TbbknZ5o>iL_97FCTexM{8~$)#Cz3ge10_ zadAc;C6F+ro=ElH^usk+aF4%DQWGA*T2yBg5g7ZKVH5u+N3FAQk+ZP4h`%yJ?I3=+ zt}}zG)5B<5$LB0!CP{Z}KW8sToq)QIO`E4<@8xeC_iaJyslk}bzS+Z_CZ$P@OC^EtIWg{GpgZ@mA}XFbwKR462bDHhD9jG}t@~Ko89=V9 zo_-R5=VcCK*gMHc>}UDci8@m`X>Sup;=pjkwH!{2MP{=^?mA2;jl$r@y-9VIz53;k zck{;5%w|fdI&Va6Rm*?&WR}HYg;)+|*xw-U67vcX&ur`2fcI2DY5m_9erAd;Vv!3; zG^lP3*W$_QWi91c42`hXH{k&4ztOgNu4ITGuXe2i0ZGs}Ep+u$L?p9)I->!jZ6Zlp zF0r6Duy8)>{gBk{W6r5}x!UT5CB3BT&f}W+?87E#CE1YlAqGooSIBaR!hD!YOEyf; zHCDV_@K<TG!G?T>$*R zHws@+_RYkBl35P>{hXjSfZ1rY3XAjItwSyf{XFrx_Oa1mIKs&$^n%On>;aq?0a#Fr zB^emrgInyn;$T#*+k5}gZFPChx8!UO`;))&J+JG>^J`uWckGNQ{61?)?DJL(UJ@zWWxfTUp_;Ry3*Iu3fB;gxN)oOrCu0F+{eHsRb zGlKLeEp5QuK^4x41y5H$+fIjS!%DP^(6gQ?kGQ$oII|)7NJ?ga+OGLU-kqiwhIr52 zd)Gr3rB&wC^v1LXOl9m_q&{IMSX_guLuB|B;qCAZz3Q9paz(D$~CZt78c7Njj&$ZJW#d#E)>~H+F8H zHkC2KcYh>Ty$lh*q9=4fDde7dqlP$e1K!ur6~735nd?ul=1v~wBf(CBPvx$%h|(!S zM#;zL42BCQvVLYf%qDDWUM$6@tat8y{|G;(oN!Jjp20A)*TaxIRZo`X`abpI-%x7@k6sy~unz65j&b8?HY5o^61(3>!WvgmIo% z1*NgoSK#S)!7aoqvGNN^wz)UZUKqXdECj$IL`Slc#<+!wdd52dv4M!PIu{L7!uFTA zIxP!28rTJ7NP^j8FLQOD?ofd7!5uupInveP>T%fn5hYfkPM>`^Pb4QLJqaAm(nXBdGU4Xrv`pb<%2z(MW~hEZ1rLZD>^?b4aaQ5{Q+yO~?D6Zi zhA-*!FA_ZS)aOtAop0lQ&!^1??4pBG8=W>LE^-a%7cE7D_Dd>jUg5@&DwUyiRvW$= zpK-8%cAy*;e#4bF<2Qkwv~VipD(e5;e|NbTD$$UUJa!3{Sd2x);ZQ!G$0OKoF$E5a=9FB(qXL<5md(DuB!*|w zB~eQkx^@>Ar(a6|gukH{1?n`2t(6s$y?;_-RE-~$h_cMfFQaCbu0G5W_qofr@1#5f z7ZNV=_ATn6Dy`A}C(iRXqwy2SWUg=RHHs+{Vj-v#Swd1iLidW+ebxx3`E2s@r5MxI z*L&+9;Lxxd++Z8+Ov5}nCl^+;34)pHESSaQ3*2JaT(^1B_;)X2w60kRX5+tqHw7X3 zXc(skFp;YxF)W88@m!P4IS%daKVL{(0FMlgAJ6iH|6F4(So&t2oKL6xV`|e(!gOLy zR%6z)om2zj%Os;LT8aj(JM&v)Azi|7u|tN#(bBk+s<+%l7zCF-J7OZ|*VoevIhU;@ zNPemSdi*w(!pLdA+os!n1b^YFW5W&B)hr}ZD`7Oo^G1Z(;IKg@-SciNkja21P%du_ zf2nB=-CZtct4q|x`yxj5Gll1|`Ob0p9`zW1 z4b$9zqNsB2)UH(fYKIaB&gdFXR?9ov9&Q6erF*_iEe?TPvtF^A;kNx%?(~mfqmBle zmGO*mJQMja;$OIy4at;f9AycIX^D#wZdrqWfo$~;W%*%FMhDg6topGqc~DC7yP)g$ zWB>QNKg>ks(S(1KenNa!5JgUsyy^G2zQXgTEi@vRz7u&e7TV6_PIG8Ekb#iU} zWs=i5O4qLI6c}rzegaSp$gz%c+st%DDmwXJfPN++%0xt=s+KN5K)@l@C)b8QrHfZ!rObDyW-qnpwpn++rp{g zVF%R0;r4(B?&fIiN9o^(pZ|hc=XY(k6+h1}XDR!|>MvU3DC<@hvro{lmah1ROCD?T zoIGgUm*ETA_8f3qGTjkw{%l-bH##BH$P8fj>>Wl39x)`p0F6 zD@T^k{L5n4p-PoxQ6Eychy~b5QWkL}TqzXac?4#M1LqI2^fhu?+N@$xj(yYbGr z^#hbF*ip<4nt^fz&ds+?!xJs+I#O^+EEV7*P@J|`hS8v0E zt48m8Np274Dtm!oVi0;0hFMt@;5{57_80w|8Y_=N2@=EG(v%ceo0qW0hGE*FxFq9A zuO>l)o`#h@UB%WeT*~K*p$}5W%Y!+Bf$tlG{Jv2lc#P9$QCCOcAnZF7V10Qe zgAMM6a~u!*R~7xQgtng%d4lX3)wbqS)qu17PmY$=Td9SF|Jc&8gSuwvSxy-Gk4Q42 zF&vvEpdK%p2D=)&%+GS(sy6p}j9k6-kTgy&z98Rmvt{soqs+=%+bQ^Z_3q<@BzUer zS*Tr?JSF9Ye9U{4HayHU>9H)Tw3xlAlzIVW*T1aLnB^#UMYPV(f0wo{fZJloQF|ZF zfM~LGh*(?-z4@b0=X7pM_sibX)sqZbsLVXWFB@CuR?kWG;hUnt<8T-dpJ0}ZzIsT}8H-Xf{;Lzu?$I>#vP$CV5N1B=un_NfdJ_AOIy z5OJ+P4Fd8;n^o?QI&Z4%G~z-{dW{=DZbT_0ph+}?C-o2x@t!`R?thXgk*5X*^#1ok z!hcp;M^=^=aNBDv)y_V?B?>&!KSJ3(b zS-LF45QtH-mw~8?o-IZhRDue-MKZSCB3eT^`9j6e46o)lyIOC>?_!1BS-;}BjIF%u zAHO)sWxQ;ik)s-lT`#cD8KC7j{)QHBy~Fhi^#F14k|{}tk)mj1}3{ zT01_Kl2S!Wa&43a2l?&aR~+-Y8MxhxSN}dzR!A%T*JNYfp*Z@x$d|G&X0IItB~x)wl&QZ zxKv>vb7GYJxm9Z_JPNc#q$}Y2J(HRR2;TCZs8FJJ|7(%2Ku{e_)E(o9(KJ+TS!_+r zc~K)zzU#Wy*t`X?5!V1E+8KupMaSun7^-gU1Ok`heAQt^auPd?@o)WjD&-*u$UAye z-}IFq{6PE2iN>`Hn%}lZFe}0@!z;69mKKYxJB9^eSnL(a;k>Kf{L7=Mko@PT2YWw~ z)wC;OW4Pl{{l4bSnv^)RSHYD9*lrl@+^+ycs0@yB_nNIxqJBN<&XfM7p~{iWO_^=f zshTj45r{PyUYszRSQ#jv@UWX)$Ze*nk`!#R=5@u>_rR?qv3kyfK18+KVkCEvbj5TP zlGt+Z>wcFR#-^^EVn&Q38WU9SaxeK=zq`SD#dj-l?Ji(h30srlrY2yXMdjwBXAE_s|mcNc#0pJs*^Z5QF8XF3;iESPo%!U>#*g3-yp_k!rc^ zOiy3em+#;H*Bn}W^$d4)lzQ7$!N;5fb;jFZVJ=($H7@s-W-nL9*x4d&1;r@Zd?v@z zv~Ukt2YtL?ZPo0fYH8N@Miu?>^0xGxNqnkQ54MBS`R{Al=&@{bZ`h|iDIrp;cXoG< zfR(FH5J(@U{Y9uaXYO_&a)+M}){J&yJT6%?zMs!^Btqyf>Oq(M{ttKG9Svu@^*tDj zK8O;1cm$Cs5xuuaQz8)}V+4s3Z4kXP7||jK(OcAL(HSLb5?$2M2GNb)>v!jQ&Uw#y z&b!uG>-+xv*1Fe9-0iw%UwiLsm*2g2#2cT^MS&>CvqeiJ+lYgZdqz3X!Cm6WnV9|_ zg=jspIN*KN?%fQft5#Gh@mnqe9DApz34Qg>q=gMqkGq$%F`r(X+!FAS2d65%0K9Hp z{ZbR`j+gtbx}0;Kd0yYA(VqwX#P3h_`zU?1DaenRjXHaHex@|dN4CY*VSaqHIpk1} zk;+i{^a-RREPR!IIE9bcT165+iEHHLq>S|(+NK^{ng<$|7vPsdZdckc5R z?6&_r-y!?UU*l4Mn1aVbbhLQt=B|c({>e;j-Du+C38wgd1a@9ykMa zYeZNx$GJpwY%R_e;#^0sC{cIvL&Fc190;mG5yfx7N&-KOdLOD?jeN=~1vVsFwAs*z zB$!@~dBB%dGD|dWBV?Q|*8l!cWpVV9zz0sjMndyVKjN5$W1=_8G12El%m!#5bbRo%wS%%3=Ze&R^rfv?M{_jf)8D(f|b?Hd6VzG_Y? zk!QWDjU(=^W~^-~Y4lPCL>-vhH+#b*F2Yhc`R7}MAqdyH>M*#)cUX!W^LEy>?rS+3 zOT?t?n^wiA3Ek@FT1Sb?(8(Do`Parks>0D>J zsSPj34T}V|v}TQdpPO^#jo&TyRk@~@AUzxZeT^Ka0DG)&qi=NsjS2&*CcWCn8j)uL zctL%=A9<%O2Z+n{8w*t_$E$&}eMTf*bUX_>J>lDfxu=`Insj%dGk?fWlTEegg?;#O zs*PONA3Wdg%r5NZJli$CU%hnaM}5rOm%PogPc3Q$*p}ph3}dYdV_(U;psxvbhU}l- zp#!#aC_BYB*4?i9m%E>x&eK!dm7lGtNvPlgBSL<1i6FZcu4~$WT$fRVhB<3aqdDh0 zLA>fjgo6acVgw>wywop}T4|3+49Y**r`!GTvoC%^*(?JR!jrarTp}nwv{TGmmA3Ud zPj%V8fEZkLJ#n1zmZi==txN8aviFY-k1E>yWlB;raT4U0FS~YV=>Lb^)5)(nV$wqC z+#rKKi{(3-Dgbmd>Oa+0zbc;PUIiC_mq3 zoa^4ZX`9sjhx&23KsmaGS86j*I=)$>#aJQ}E&U%x?nW`l+9{g)`*s!2^rK{pDWlG< zOz%_AE`D+!yjQsiv_!JHDwEo-H?2;5cBjn z;wJ~)?YlnJmhxrp@R>aCrceo>Xs>{!hNG=H1(;*|GIuuJKH% z{8%)j=Z8{F;mP|Tu29^E9jIZw9~Jd9Qr{aMIBl#TCM3mi34-&{U(mG-SiEu;SC{f?IdS@Ys{<>HGEIiXk!4?}PWX-OGUZ1PV;saf zzqJCrS?&ijemZ3Bm#Ifa22>;oe)x_;3@kbW;g%StVUqKk?IbnwHyYzvkU0UyV zXW(1+oGHA2HXmQ)L^j@Kazm@y0V^>UY$sVUywi>< zynYp9B9wE<6jG$_$QD?eB)Ysxgd|%cbi#FT#h+3Mt~w2UX}e8rdOcFr4Hf^H%4K#^s_6Dr$V#F~47 zj3{TDm+=N~`w43};2gbq)$CP7nXd2s!q&hG9Mxbr$bI$|fr@pF%G>m&=*!T$p9uEn zCnU!lyB)L$%z@Nv98196u&N0q#d(&cVqeHl=m%Jp>BAlg>zWgaW0l)go2>M1S8+`* zqzmP$Rk!L0@Op?oXj;)bo|=t;Yxt=~+kGKt<4vqV;hcAtLb0e_36$1=|BB0%w~!mq zUy6|cJq=WTRV;cq^sahEc-!MO_}7zug~K|MNPlGO*6ZNh<6t#Sx& zBm3(K@H_j{Vu94`{bKt9KDv|7O1u^tSm)c%)aB#CS06MK?*_?K?6XQ{b?=~;asu?5 z251iMlQrFkHf9M&q#B#0%($!LVvjbk=suOzvlq#?=>7fg6eP-}zY44>o=Jbs!KT?h z%uLIk^AZ#Xjz?l3le#|1Oit%t28IqJKde=(8SiCJtCHu-U2#-vEu85lT=jP1I%8Ri zf2RH`3?g^X<8c2W+YWrsLybA$efy{Tkf|q6O~#;aGeYPTFBjU~@8c?4ulh02`{2W} z4;$~r3HU$rZ1ssxO|7``=HdchxH7!idho!*_JHKN=O&rrx%_5`8cv?!)ap3Eb(w5z z!y{%f2fTcKTW}1y;r~d`a(eQ(qUVb4sAfjv-Jf`wlLIQBgiv1}yU-nSt(0a(U-u3I zxWe1s9~{nvV3vOu2 zw=t8teXdJCrE19_0qe+XK}q5gtlZ_Tk1~jH{@o=>@0dEK*2Q>n9_Q$~TDh>egw^o@ zKEv>lt&%tr=Y|@vcR0}?_m#k_uT?5L*Jys(QgpFcQncyzb}@}Yendh_Ir@cuMZ~io zeGi?pq)-t}u0){H9}bcIk_#bVPbETh*O=RUWpoYvaUojek1)^ zR(zgBnrTZ~g#{XN;|jSfxjL8;vcnnA#nvX2_-2&zx(Scd1=;z2b))9Y$;i#U-Q1${6xDImkz7N5?4#nG0~Zp(Yc*S~iDRPKvIEgQ9pVhgCNL`GU5g#=Y&F51P{z zc&k^ZAIamlRwkqO1osc4mXD*nGxTf|BwVj~Z*^-2jMW2uQF;ttAhhi-nL9lOoHm7D zqK4%6rin^P&pMSW++s{-bngpa#<-r5Hf38NKB^V!+vgTId*;0!_DB@}mFwevxkagI ze33sY+_fCTSN!;Ratp_7h%r?x;(w-25ytdrKN#=hh4T&4nGQ@nYA!kL2yM(eca8E* zm-BhPy3i>UTtgv@E5%K>yjSnC`rbmfUyp8HnP2vg5uhET$!%_wc)}5Eh`Obi5mNf` z*M6v+LnOW-kH7?IpT2y}5Y#66YIo#X_w@7oO6qw;>{vgdw);bzEwH8YP^nhvcN<~z z3aS>C)mNS;uTA3VDaCcJp)7bWL~Vgyx|NH%=XHmu$6R-Ov$Ty=1Wz>GgITttVJ}qH zNpv{fu#Tc>1T#d(c|VD2kN4bpK|Egm!F0%mHKNhz2hHu3flj6O*umQLmSJ^IF)mIf zETAE#OvRJ-D~_<@%rwz!LM1ln+-3U5S4Nwi31+V%s;Uk}J5-j!Zq-;*okv_WKcJ(R z(htQV(rU1%#=+49x`m?nt3Sk;M_K2^sq)sotv#lfRhTN=u-=-NWR%`qJD{ZX4^lDF z$Urv}Dqm!!*ipu_zJ*L)=!lb;rkc%H%LSsFrB%$z-kwj=jmr4;f8gF~t3Dt5(u!c= zkcm8MfbuFD)r!7hQ(2u*{ub;jkf16nq4(S+4f>AhA^ZIXSh7ofn!@$nC|?=h8&jVm zT+90+ZFt)lQ+6F*tYJeK0;O|-zv_84ZP1-2smSo9mC=7z-X=Be=I$e~ z!k0}H-P=<4*qeV?DXFQmJ=Ow+(cdC)+f*5H3XK4@-Gv~lj)JM`x%e{?ley~#zF#-U zl?jv#RM~ZWC)Td`LCNglT2g8<6<6_+{%-B7KQCGbYCf|-tzb3R`!qXt4MImN`2w0U z{gCJ{(s^!G!=2wEpMNIsXY!_=)g>b2-wDL!wcGwI9|sQ#_B|@`l~xkjC+_>;V90;$7-H$8_|Y`8=wga!JwE)83x2iBOT`}69(H`lQS zF?9YG4YjBSU#zSen2|h)J&2o-0(=}L0rijXr}$BS_~~i;VNWnOVKu!Cv$A~CM2rW? zEV~EhNvf+U2Pv)p^FWX59{u&^m^G5^T-wp9P?O{eipRs(gG-G`4-X8i)JotQ^TI4Aa|1Yp(s6 zmDc>r>f$U;8Ud;x6D4%WAn(tW_UCm2GgBvoMfZbD@-p3p9hqA!0DY3J$TDAg9&34CRpuR{?O=y`k$PHO$Qq_mZ) zxXxa!i{^i!kI@CR7(^YmwxyU25@Ik)Y}<0qA6>c~-U5xJTN7#Ay=*^yhE z>bMi^{C)n!3v2-HE;CdMhE>Lut3P22v0S24fhB+C;2TtF3qV?i*+UPw-ooUWij_jE zE6=Vuut3+}UFMrAyV=O4}N_U#xu!9IXS$>Hm|g|fK{)4@d|HLxA~598hWebd0^fuv*7 zk;urZEB=bfQtF|h_TkK*E~YVdtTzoy4cv3EOQz+w(GgvYCFyUSXOSd{(05Bc0y(y8 zuBb?!mtfSav!Em@(z;m4sL%$gTDwIDNh^Il&cgeLDmp078XIjsd)EHLoP)INBg-6% zaLNw(Ih_iApUBdXtN*p>PgxtFVNz13a?|=Pb=fwsM6YU{cvsdraV8>@( z9LOfr$%E*IP&WZcl%|ChV{W9xPuzxCkkf#@g(-+GDN{b8%4BbOTm#x>AY?JkcjOiZ z_M<3@B|m5O7VUKE9B7!wNotDMLjxr;*N`K(YLJ?>&E6uN?72(Y3wcRrtXKpFM48?L z<>=XiYB(F6T_AHcPT!G|qO->)X`FBSU+4wM&1)9<`vfPD5wIjBe^jdH0!_@q9`b9y zDKWOyHb!R|vIXmO6Ra5y^)U`_^I<|v%H@#+3+g@<_XhUAD0gO6`CCIe=TE@Sqh&s} z(fT1h&0`MJJvAjLP3koqxQRiw#?UFO0iXhku>_3atUxH(xZ+RRvi8B7byq zz#njGTDQG5Gt15#`HFNmSdl@U41Sy76mz;-GRQJ9E66?)$4ah%DHhGAaL z>N3!eRjJ+$0l^z3Sa@n|R!(9eHl0j%zt>WsVkwXAEtGx?jG-vw@Z)0M){)!&DEv}J zjtBPe>+{<}2)oCv&v;E8=2*ezkc`RPLuinR?A5lag>=bnU5@1NWe5Qip_tk|R!c!_ zcoh#@Q1C%cyMsP6F*=){Dc}#ji+rj2`be2}El9w!zU;Lde%pR8>^zebm8N3#D`DNL zTf^L$CsTs0#1(y&18j27VME*sM;REkTjPcEnmdgq!B>fVFyl`GCP6GJ)!$cN#{LFP+{z_1pC{sBw)iKfnx0~I*_SM zZ`njpU8WHR(!&Uo+JEMV?;eef-?*@9Qva-pz5`G$eq!gb~r zw0?cShZ_bg($p>Gj2vIa{CW>uh-bI!Z5;CHKFo@Cg|aXyuLB>)6!PW3=3^B1*4HTw1KOjP=T`Re&?zVtp`OvcL#|~=O%arP zR(_qZP1Y^eaXM~G>W)V*i35EmD<%fdoW}ySgQY7&0?`&>zEBDnxyqA@UB4OXfs2L?5st!i zAatHHBFzcVM(UnY+RkpXb22)Jn#eV_dGtlqM7El3Nn^1CNo+{NC}&lXPxUO` zh1p|#^lIsfYbE@*67LxHl4x7*9b%sC4%ak@&S@TC{*+Ikez_|5);Mn0r#FcqGXYQa zTE*L=DgF=x=3-N^L4A8gl+^Tfk?y_&n~R%O9mxI>#M7X2Ow<#0TZJtUZQdz5lF4n^ zsSx}k52F%Q*I)!Q%(90sr7ILyGOQ9t>DuhH6PV(n6JyF$1lqKmvLpc`fu_xdeRSNe zJecmCD`ytPhpi$9Le@ifZ*#kgkc$>kIK@#);AVap6Cu^AE4?KJcvVW7Gk|v;`lLyu zO((m(((js=2bQkKEQW}m0Nrr*OIe5?I;)5uZ#_rsunNa2Mjx-pkc?vhJFP2rw zK(iR!H}bHzNod!AOaUrVr@D0c$M8`zvkPVB;dC=a8vCkoxHl2MwaqxQm|>vDjE z8%Z2-(kKkN+;4TVlJxBTlws}vQ3$Z;D@Xv7>-}#h&U0Q-$8@gEbvkMcI!B;-(lx%G4Y&Wi$h#1-?uHnb~S-mxkw&8p9VkCuuvG z5M2->e;xWN_Wj`a&BaPD_^Vj5D4X6Y`%{z32GZ^OAC?N*gAjFj zL~ZiD^V@1E!rZE8iHsoWp{@h3U?u@m9CxyR;S{thnz04OH^gmU5kZ6+mg?*6;PFY5 z4SHIl$c4k2^0y(;vCyk!IZn2wt_OS4G8MadkC{=DH-1Q(@nl6To1vSF`MW-mt;eEV!5#l5DcwJ1_Gn+P+hZRa_)l_{oAW=&tHp4+IR$)lI3X9 zBPzCXf~2e_Vs#8#6esPj7wWWZC+beuyRGHw-9HP;6{%jis=}{yccJC6(vVRd^5ag9 zI;)K=8cGjoZ?2>;;qAW$DN9B_Umd1!IhBG48$r7G>tCDQ?rwEyJ64~38M4(-H$+2g zV*evIU&Y!DAAHxS%q4`SdcZ#XRbS6eP7v;|4E zz(WzRNOGrgXu#=N!j}WS4Uo!1e*Z99LzuY=D026O8Z)umtjf#zRZ!A{^xzldkNT7czyp60;Pb zVn#4fKNG~t&rynNv3v(w1-`e(djd|)WfKX8-1Dn)Fv~dkW8;i~otmoxc5poXZvjP0tKptK zUP48-hA}3kIwXDYM`uxF#P4R`G@4_)4@0T^Vbu60?(K@@glp{PBdgia`d3CzK+_GM zp1Om0a*AzP`OHmfT zyvtZ{GA@CfDu}pkuO*5%MOi>Ic(}OijW$CX^UmJHvs8;_O+WH=O2v+g0B&odt#D*n#i3-aqPFzw@@ zrg#t7qwhea^vETLKI1^f4iAmL?CGn3GCmSu|U z1(YEM+-%?vhmhg*B77Fo<(+HSC`Us6P6Zg`uRpk2oOwd zFrg#|SeG2!UxXts9p<*$!!D#$K^@w=yg$B9`FysA&T`c--N3W|&1X$Q1-dPzwbL8) zA9MY~3PK#+djKIsnfUduV)rN=JWQ74toH99lcmBFTCtBAO^aQ2f0Z%obt{Lsoqdxu!u_QaO!|w7!rNxghBazH)&_2}wtn>E zvIvV{S=~L7-;inqsVSrFlbOl=8BO_sxI!ny#rBL?i8-)2{FYR-p(m|VCU2XRTd;~; zdC=I;&cryT(g$OJ4fY7?-8#%3z$=$q8?g^U`&+W|nX6=@mOF!f^=}b>FMY8Y@?vYg z+tDW#_o-d=P_6_mXMn}WO}c5KrT4yH1GYaHCdt1qhAIXVr;fY6q3@_q@i z{)JSexU8Dz zWnmqnN#H19(?OE)#Pf%v3#3^n1L!OYwRc@RVW(wPQR6BS3~!a$lp4&W>}`A))Pq~2 ziZc5Zp}8R?i{4_a^21m^j$o-Cy@Sa>+5a42SMAAL@vZvwe7UKbsF zE{057B+}$diaY8E&4)73v`*~58Op)PSZAr^lA+T5*a=0LvKY!NVIZeWo>ig&=n7u@ z62_cmgVMwK)un6=X5f?aASO&EEIPu}c{D8|$hk6d3*7KkM3xYn>iB7^Xz;)#bZ)?W zPd&N+Z53>tfV6_tx%{WizhP?)PB8>`$WPP$Eih0K3c>iE=H6aSyL?yo=9u;5 z_?z~Gjg!>hl;nPwf%Qg|jmo^A7Kr(@P_#-cl9VjaBqieGR+9E!c8*OrjKH zQ%L7mOSP(GQ?Nv5HA_V`i;a`4$$TjegqoxrF&@Z9S{HHE3(O=yy8;=;+S&bKW>@gyc?W?INJ>gtr4p)ym$Y!pq z5%b(taZ^`R#YBXiuX=XvSj+Jad6=0VUR6`yBWv@qdwfgDoy^mD3H$(3bp2r~OyMDs z-!17V&i0OwlW(lEH{sh6+JzqM`lCryke|!+O#S-I z!jKtwQ=7Y{ncHj;6^$wNG6kG3eMOvV;A$t=ht26Z1_i18mYiW*yPZNl#m2suHGbsD==Y zP?q?EZh?()^W;g+g?e>WT9`A zh42kHNZ@`5>5#cF#Fg+|@&<1<*m!6bGVz44#7RnFE3C6F9ZA)^aFLL|xqwuX!Ck>c z6A!iiT{YIYZ4esP+K78bl81;`a1HftV?OPyTCp&z zsV@;vo__IvDtsLKRLHPU>M|y1@4|D_^WZq&>{>3PR+a##<||v?UH$&k7=`rAm9)}> zTJ>G4!M1x9mUf{hp2KsMPpaND6#NWkcy9T0fbHbvBAea}eyz}IDW{>rb5{ZT+(D-u z6byt-=1uu{Chu$~n~K}Tba*qDRR{lG`$=3UwUL5Z!^!C!6I0w_)`Ro_DH72^-c&2v z54*uZP{{TV-U@Cnl+L5IEPRv*JvA&LWD*FA22Ex&sFJFJtfY}gy7 z?=I|l{=Jf{S^Tq#k5t>4lZ6mJK|&ZWm9{koS31v_)a;h8L&brwYGq}61;_@B)BDH% z$C+aKRT=RHptVA)7$>1fF>S2p3yZD}q{aa|P4D{ul3#?1a4Dj3rF0(Xm$2zML+@g` zs0&`)VRG@Rs)Wex>_1v{O^WI8y;~CL7SVx`{!keR{7)~%g45UcOk)p=9<^#KrQJJ7 znqMxG>=?`&z4LReRbvpXkI0Ir2p;^YO_OawLY=vk>YbX(E|ox{uumy}-1d0yC@bn` z$?0VxhsQF^<0vB}-+rcX4@X2(Q0GFTu+>a7RATJwaaP7>+5fX|kfN{QE|7`TDxvmF zL3pgZ@;D!s9CGJ;$p5la{Q2*F)$AO>d!;C`^3gd}G-FzslOGc|c>3D4HX_h0IUy73 zUhl40Z|kFw`^eBP5LgP5n3s0*y|S13R9SFnEdJa*bE9JC?!r#USg6bRwxu{1i)!~T zz{x13R`ZBQhk3vnZ;}35p~%2+h0a6&uq~s{6htYa2cZo{CemEjGwp=x869kuzNsbE zy7huij6Jg*@psbr%l5?d(^M?wb4(Ag&CIYT0Xvzmih0=U6DY(3d>)MwV4|)*Vp9>o zI-h`rTYwEpl4*i@KcI&}fw0-_D-Y9X-IOpttFQoEn5aVk_8}dZP_ct-rN^51)r+%7 zK&ugaC2OYlRBDz0ZO!aA0dbCHreehj!M~^T-jaE_yVa`WSR0+QbQ0)k6T0*h6m0}) znF)EoV|<*xZ%NKP{OF7}n=UWVA_5q>^`E&k@;i4G#U-NTPZH>YNohP%$y0r!7kj@= zWrtxeleG|HLTPN(pF+7MmZT&x*1*n)$-+7@Ntvt zlVUOzH5>Rd<)nrKZaJ{Z^SplObDA&SE&{)U&6ga@S}cEkoA}8&zu%&kymR+g>Yt4AcjEPoysVyEymiAA-BggVqL651+AW+lQF~eOH7De;m)Ev zWiNX}P(iVj5Re_bN?$Tvte{*&%fJ%o{?V_3SrCK9p{&CRPUsh${M^Dk99k+?KJ|u{ zf_8%6y;hB%!0|IF8FTp-Z!DYLv~e1ZW=t>=R(@08?G;UPLJ%PDqtT#AWAYf>aY^gu z;(GgUKo@SpP9?P$89Mw|7I}Ur%x+8+{obe7RJh-hK^|@~JGUpx2WfT{M3v26iLDlt z<7el-qE=qxtO3wKHiZ80O$||RaK*Q@T^3Du30W06r5e?b(9N2o&v$8r5%C%Yk?dWe ztKVo4DaoAz$N@nu_iP!It}#^clh2Z$LI}v~Lue(1l9Iyug1x7vU`9m;Q-hI`=YTzY z{GK-fcP@zj9+iDj7Nn zQGk8vHu2gRjF4>NmZA%$%Q#@uS79P$=&U%A5^Pki(IqUMr}w}MtPx`1uvAzwA>U2` zk`!@~EP10%3X-2(lOr!S-_{#zA@=~21U ztk>*9QIOHuuZ|O-h`F<|www7KWcIxB?Te z<6zeBzzM(;cYX`Th+ciHL4~!MEi#)Jnt?MSGl>q|+1X$32}hDD_bl{~15if7;s@~H zWr|I+%iWW_7okX^yM1Ku@0qZbmcid!h+*5~D3m_`l6XvnYbPhD8K*TRL=2;iy2&Cz ztLr;=)s0&=-VUE0TR=n|;O;P|e(&DUI3TqD?nfxTxmIi`Zc5MPef5nKit`TT_E!fT zUA%B>SRCtvaN3R^3-|)WmAta>Mt%_VK&9aqP9+q)ooX~nkWOsg>-p?3|AE(?wAx}~ z%w44h7&1voy2dA(9IJG3!lHFW4QpW8h5wDeZWcoQ*XTc7tG0_1i+5Y@4$BcrxHZ zP_8k%8917-orzH0OYPav>^bXa#vn<{Xb@EfOpg5-f;AolSe`f{lW1Ag_?Z}qbf zMFg~GV98@%guWD3Bp6HB(adCc?2>=fBHB`AzcdkaQlsJmk}8&q|G2G^G#p0KDef@> z%YL0M9G8oKHXQ>GJ&uG5_|d&P+{q9{rf}H{$$|ZJxbkHAMCRK`I*82d2&dcJ$oy{S zMoO0BM`n3nSF`IL?(gX{fj>k7$dZ3ylz2M1w{mY=HNWtFfkE;nj>P86jEF=|v|WazIrq7;5b z?O?)I;pUaP%|Q~|ka%cAa%ND6K~tQI39$uW^KRa`@-S%X#m*gXSZtk|fz6xXi{KD` zQg)sM`rC11#_SAi;<2Up;jV>&qBT5S)qn*gw>d~vY21N(<73AQC)p6LC=poAQz?Gh z(X~ynNS#022r>|ABCtP@RNUrY)=o%}=gP^8t2w6=aflGnSxm8m_dJVg6IMwEbL3|k z#Y7HI%T&Y}d10+3H`bon-=w<1PV={|rwtqlWY2#Spj!j5dEkwo?}w^KXgo*NNI6gu zK1;?ehrC02Vhh}=BP&2VL3!WEn#W!i^_G7niXn>3veAN{P8LS(n9F|+OSw-ug+H(f z2J~1llI*$~DyvU7wU4@FFjk|0;gms15ts!=8X3>nB`3gVdRvlIy+ zi$7WuXx}+cTd=tcZocg6b$@C08J6I5$cU-f4p63OhSyi-N}sSU%{QiAZoF;AZUbSq z{{#>GOF=&V?!wCmTNDHlkk2JjK)*#wx5!y;-q*qrK9PfWE+YJPDx4wB=o!`}adKsv zxkSNif4X9JMBV+{sG4;s)&3oO_?Zb;vC$9fIxKrG({v}P>kdag=!BK9T+pYC+65bAtAL~52r=K%GN1EA9wr4CtJ%Af~`^=)Ed zobr)oXi5ZTEFU9$n3`VF2%sn0TI8Q~hV{w3^zY&srul?y5pYpieiK1D*hV(-TD6?T zr(7I|8nw`1YjOEeh=mR2sO=`i6XZ#{rYNsR?#=~`xmz-Eb~4@d7{E1ABHblU2}`;r zp111-gVt*p&&n2eM_9*F98t>tHDTL0xn&0LpjgWcoH55&v6yz38R%7xQr7w`*x?3n$su)bzp^vG_=I2Y#*BKLbwR%Q znsUK3Ce6k6^QQvsL)}-;DtFQmqTWPBNlofx9+1MaSI5-Yy4Os$#?LjAKG@_^QB)74 zz9o9{+#P+eDfcj>AgX^h(F}2w&vNv{uwcpst(7K~C84Bh0BIXhpPcVgJWZxv-|C(6 zeE3P#-EfHBg=4Or5OEzh?PYjwi2sG${!I}=e5ptU9W-d0m?R(tP%+qR)0of$`C_S#pAm}={ILKaMwc@gnK}qwYa#uod zvrDgB;=27~w^X4NO5yL~tNzv(Z49^Vy##vPJwMVl;I5K6e720;>?vq?4MVER1@ayK z(qGh_?Q&YqVp`(QI1PlQ@e3Qdt+iM)&HY4KnD-y$T7X*)Z!av!0*x*NmKZow);-~h zD`8p8eBtAsTNNC)!W8(A)FK4ibi?pu4>iDUZt> zVBuylx6$)}2_oQ+0i0)mC^z zCdypy_9@at?nt@syj-1(V3j1%m>R>lJFT9Xbw(0vxZjoi>&SZNvh&X64h;Y6&m&v~ z`&|`#HK-yc?19; z+HrX?b1CbraBV^AGonGpc_YgAY#x`#s(5+a_jpMdfsvB>p^E*U)llX1{XOs3M;zv| zCAEu*z#X>3y|NoMJW{K7o>%V1=_AM@>rAT3GhjYi(f>DdONgmLXWayTj%A?>;zv8c z5I1XL!>(IdTp85j_j`OxOb?NRVUN{5jxp zf3NGbJD=2QaD~|0gCQ+1t^7XxH@uwZfekYDFWSC+FxC4tkTkb9-sR-qSHVHxpA-5^ z-Ts?QD0A@qLFnsz0Uh|*M2SV3K0F!AwSRlTU*eMT zJ4gRd+54y1{6ER-H`)(3pWl(3dK2^K2B~6FpZi;4WtjdUq+A*Xb#DZmR|bQ;`{{n$ zIy(F|!fhVvzXd(zH+uT_EZ*_lKwu2ypWHNVl#}BAYqtH3Ta!+Gp?~U%6zIQhgxBcE zKPfV-y#B8n0fQPC^NrR&6pM1Ci0_5hzD}iiPa+Qem;O*b4fs<(^#4YJ{dr2d1>?Ru zq&keP1o9^TAOZfb2`MiDxz#^U{en0xc0Z!gOY=FgLaP0xoYK7k4<8`o@^6^R?^6M6 zvoYEVz+J489+!dtpeh#jCjkO_$^N0(zR{ZZCg*b${OIcXZ{u#U&bGiYBT^br5R_|4o0f1Gv{pDXWnpwOVFiy`=ifrfl^I~~(BV@5H=>Q3p zzO;D~FbI2DR@%cc7wG=aW&?+%7n8U9H70>8wcE<)x8Vy3-bq^0lgv3ks~4rRY$v4` zox37TWW1BCq{pkq*C$5c)r$JeUB#dCiNr=c@qKLsQUovTz2W3F0^&HxkHrUG9nyUiMJ-6y@ ztGi}+yr*Jzse%b`vZsPdKt8*E`6hl-Kx7T zoLyGJkvh@}2cMu$^zMu!N~9TuOA9yb>Au+iy;5v){Z30Sb*Vf0s{(VNfoH%#$^`p! zy`ra#X0IoVJ@J*zt7iS_%F#~mR|9S@R`v|HZ#*wIe#hZE(ZUfqUQxog!M5uI)W@lp z4qoO2#=t8pPPm+2B*-|w{>8S7P?rV>VhvR0bIBH+XssN-L5f>7{Sa6+?yQ`c?bz)@ zv{kL?vxV6>26mY^m5?Ppce0tjHOQ;8IbtSFm$`QR${?uz6$ zrz0E|>)kV^^k-TO&}$!jG|h$8(h zPnm@T+YV-|%c<{GeWqOx6{GI`n4D+Sz14WP+82b*WqoB!%HGt7H(c8NH%OfaIyDn< zi+sCS($Z@k4j4GHD62=jDu--?%_;o1%vb)Uh;FO#ht(|=Gw#70}0e* zZyX50Ytnn&fr{)rOD|yLtbXb#wxnTUVR7t8dA|e-?P4}3(MXwb7bwkL76TE0`C5oE z@LGUHTq=$M8$#b@m!K`X_C+UQqHHJp?!0nuRT5tG^k6`PQ}f*J1F#B=2#cOXhBAP* zi{g%b@`r%dao=4Om^j{D((_rAPUO={)>#5ATTCdp@BA97BuqD6ck13Ir~rUF#@U_w zW>uu45ry5_!Hg(O2DaPE0Ku*X1 zMEP4}Ww>KEDQ~}5Dbb!ZWPo7KwVo|obQgp_Sf8vFdnws6+bRE}bXY(RU+cO|suctXIP~qp!#5A)B0=$V;N0INO!Jd^uCdiQJEI*OLrTqdHzn{l6IG8_thglE-kIugQW_csUq~!b_9uJ7od?bX zB+M{1P9X1_$X)WtSs9i3T1OGa#j}@Du+<2sZXbvL!_-^HHTi!3<8(-ibc2G5lyo;r zgQy^lNJ&m|^gu!>5f!9SS{V%kHoByHbTeW!3~8B-AwcQD{b$w_KOvO$&lGWkMFV}j# z{QdQZT`rY^T`sgEDXtosz*c_r(bd&;0niIHH|eu~cChkRkWkpNhV5@^SuYq}Qp@c+ zA>oN|m>(=;h@V<Ct)U|nk|YOo6Mc`mRcsHVD_Fe5GiE+ zpRR8Z($oJkxx2K4qgc+^9)ABl;^W6#-m`bp7)K)5`4biH!fM@~`+a@}zl7XH+UrlgVnHXc*6pX0AuXC&nj;@+>Hx7)$&~s$Df&exNd?14d#Y?RQCN z% zxy;ev+paY`_Zg7iBd6>Eh0z(-D}CY-c)T&ZRUqB9`EFZ+$DIN|b!B;~*W?GcXR2<# z8^FTCGEQc9&bk^0(r6HQ3Glc}N=uDT8AH*tHMVg-u0wE8%sR*!xT1l3Eb&NdWR?^X z{!%#e#tAkG2nfU-e`tJZz)=NWZvVg~8x6-?69Y>ahD4*VSV?4f>W2IAFW_;;7;NLP zmorV5Y4!TU5@Q@EAs1aC&(U-jC0*2va(-nM-(4S<(k%kfBPoUyFtx2X?@u+5`%g{< zrr$Gs$!Al$1hFk$;SAY-alL)okSRLkTf=rO1PXbZt>fBu&3??ZD^l5CD7!61DLWuK zgTDc{5Q^k1+k#gEkheJ9fh7f}`A|1kCj>7|$DXSWGIbBu{L~R8XWFuxE{{}3^rK(a zGQFo*-$2t#^p)CkkP?Jl0A+X}^0me#<9-zEvJv&lv&9v!%LYlcDRGRK5MJ}0N|@Jc zUNFFX9>)dhZ);P`FvjXNCCKUiWl&jh@l8_g?j8&d{Ck815K`vT&r*+C?$u> zp#9Ux{BgQIy`0&RmZf05Nfoub#4x74O8qv9_G9=_*J)YtSW+{yA%G4mfVwRilBE zI>j>*91pFfm=uom|Gh&jei3)#vd}0mJ|Q8@Zl9HMnGbv|yq;Bl0(2TBJt1M;Ce2!f zvshm$