From 17ea44e0cf5bbf66b238100db45e8edfc14573ec Mon Sep 17 00:00:00 2001 From: Lewen Wang <49936435+lwwang1995@users.noreply.github.com> Date: Tue, 12 Oct 2021 10:08:48 +0800 Subject: [PATCH] Update TCTS. (#643) * Update TCTS. * Update TCTS README. * Update TCTS README. * Update TCTS. Co-authored-by: lewwang --- examples/benchmarks/README.md | 3 +- examples/benchmarks/TCTS/README.md | 60 +++++++----------- examples/benchmarks/TCTS/task_description.png | Bin 25240 -> 0 bytes .../TCTS/workflow_config_tcts_Alpha360.yaml | 19 +++--- qlib/contrib/model/pytorch_tcts.py | 44 +++++++++---- 5 files changed, 63 insertions(+), 63 deletions(-) delete mode 100644 examples/benchmarks/TCTS/task_description.png diff --git a/examples/benchmarks/README.md b/examples/benchmarks/README.md index 9dc7bec5e..b1b1be82a 100644 --- a/examples/benchmarks/README.md +++ b/examples/benchmarks/README.md @@ -47,9 +47,10 @@ The numbers shown below demonstrate the performance of the entire `workflow` of | ALSTM (Yao Qin, et al.) | Alpha360 | 0.0497±0.00 | 0.3829±0.04 | 0.0599±0.00 | 0.4736±0.03 | 0.0626±0.02 | 0.8651±0.31 | -0.0994±0.03 | | LSTM(Sepp Hochreiter, et al.) | Alpha360 | 0.0448±0.00 | 0.3474±0.04 | 0.0549±0.00 | 0.4366±0.03 | 0.0647±0.03 | 0.8963±0.39 | -0.0875±0.02 | | GRU(Kyunghyun Cho, et al.) | Alpha360 | 0.0493±0.00 | 0.3772±0.04 | 0.0584±0.00 | 0.4638±0.03 | 0.0720±0.02 | 0.9730±0.33 | -0.0821±0.02 | -| TCTS(Xueqing Wu, et al.) | Alpha360 | 0.0454±0.01 | 0.3457±0.06 | 0.0566±0.01 | 0.4492±0.05 | 0.0744±0.03 | 1.0594±0.41 | -0.0761±0.03 | | GATs (Petar Velickovic, et al.) | Alpha360 | 0.0476±0.00 | 0.3508±0.02 | 0.0598±0.00 | 0.4604±0.01 | 0.0824±0.02 | 1.1079±0.26 | -0.0894±0.03 | +| TCTS(Xueqing Wu, et al.) | Alpha360 | 0.0508±0.00 | 0.3931±0.04 | 0.0599±0.00 | 0.4756±0.03 | 0.0893±0.03 | 1.2256±0.36 | -0.0857±0.02 | | TRA(Hengxu Lin, et al.) | Alpha360 | 0.0485±0.00 | 0.3787±0.03 | 0.0587±0.00 | 0.4756±0.03 | 0.0920±0.03 | 1.2789±0.42 | -0.0834±0.02 | - The selected 20 features are based on the feature importance of a lightgbm-based model. - The base model of DoubleEnsemble is LGBM. +- The base model of TCTS is GRU. diff --git a/examples/benchmarks/TCTS/README.md b/examples/benchmarks/TCTS/README.md index ee67ffbeb..0b405c6be 100644 --- a/examples/benchmarks/TCTS/README.md +++ b/examples/benchmarks/TCTS/README.md @@ -1,52 +1,38 @@ # Temporally Correlated Task Scheduling for Sequence Learning -We provide the [code](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py) for reproducing the stock trend forecasting experiments. - ### Background Sequence learning has attracted much research attention from the machine learning community in recent years. In many applications, a sequence learning task is usually associated with multiple temporally correlated auxiliary tasks, which are different in terms of how much input information to use or which future step to predict. In stock trend forecasting, as demonstrated in Figure1, one can predict the price of a stock in different future days (e.g., tomorrow, the day after tomorrow). In this paper, we propose a framework to make use of those temporally correlated tasks to help each other. -

- -

- - ### Method -Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. In this work, we introduce a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in current minibatch), and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2. +Given that there are usually multiple temporally correlated tasks, the key challenge lies in which tasks to use and when to use them in the training process. This work introduces a learnable task scheduler for sequence learning, which adaptively selects temporally correlated tasks during the training process. The scheduler accesses the model status and the current training data (e.g., in the current minibatch) and selects the best auxiliary task to help the training of the main task. The scheduler and the model for the main task are jointly trained through bi-level optimization: the scheduler is trained to maximize the validation performance of the model, and the model is trained to minimize the training loss guided by the scheduler. The process is demonstrated in Figure2.

-At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). - -### DataSet -* We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. -* We split the data into training (01/01/2008-12/31/2013), validation (01/01/2014-12/31/2015), and test sets (01/01/2016-08/01/2020) based on the transaction time. +At step , with training data , the scheduler chooses a suitable task (green solid lines) to update the model (blue solid lines). After steps, we evaluate the model on the validation set and update the scheduler (green dashed lines). ### Experiments -#### Task Description -* The main tasks ( in Figure1) refers to forecasting return of stock as following, +Due to different data versions and different Qlib versions, the original data and data preprocessing methods of the experimental settings in the paper are different from those experimental settings in the existing Qlib version. Therefore, we provide two versions of the code according to the two kinds of settings, 1) the [code](https://github.com/lwwang1995/tcts) that can be used to reproduce the experimental results and 2) the [code](https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_tcts.py) in the current Qlib baseline. + +#### Setting1 +* Dataset: We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. We split the data into training (01/01/2008-12/31/2013), validation (01/01/2014-12/31/2015), and test sets (01/01/2016-08/01/2020) based on the transaction time. + +* The main tasks refers to forecasting return of stock as following,
- +
-* Temporally correlated task sets , in this paper, , and are used. -#### Baselines -* GRU/MLP/LightGBM (LGB)/Graph Attention Networks (GAT) -* Multi-task learning (MTL): In multi-task learning, multiple tasks are jointly trained and mutually boosted. Each task is treated equally, while in our setting, we focus on the main task. -* Curriculum transfer learning (CL): Transfer learning also leverages auxiliary tasks to boost the main task. [Curriculum transfer learning](https://arxiv.org/pdf/1804.00810.pdf) is one kind of transfer learning which schedules auxiliary tasks according to certain rules. Our problem can also be regarded as a special kind of transfer learning, where the auxiliary tasks are temporally correlated with the main task. Our learning process is dynamically controlled by a scheduler rather than some pre-defined rules. In the CL baseline, we start from the task , then , and gradually move to the last one. -#### Result -| Methods | | | | -| :----: | :----: | :----: | :----: | -| GRU | 0.049 / 1.903 | 0.018 / 1.972 | 0.014 / 1.989 | -| MLP | 0.023 / 1.961 | 0.022 / 1.962 | 0.015 / 1.978 | -| LGB | 0.038 / 1.883 | 0.023 / 1.952 | 0.007 / 1.987 | -| GAT | 0.052 / 1.898 | 0.024 / 1.954 | 0.015 / 1.973 | -| MTL() | 0.061 / 1.862 | 0.023 / 1.942 | 0.012 / 1.956 | -| CL() | 0.051 / 1.880 | 0.028 / 1.941 | 0.016 / 1.962 | -| Ours() | 0.071 / 1.851 | 0.030 / 1.939 | 0.017 / 1.963 | -| MTL() | 0.057 / 1.875 | 0.021 / 1.939 | 0.017 / 1.959 | -| CL() | 0.056 / 1.877 | 0.028 / 1.942 | 0.015 / 1.962 | -| Ours() | 0.075 / 1.849 | 0.032 /1.939 | 0.021 / 1.955 | -| MTL() | 0.052 / 1.882 | 0.020 / 1.947 | 0.019 / 1.952 | -| CL() | 0.051 / 1.882 | 0.028 / 1.950 | 0.016 / 1.961 | -| Ours() | 0.067 / 1.867 | 0.030 / 1.960 | 0.022 / 1.942| \ No newline at end of file +* Temporally correlated task sets , in this paper, , and are used in , , and . + +#### Setting2 +* Dataset: We use the historical transaction data for 300 stocks on [CSI300](http://www.csindex.com.cn/en/indices/index-detail/000300) from 01/01/2008 to 08/01/2020. We split the data into training (01/01/2008-12/31/2014), validation (01/01/2015-12/31/2016), and test sets (01/01/2017-08/01/2020) based on the transaction time. + +* The main tasks refers to forecasting return of stock as following, +
+ +
+ +* In Qlib baseline, , is used in . + +### Experimental Result +You can find the experimental result of setting1 in the [paper](http://proceedings.mlr.press/v139/wu21e/wu21e.pdf) and the experimental result of setting2 in this [page](https://github.com/microsoft/qlib/tree/main/examples/benchmarks). \ No newline at end of file diff --git a/examples/benchmarks/TCTS/task_description.png b/examples/benchmarks/TCTS/task_description.png deleted file mode 100644 index 7a9005bf24869c6c88bec34513a67704743492ee..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25240 zcmdpd*I!f57cHI8n+ix5Lk(RafOLc}M7q*@N4oS9x*$po5J0+!fItYnccch}CN)&) zz4vzG??1Q?_vJq1BzfXKEME9+&t#>GaC++Lz0;c#_fP<^MNrEB-yN3! zp1hg5kEHB0aL<5ka924m`C9BfFS=3l=?G2jR0Q8LJ@)~GCf2NzGE#G@zqIh#>pi;-;mjy#ILrTJKT=}l1 zsGTp9VHISHWzQqrl&s-UFTYS&Kyneo--@kt` zHSNVmN)(Yh@p28~?&ZqeaDH%CT(f_4?%TaDtS@(ajZ%G0jbu>gSoxijSkKldM*=ti0?^50IG` zCTSe+d8ntlxl|kGowS8c)rSL557QwSXUxPQj(yyaaV6a*Cc32SHb7ZP_POC9Z#7F5 z!Si|UG2H_sTC!2JgX%pm#6spRY-uTy!L%S^U#g;;dWxF)<_bCj#{~1b z>vX&uZZB!je%P@6oO;-UtL^3Q247OJ{;FQ)X$Cs)+P z&kS^{D7^oqLa58yWP_4K}8N7B?*1Dj{B7WI3AOq`+5R09L0J zm&u@zbw0>u$$21&tFguY+i%b}f!0^>+>CwH_d1MS(K5Dw#! zyZcgZI^WludWm3N<~ivA&Mv3wmcywA0}>8P%1q}!t?4e_!6!;6JEN1HU6iOTPC9#| z^-+zkDgtPIe`F@cJcIBaE2clJSH`-_KS$fH)(zBSk35HA8>TH1{_t^LxW29$_Hk@q z$hoVjzpP{aopj50kwAE>bVawqf6AtZvx{n>zuqm`Anu@@HEko_7d%@9GID=XQ+>6A ztTH5%F9%n-Bix9ew>QpuwtxT6h$+!*<>5Cq1?!%T0P4;9wty<#Y7Z=rb-)K<$)qeZ zCVyD0`(MJAotbp?cgIS+cSNNw=F_Mgonv<((`r0jZ)Mnl&a;HON?17$5ymq+FN+yb>bYn2)CNCuc&J{y^x>m= z)7KeOqfkj1w5$rHQKshUuv?7B!@Oz~ zb3J5CkE4KDEe*f>;>Nh0i$0k!T&HtTuw=Q@d=V<$o6BSqNn?p5XN%C0|M9wKEkpHB zce7Q$lcyNI{yxXs44a1KpUc;aO^WTS7fTBTiX4tiL`av<%DVMc#}J5gQ+GbD(%shz zPv_zt6=3;xEYdJkbF-Yx24VeU62EBnfFIaXB`Jb$^?9-BP%475(<@Tz3EgFE)hISG z#~jnflWRmF(GJEH!C|;!k9`N*c>~xPGkmxb8}374$)7Q^U~T@<`Et}Qw1Wwv1f_E7 z|Le;PH!P)8$MsUU5ICxm*zc*vBWT>{s9P2UIR-qLgrqq65B>*OmZiotJXK)k1I7$Q zeU6>zD9YFhJ3unMG|Yf^cB)r9SNr_#{7XpCx5FO(PMjm}`iT_OJddDVk$C)<#H=S*n(qo%|z7@%&fq)MvV+ zRkCtge1vrHqkz~j?T$Z}9uh&p!%UtOe14zAI-dpk9Da$rH(8|uW-z~f8dv!?$M9Kr z0ai9Ze?mhVX+20BNvWQIr*FYB7|AEz*A1Y&Hc+yVnsuB9SG`DDOFy%(Q#+G$Xq%_9 zPn>*B@2~>?zuqzR@YUq;!B zF6i0D6Xl0U+}wc5BDM&TXV|*`_VZ4nBrM*4Pr4kT67p}4re&+mh&#zT!jxv`ei=XZ-9cy7 z)&|Vps&S|^{pCWyF9Q}(YWfj+bp$)$x!Rs6L3Xp3-_7^vs50vxmjNF(V=2AauqS7Z z8G&uu$HXOi7;{j(@mvi)Dq*Ea8X_d>$=<^pP}l%-9l@f}a_++r&vR7|Gvm7w_y<2Q z=G)x+e7W`PcTBOy!FVBC1d<&h_xLIu)&yVWK>qIzsdU_u9A3Y0STgE=O&CArs;W&0 zz~_b-{s)PceQ&k<=i~3m{&&Wf8BjUN$S;}wd(H>@?sdNm;~hh1!_qHE_Ri)a0|wL_ z#QpD1l-OroDLT!TUjhE?j3hEl$5p^K;|~~6WaWMXTNKF4VOfsF3~!-lonYFw`b)PQ zA=9SY8Jabf^SseM(4_w%IdEiqVy%5xe!gK9RC&Fqe>@BVs1CZafly=9q2Ez0s zB|jPW2a3X2P?Xtus@G9~sfIwu1$opFrT+b3M!*irCEueEO01Va^?hv|ANf+9(IRk~WH+FG=F1ayn7j9?!&`u5|%Pc>Wnuh{mZViH(Tm9?Twi&M1uS;etfkuPE`lW-NmHyPn>Yr=BxFZBgKV-ouc&|EPgz&*4xnml(;CGe*dkE*n@7Ak) zs^A+9qZWyux0sB-PE{#VDA_m9$TG<~|GJe?tqZxZ5l`3;=5IZz;c=zLx_A>mK(k z)XRc>3?6mFzX9~3g`d?QH?+%%DnyC_~Vi7Y--i`*K9RDT5Re6sXAz*y*d01h?~ zCOoYyc`*F2Gf~1`UD4kq`Ol~10VIJRh&E2)C3<4YCh);tjpS;PFq=*p{))FEc1_ax z5x7ZS4tScCr0~F}K77BQ>7y!LLji<#273jn{Lh4W5hjt=L{nS-__^gPvDgN*2gBGq zg{i*5rHM4MfV6dQZGa2fw3B-9PCt5W>Wgpc`krca;hP<59CkVBEc)`{%AC^Tn8m>P z10(s3MccD&CA|0WM+{IK?U8{mIL#LVR}-+RB0W+}I1EJ&K5r5tkY*3Vo=FlXIqU?6 zv5VZMi}7gL0VQp=T<57jm&QVko~$52!^VGPb1QbYXD5F0Eq}$AHEZ$rV&9Gc(NI6c z1H$m>!-i3(d~BOFp1{7bb>_df^@0e6Int|w6_h)FyDrF@PQ-5-&Tby1zd zA6eEt)^1Z<8%z+!8#KiFzO}V`h^{gjGxeQ@VCDhsZ4LO+_dtZ8iTlB)0_o7src%@S z%$t;=jH5dZzC^7wE%0DyKXKrG&BJ41*OeL7?1RlkkB>1?vm%fPX3YjX!aa$N0tra4 zuuwcG9T#Vy;(ZMs_4bVEcWWYunXpJacSY89emTfMUNetuv3YeFh+{;-U1RKLI+DHk z))EJakm3UlAW>d6pT}!{^>{hm<~EN}3n zCQ7vOMNzF<*%Y;_;Ws6f#JV*L1OCv#i`3E*uf~PGRqo$DBEPhkIAy;Z!HbnQd9o&- zxV+MjBAuv90c09qFn)&yMML#}B)>~BseT345FM%UfzwE3E12nj^E~;~%JtDs@N#(D!Lp)>F1KK9?rfC>zNeCX&w?dZ5I| znm((UD65cP%Z|ehEnm^e_0sWmbhx0PunXbVt|us@6BR(>0|@-XHcf0gH>vr4!x7?{ok% z1mEUet_u#bb-onDZ&>p3DB;3j@`0!DXt2pk&Ak`a>39a23520R=KbZV00PiQ_x1PcuNb~0C2?0caZzp$ zekzT_g48;aHiJHjpZgj$H_{&%M7o2pr?knw|hX|1pI66mF-8R zC3A80#jMomEc1nz2L3t}Yom|$s~L9=HW5JSc=`&^hJl4}0en6C?YBu)vOK%@a!R{Y zK&gh@q-zMJ7(poqS*6v=5MWjxX;VDHR8d(Dr?C4oNaU@?{D%uA=Y_#N5OTsEIrmPu z4(KuJ3`jfzjg53cFqxOS9oemZ_G{P9%B+$LH}P@=XlYHypf?Zt>aLrp{&s3V1_BSz>? z5_Ra|*s&AFLSC#w3(;YEfuZV_4H5b{lvBq8{L(fGBum41IT*BY@bY}??oCOm{FV37)>bGcFybi|+IFl65rtI3pY|OM#<~zZ1rbrlA;rG=ehV81_8Ax{UCs zzfnJYJ`N@vj%`}Cw&}UfJCWb%l8}<$LHj~y?Nns;y3Wt*RAl&QuLJK(hwV&Qw*!RR zt`*2qi6RH}Fwf_>%Bzj-2{N%;1E^SZD9RC91tUh2UQT}I{$MMO5vi)|cw|f1qDRYgh z!pEJ!9PQ41I^{aPnaHEdC}5k(hXB`15$Wdcn!pK6Yjfet79flEF+K3X_}CFx$4#`X z`i$h*6=+t0!Tc8O50x0kah-JNUGJMipz1whwH(rewF87?8y;_!=ta5r!t5{%pMJ9! zQ5QcArFy&`w|K0Oe0fm8>V1ixa-Zd$LO(;PsY!5l;i^sCz0_ko9AW7;1IrIcNQhS* z?18`H?jT?#(|v;)>U}U#QnjoF8nUWzwqk$_^-L;n=)t(?M? z=D*wTz>lSiB1U~rFbT|;x0O~Idl5F-n!<$oy|2yB)PMp=f3HA(h{=-QYwrt5h`Lq7 znmPvfAN!Z+ZG>Yo$<+K{jTBeDnV0R<$x!?Ls1&#QaxzVeBhRIrYn2bJ`U?K>^F5rw7sPSRcnj*>2mZ z>3hdz*R@19nh!03PGcfvrdcc?pZ?>AH~m)4R4$$Ks{Ck0h~`}f@WJpnnK~#oQ;EX> z4Grx7Qfitd-9!IScB`lovh#Obz=&mkNCjViENm=M;O((8uubzDvubPyOY)1S_Zbjo z6HEr!rgz*5)Bs(L!WxRS%Q`iZi&$Vm)fye>x@5q}QzmXe)t|G%1+M!7MO6a{xX$Pm z=SOc3#Bhs$4byMiVjP06Yj0jxK=xeG+4ulzXGsab)O&`5l85Q)7Z*;z>j?V;FL0gL z)H7di(2d(%q=oMQHFV__)TK#{e6pd*Wn(RPN*v{DTsS4oGOE5%7u@>{<)~z2PlMhQ zX+MiTk*{mGH`z#QIq*RD&kZS!x$m9mV%*5N1iF4Q%4z!*+6t-n-mqmr^VgKu9;V;L zqa~wyD!w1d)F|F~OYT`N?Ip!c2dShs4qZQObY4Gdj_??bY`K%}NT}UBuVU%-k(%v* zT`E9a^uDn;jYH?AYva}0;5}C3==WmI6>4?613}94eZT?8{rxySW14cjrD zH=fB$mO4s87reiF`7X3Bn!WEe@jHOI3M4w^;>uun{j^L^wRF+jPmbS)_7<<=8#k^` zTP|vxcLv!oS!aBTJxi#rS?YDe$%~oB!?`>~PyR~-dCnl8=$gB!v!0ipgTe4VKoRW0qS^tBhsuaO*Bjy6Yny(H_U(46P*_3>QW3@iKD1bc*CQA5ICN#|XY z>vgdg^-@n?>UfF)E^iAVq&>Z<)3qYuF>r$f&r+PX9i^Sa1*9Pa-c)>Ol>H;6}}f=j?wXVSTK*DTl80=saICAkQ+x5uMffr zIM(rFtkMA!Fae%X{;VC};) zwf_$dJ`MfGt6G@&p~YO=kBF1=h8_1gZ(`Efn-p+sP^Ga8?C1d4vMOh{kk@_DTDfWm z*+ejq_nZ_km}ws$!d$toPN*g3eF7!K#FB5VqYWx{Msqkor|a&(7_QB1=^wZerkmx? zap9&Ct?qkYAKQ6QTB%K#Kz4UhRT=e+{O^Q-JDChI>?j!}@1Apqs*R90tl?HnShX}aS1-N}098?g#{;rBvui)GzjV}u(k#x);3YRcRl z7Z)B%scGgoQ<254hJQ1eHUHia4=PW;Jy&jW(1`ES3Rn+dN2RPdXH^JDRF&oCkRcb9 z;b5P}<(dlDxd#3fovbp+Uo+BHULX>oQJ)~E5(b~sl8W&aSxGt)r__oYT_Akdn)>SL zfrEhDSD;j{8^4QXy4F|t%#t!z>jILh$@LkJ{F6t% z{f@1EZ^wu4F;9CNxZ5iqv4cnWW!`oA3Azvna`uAhu+BS#`l?kbgr8-y;r80AUT6n- zG0c|RImNIQ-dq6pJdY4S9PYZ`UH3!w)oXO-4!y$y?m<=|1;Nq{5uxKUBXKd6tc|H3 zhVJPk3BG!mMcr|$uk|qwyfmtONAg_O>Xxp7MPQV}D#R;?8eM}*dweI`4^E>uiYvEU zWlRr&sXP5_TXvF0bc+ii8S`o3#%5Fg>%X^>q*}e^+Z7#D zs1dW>xZC~7+~DDTylCqwMu>zrEOav9?%#XPH~mR-OY4bu@~IPmPQY;x&VWsPoY>Y1 zft4a53wPOKPI%qf%1ACh&23%IKk@LxR< zYX88vGTDm$L5#xJ=utJRJJ0jLX_=(oQ>$aj=X{poX57nyPK~|zdY{o8Xb~;>Zsxn` z&=Ti#(vgsqtFa}HJ@wl3>?k|ev{fUAFU{_@ne;Xb#5mr4AZLUMvbfvQzqILdF2|Sa z=$|uDGX85?SX#}uC%2z&bOTZ1H5Q;7y%}wOL%BPCq<&%x2pRd{P7NX-V>`2$&d<*R^j=_F{N75=opdS;9EjIiHefV|H zdosXG2(Mj(9nN(^E4;2r@D|73nW1?lm) zXp8=IvMr4T^~uiEWDgPc-p!xkZ2y3GPrF6R#F#gZ(9o3Fx68pApw3q5p?gz7HruX0 zLm&m8cZez^Q#-E3*YANzKN=j~G(!)3n$B@%ifWAmgk%=>tBVlmBNV(n7BiqRT2S^R z(eox7#f?fLG`O|&4;>yLuBCkO20CMUKqPz(UZG|)+QH1bC5dqrX#5iqp;W-xL`l9% zL<~BUKEbcIe-K zE@}m}p1f`A-=4T1Ve?Lpm2rmSfP!NQ?;{VMCK_0GO(*)59)!h+OEPe%^I+xWl1i_M zg{LhmlF9abj-lH{DM&zj-k7`{64DIh9uflETZ;@HTZ?j0uc#%G<*K=nqZ}8#DJcz6 zZY^kJ8Mfn-SEiD^oM4Mj>))&#oYm>zZ^bo4gnASy`+e*5)u|Zinh_>4_W1|A^g@ys zfLN1+bYvNpMZ~hlE4g9!CE2ss+GXpSVTnP+pG$WFVMau()y8wSqNTT6{nPxyX0soq z<03!w_)%VqoEJL!IMIt>Gi^LCs)~5-ySJi$ZDY*n=pAM|bd3*BW>SXB63BX&xZmOM zL=Jn&KY=ME;%`}9<*>crZ}k37l@{+6nI%Lgl5xYzy1Gdlu=j9*cM_A=KgSnZYI)$J;mN>9_}cMed~vSo=(WYnZpT ze)H7&W!@g+0L*!ZPbesc%Idy;gz1_bJKRys;+-yAeTI;dGrRrvAX$uZQk4tt>4h3d zsp-4xbfOR)`tgjdXt)0RrdGTr^ADEk2P5uH=7Q3Xaq^A;_QnHZqqhNVAhptCU|M057P%o7S@@bjySu zkSGkf6IbHB!l6-W&idpOx>L*es`d*g&AUOlUv#57j{Gm4#OWKs!O+-VwlI#~4?$>q zT+uoUyZ1N7_?2*(KhRTm>#f1uJ%#n97gYN8MiR(JSVjvpf7}pJ=508#=~Nwfg_dxc zojD0>c~Hq$-&@OQ*)QSqlT%r-#$y(8f-S=NcI==+_n#-yEt7g4OGE?qo?fWa-7rTx zimkK)Q+oP=Z4OEZ6iLGPbE)@i(4YXC5WCa=_fvF6?Ph!Q!f!!?R)*eB{X{Jj(uHyY z7yM@>OIjf|!fsD$Tf>cL>{=a&ax&`saExldk4M=Kz(*c*znm_P(XBL9K+gTT&(E)l zCQ8mmq%^RQPEZ?gF%~2wAKQimE>Ui$?%@s5Y^O_C8A>ms+PQu2Z7ogrcF|!ls0Gd@ zJbhQw((&rYZ-y$DGlR+QAyF|KG4+gWGS$U{Fu8BiijX3I!%mW=G3-TWG3rx;Bhr1OmwyRixKTE^A+A?isZlC zYqH$uC0;(Gqq09WX}Ym=A}+En16vJvpewo9dm47d<`X`e(>|ISc<=lz{Fx8eFhI$; z>fZZqUe8KxSQL+@r)pMuqgtSqSM!m+qtR5CH1l+1Ug82{(=4TS*HYBc= zym20@q~M_T9BdF+_VqMC2c(^`K@w=;NyNxqxjKVzq@@F{!Mm zvxaW5jmVHseW2@gqBeiRbtBGxIJ?IGb+VyA^!o)|G%rYgU7lawgej=z{0SKtWm`4s&;gPTh#xt7=2PU(${ z^_ZM1e!0tTnls^DwksISZ5>dNEWWy$^<`-pOoLF^0v&9+X4);vT4$C7taWY1dQ_7- zPguU;U?jX*vrE>e8yC4IaDh(~uJPfN@|O!TzloI8DN@ouYT8{kz^k+S{R)fx*V2UV z=QNvwYh6c&0KvE2!XaF`*8Am6TXcfIe_8xi6-=|0W&9*JTbH{1ALRMieEzp{rpT3m z&t_f{`QOz$g{_A10vEL{39X`7bG6v5h?X_!?5DVn4^DSEK#$%0i8ZCX^T=8~s65^8 z5rvL3cu&gRP7l;ZveDNjwsK1w>a2G&Xpf)!(-o#|aY@;EzJKHw8D)E|S?F122AhYCkT@B=Aia;Ma&Gmb zTz-oOoL||LBjP~pb1`+E3%Liw&h^b6qQlnulaVU`vL{NE;b7N$YN?2JoCWSSbFj}I zH%~WXBkTv*cX{D`$z&Czh}EaQJ*D7$ufhs=?2Cj1YF`i<$;T_PtOw`pG$n?Meu$&pMJYsV$cfM@qjg4Fbb4C9*>5*ZpQ2Zov zj4Ro**MbnRZ)!F8N4>_w_7UBSP|B{q9!v)=3Es6!OL#gl+xlE6P4{k;a_E{b;VdAk zXgbh=$eM;!1pfmcdh!%MB=eK^mZTXgT@Btyomja05)wpIoyfG+|Ij{tb>XXlc~Gp?|;Rwp<2i|NVfcs{%ub2JFE6;P6c$mH|n=2+;`4l+=qs* z!(e(+oUa4xaxT}&8UZGhm;}>Kji+smV=or)WDX!>17o%|i{qK9f3F znZ?wvP(lkRs=q|}ikgU2?Yn@zKA^@$P2LIP%y89~`2JqiL$19az$ zlEMPfp0~0xdXtXp&Sy1qGsuWhAGWD&+tA3ZqP> zR0T2%NiU|$PY1-)72`I8d$U>OEWfQ7HuZS1yiu}l-8cEbt zI|GoMIM-wtlbI2`ejnDs5*u$VXnHofY2;JD;-PR!A|Vs^g(^p2ez2B&#&{1%(8tco z&l*LbpUKb8Tf|7XN6lo;o#staLkJ-j#81u)U^kM#qSvwX%L~4@>lRKnIuqBZ>Nsl9 zC*yp!GlE^ln#}&VUVKB{faDpbt}lB>D*|krYfXyb-7`g=G6h(*;XI{}_c{|m4VS72 zl|l`EQi9?;C^k4*Pa8_LSRj)2v=FS!kGm7Ns_Wi$PCpJausLFG?U9qC*l=&buE&Af zEJ}W)h^LK~@SabSdXQ&U9NmiY<`d5Ezagy|Ppt?AW_978MbZ-Sk*}O2gL`|m>;nkY zSZ<7ara=ZHPNLgVEBy@RRHfZ;QVAGL`S({Cw& zOO(kwymI)3>aKhT7 z{;Ysn7VMp(rZ9(&ypZ9I*}eu6EWDTn_EU$T-S(K;$<|$%(84vpZw_av2$Ok(`oQN| z#q+@((|aNEnnX~mhpjryU<x$&&+fY7afihp$(1shkCrKzo5es^tg5YgB6^pRQl}xUM1RSN z5KrlOz26thD4mlbH~BI+(-vQnFruh1-f=U64R7bjl#yts?*8bewS~1qxxs>ZB3C+i zs-Hi|PqcZuV`H(1wbNO}cSHS;vmfuOM3Ao4J8?T5d_lgWa09s>PIuL(H=w62wT%i=uWg3-@V zasImz`7>E7J_RsK{f=fEpf)v|f~M0?8LCA)#ao23KKh%7QfF(FFbI7dVC{D}(xg00 zvj6?vMX^#+_VuE$_u^e}alh;&yMp=Je|}=U?WmQE zcvRmA(Kay!&|>>cHjMtV;(Sz5a`Cpkj4n$(>I<&SC`Fs-mDzPc$d|~8yn>9_Os-xu5*6K8*U|RzfYhH4n)g9#;)& zTdmyI0vHK(CA_}A{wFM8UvfW>|{y)MI#ng9nR#<#JP|9!L1AjnTH=zhsd6 z{nAI6k&~UNd|1sQGzMs*YiB|6HBH7J)`8Gc%sA%;C2V1$$GkePREL!Q> zRTGQ^^{%zXdI}4o$#EFiOPyZsrh}*TfYF*t^irdkSq^Iby&C&Y@67wph>_BDs|OI% z4`)23f;wPs@GONug~ouVfR9>sopq$U-pR^|=|*rXtl*{hO~5@Ym9L`Z^qM4fP}8*P z_y9BU%aL|GTL+ZIw{Dem%`Dm<{+0PJ2*IT*tW#N+C>N?%LBDY)EyjQ%0Wk0V=a+TV za{DP8##0U#;)|R&y{@JdZ|IW00+679C>l&X4zt!XiH8m9 zt$8zns~Xr8j{{eFb=7b|z+DaEUJbbY!gH*TPgS}=as4$k1-h+R)fz8Ae^pCO50ZOK zcO8|Q_Kw<%niC)YNp-Q<48EMT^7(cb-g5yACHscm_fg=}PsDc4l1~Z(7Vnz{_?jNI zixRsypL_Iq&5>ZnNRS zSSRJi9M|xoqPpy23;4%=d#b0#&(%cNFvr16q7JbzA!MzU84eYQoua>~xwA{PAJAw# z|GMW8^H57KrE8Z(e`cxERw=W~Fgy8Fa^?K)-W}e$dvcG_vO1OTKlBfq(K!3UHiKJH zs$;InMMoa@`Q@W+*9$kEPPKJ6xp!?1iotgZDrU67r7WF97P7>M$pWC)&jJN>pgPg7 z+AOTyHdW6NC(`dD+@EtxnL@Ae59U42wgEX(KNLqqC46z;xvvtYg78J6zODaf&Sq{P z=G(y<42i~4q;ycbRkD16U4YzIXNUr%P~J_pq&~p z@B~Dr3w71~NoITz0LPRl<{+Zw_Wj$cx;LzHhs>wXyHw}W&p!fZ*Y<(2JJ22K4X(o= z>*!~ecWuSNbgkw5S6^T(+==*)1MNNhuRVaIy8xy;1U0w2^AD-u;_u_N{`GKnB3egh%~c7dK^DxD?TFeG~jZeSbrfjD#i)oY+_BBJBG z2M89wI}rn-lo$P5h5*<18gP5gMo#wKQHy3m*#TbbZs^5U>`9)_xy*J9QHMr~y~WH&V6Wmn+l9Nl=C) zRAFze6?)Py(7k^V9;h3AWNDN%9*~T5rf50O-aQ3m>`AKP%(b@%(cA4GGFXJ45oTG8 zSMz1jbD|-el6TgD;yxu*ri+D&O%i%HS_c?o%Qnok#^%Z*S{t?2JAk!QKC_Q!M~_Rf zmRKOuu|8bOk}+E6QNh&JM4toO&{|E*I7Y?$sv*BbiRg?(W8)P@dUjv7j7wi{wF1im zoFeZIs~kRrJixyei6;@%*Z&V+W5w7PYK5-9@ogw!|70+=rxH zy`w`jj|m(Dhj$lS#Z^@0(@FgaZeHw5hStB|56{Ng=KG6HjKlY z#0jkgN^PuqCQXWR)lq>4F2vgZWV8$_xBswv{E{+)nIqG*FG%TNM82f zYY&60I!S0#5bdkj+W3}$x6$si62)zA4yOfc&na8KN0pfh(ZhOSDesYgZ9&Y(bjSi~i! zncmFR5lfJm^kaw2FllRZcMzQnSfH0CBF-%)W|_{sW8UPS_L222lgWk!B1pDT3kjJJ zTyGurrN6W$ttC4YZx?2;x$Rw3k$7GdR?560PAgoeUSCpe=L$)i`TdP3!M_3Py0Zb` z54Hwe4n->Gzow;w_Wpg_9_`Bq-j`LvS_Iy4WhU>i-btPMh1%+Vg-K_ank# z=*ab;!^NH2ZN%FdlE!|Ll~ho1P+`Id&b;OTfv6#pIQ4 z>Oz7%#Vx$W#Z-X)(*CyBR;}1JrWW`9m2zL)R*6tM11`w2xlO5G+8)vo-tI(NWSu{S zuOAy7u`-|^FL5FbToB81%yduSO&!g1`$Tb^AWq;EP~r6TuQ$r_{UDAnzAeYbWFP) zo^05vp~CaCsXPt736rckf$z1p7(}Sad(3dVh2ev^?5`AvOxq$(1VvsnCGwjPlTKNf zYriFTVyOM-k#`!)`r9dt?JFS@?qC0Q`V0pHbqiKL+TWAaeYg5w6Ms*N3M*vt;>vUa zo-Q?++iuQ+9%2FJX9c+%;8VW6@8up$I5u(#?VVLA)&!tvue_q#l~VDv3X|qZeqn{U zsyJy15vMUhI?wpl5nl37R+ce?m)GD*CDgBq%?E(V)c#@mihkrOh`z1oCJdskfJHd4 z5~dp06u%biuUNO?o!^@uFrFjzgW&Zc?34*?I-f1-?b@MaNh$NRThU%c}Fd zZ*hXdfr6eS-J+xRJ}6&z1@K%^&Wh*tZBzb@b5qMKH-F9<+eTw#`25ck9W8@bVG$Dtn3Du`S*w4 z18fJi@iq47=sY$Yyd`SCV7+eA$C-O^C^kwyCPZh8jgV%DS73}98FsG5&Oncd3vb`G z77+C;XkYk(;!qWTT(dY34I=JU%QySv{?pfa2MO=Yl65=*klo8@fmLir$3;Wy?XZjE#HNzABtz- zfY22yE_9L(m!jzp1=10v?2WfpdYc?9AOj@c1OKJ=#YH1<`;CnQen4tAhNvHey{?3_ zz+=eQI_HSK8Eyqs%@oiB!P+9PnFM63PG9ohiSx6&64>TS5?1-W9%pPP?ak%Vh^4XO z$>pOKAv0U2dP2;NvR?3F?}E@w?>T(n=I&(z4d2uuWZifTv&yz?7S=+SFLsa zQoY3s`g66t4}f_ATi@6f`;`K#J)C((0ZAA6rFruE1S=-OxFnr#%8->$`G;4vF{2VO zM)cw}mK^v&4(1LYFl-i<7atClm`9(i$D5VoW9qDeM6>3qyxK|PEc8JAM4jT) zzy92On^dkfBVhAZK^MFw`G+i~}7~1#^QMbJ^7)r(2ANt?Bt_@py0?tB-Cy$;oY7f=2 z5H<|P;au}K+dMDWbRPc|x3a6jgC6(n4c)&j$kcE=EYIHw1`)Y*vGaiR0HSwHqj1|( zM3UC|l<;Prrwq!creo=aCW3bC`udkwI=+WtX3kE59%^DEN2_x6MZ5=Lu_1nMrBl~` zC0YqW71mPhbZd|5MzN#)tF;6y2+&>vq)+ZNd4*j!?i3v5yg!-}6HL5_s|hEQ5}fpR zus1s`*{wGBGoGD$Ov82OqVjHtHtT!I%6MBU<*TcOu^|DI;rupnd3r_k*N?;M{k7M& z%q+FEm!q`57IuWUAk&4iqtbss=^;Pwj=rvBote|dP(e2X@eHHBsV)78y?O<@^6ZIQ zkjQ?sFei#rKJJhCBTHoU4C&%`rpxpKymkln`Vhl0TPKD4{|;64{KueU0MHe^!JCeS znASYIuy*K@9iBeiFhdA|{AO*?iVNvA$E6ruBk7%{?3K$84PSmvr|$4zy7TyP2ALsB zxJy&NY+t2OzZcTG<5?VpQs_R=HK@AX2dHcR7?ifj`-;edMZfni?s3v?T$~R$Z-_Y2 zqI9N*QO)A3{`9tH;S{TTY4!fc_N1GSq>I9N59y!dtT6A;g7)9-o?^_nMJ(1UbX4hb zd#`G9JeELp_>7fAwoKQfOBYOX0$s(j^WyPf0t-zagZ(+e;?+^s&4F@0hxuc@vPUgd zr;bVmCYe=|;H`0frJC}*Bbi>FkX()PJePPO_nW#wlB^FPlwnh8#;@2-7Jm*dK?1x1 z&ko_H()Uw!+(lxdAbPHZ0*|;gf4rGqq3*ITnDVC#8nD$=xA>4;cF-gWntH%?I`4f- zj4IA~uzfI%KNA-DTyWELXpfx)IujQuJ3zd<`RNLa&P+YRBSzGHIolF1bSnM>E0zB6 zrqTA6H(LMqC;ewSyUu1kKlgk(&a+<~xUwT*a!fn8VnTK49=aEHL{J3PT-1Fl`by1Z zv#fT+_U@8g--6ONgX?lDnVG!@xpuFBd+e;Qz9v%&2ec|TZx}bl{4KSIzRa`hZIm7x zs}FgrTkANWjDN%ghs0opGP7dYf1s*d6 z+sft}*(NkR4c>`4+TBpoq6_V$CXmT@dLqR$CyJ*_%9~%;PZT07E~BKUJIrb&%iUC( zEL)t&B-C#02p_9|Pv5xEAwBf8y1414=fn={Cap3erbYR=+ivU5>?lU4eo6hHh= z-wQ9>Ft^GG<~Y=QBK3K%F)@*{eHJm*w8uWISfL3Z9wu+by<&<$Go>FE)`1AOWQz=; z{QuM1x&Jfy`2RmbXbDN=oY#=^Ea&q%%^@8fYs}f?d-<%vgI0;`sIpUNX4=Ow3&DD06h zhxKD^Eq_w_$)`u8uX^jg-;4#E^R<;U}{gYZpBXNg=owuE_ii)_Du1+ay)-XU%k2I*GO%q zPhlSExDLgc(s?%tM0k^`a@z#IiXoi=;-`RoEd>+jYo+v52ndCc?{BBvh9;dz!Rihg zEgZi;>54QTOEeoZKaC{+AUuk(l=6o=lFuKndAuZbTiYiy5*rNmPWqfwD6_jCM4D2d zR?>w#zzQ^?U=2MFYkvLi_~}Xe`1S?ISSM#=(q$Px>l`FOF721#i~rocpySG$oIhEl zVdg6TAqP4;`|5Im=i8$dH}n(*6B2@604VE&k;(0wZE|DrS=^n?r9UonIpe(43?LzK z?qTi^)*@6S_+oX`&Bo;kOOj92AHDzfs-CoRy-8b%)dtZ0JUCNfMOnTSban1-QbO>%(do87|v}P^Tk$TE^$i zwsa)CB$1k=e&zPnws}Erut#_+q06x`93;RPDDIA^WHHdg>U4C%#hG5a^Ew4`YpPfI z)+%c|toBHECMbgL^NaLUj3O(QX2k8Q9UIlg#2jRu*|*asWU5mzS?ENcixQS7pSvU) z+YCMX zQ|aU{1#Qq^ji|L1#Zg<&SrO}G4CwFJ!>|a(2o%{3noS2FPyF*;?9J=m7b2b*L2;vO zTIO3kr~Zx(o{Q@c1CLI8coxc;GhMqo6CpX5^burB7QYq69)x_j)}M?o4$K@dqv#h4 znJ1j!i+o{BFUOmSe|6eu9FT}( zpXfs@2)r*xgfZ3@2ZB^qWT+txe*}MhK;Z7V?{Bk+D}i5W>bjh#=db5GBBr-fN;KXZQ4Lp}P3NvO+TTT!cD2E~s zy~CZ(*_9myoRArvGGDK0#n}&_J~2gpX+dbAw570BE^94JIhJDo=A(0%Q)6CD#id5o zo{RDo_AUQIMS6$CNeaGF#&F7={joo%io;1)-qZ*{+Zt^Zl8NROgB5>co)XY zb#ni9oY;c#wtA%|riaOyY%w#F822vY9AKZOnMhLTHqidPh_>RL1I|628fnhZlYpv{ zlULYBvIQ5GJ7FA^v1`HDWne)pUw%#gI&+lz-J8tS47aPWEPsm%1I6dvB?f2C)lDc6tSnONFE*|p~mFThUUFzKQUoy7k!6yqfO#P$z-W<~Hb&&kD& zofp93$A3J5-E3TA;$e}&8h4m_IphqlqS$m`YF6u+_g+o=Fm(%dBdZj zQ?lY)dwePk)W2=iwNSPq&)r>jvFcgeZ{u5Y`|G^W&sM3aNgK`IpZ1xobwRN3z2ESO z%8+xkJ?&d$ylKlutL$NCW-CK1f+#wkTcq~XWHY)z2!sE>OZozNv_xK{a>7oufwA}h z{99mGU(SP^RJ(E2iNk1R=BJw%%;~4Mtjop)lfv8f)eE?oQR51rN7iq06rM&730P8w zTV*xVNE^M>?7^f0wMB>j4Kvq(U3Py4Gf%g1nk4&MEKCgbQ0TEs7i)xAQ)DQMZrQ^t znu(z^yoP0G_M7l2%thaFZz@OxWbawVg}xBTIK30Z->5QxHj9Z)6}0)JX(MCLS%*rj z7+93od|9HkK2AmJw5;4<7PWUX#8DZum2SaKdUog0@9&fbo4D_tZbSYInuu+o(Zmru z^FNf|WGblY40U=z!+q2C&w1dCWH;|WhDU`(4?w16t)$&zQP*!~@HwyIB|MTDUJc4_ zFYw~^rwjjETb3ufFe^>nvRQ1<)7`$NuJ8tbp{L0Mg1+1Kdh~8_k zN_CHb4`k?h4pqIj`}#I6SU0|fK9ZU8`t}2ee@pWS-Y)qmbSNEtd*$f0PWTWKDSB?# z8JI5fbc=3f3D*>e58-)iM+7whqa$uOzOR(2g?v{L+Y<*i`ccoh75cYTjdVl^Rn*w zgvleY_foffV_Emd%yDDbJwKSd!}}fQ88;mt-(&4^=fyS?w%!%|-y0b7RRH>UxQKEt zY^M|mJ41g4R8a+EsLl`#g#p|2zpOx5@wkR(mSdzVT(EV~>F4a+n)|J*#Gr24yj@w% zqFvbqGV?K_f_J7p`w;eWPw?mf1`L6FU*Ay#egtDq&q7-ED1=n_}!yy z-kC}TeQdv7=*9)&YUf@G>HWOx=|GICl#?WJknNkh`k|K+37@7P&>3^u1B{-MsH4K! zv@K#cH3ls_b2lIKPTRRQyHy?2?%O1!nd@ciPsVoL`(q1C!5A=Dm-&h}Mv?d#Q=486 z36Y^{@WScs-nxGQsVI>PJiycKOlgK~q(2P(Q6T0kR9_OX{V@CA)1mTP>$~rO!u_5( zNSVpXZa+&I-m{Be-WL3^=vlrEUF%``m?#-kh#HR*>MYPt-1;1=`1FYPN25uU#$(jf z8S>WEAIjUT;>_pF3^|V%Au%m(H@eww^ufdu>{0o~&0C=ENht)tEIP@Sew?YU$kW*w zIi&w6Y8)D>lQ?;>cX7a@1DNRIIA+61sMjkqf&RkC)owZcygDDcDm+TNYM4WX-Bvhc zoNUj3ev9V^$e!JZ?MdnYQV5mtm<1{xI2ON5Vo@Oci%i}9JDyq3m2{$Vt=Q@3Tw#`8 zbD_bN1d*c#o0-8OJEOh1`4H~)^I^_aEp8d)y)85{8D$;NPah8pDO%tNDV;}Z3pzHZ zpvin}M+GRtv|jf+c9h6NakpZ+@1>A@RDelCUxB&l;_=HkAm8%u=C2pGe2!M*;5#oIi7 zK%>&Rv4yXI;?I;igX58&A7MfVa|7xeG5%aPX5101_d?8Wo3kKr5$C>w7p|=6RP^sS z?{2*wToIC90FP-Hk#5JHIHVp}Ik4o$Mr<_4eOs|ISZ*!}UGamEQA-w4Z&vDDnDgpu zzxyMsEU3u!v>5lZ>dXJu56U6Ji*E_Oi0j*GlzK6y&eZoDl{3sC8t3R+NAtC~^qBi~ z{{QBRu&7uDjTDQ~rzU zUd7)nuCEQr6GK1KVu|cOK#?R0l(j6YRGc8z=pe9{R___FM3<$zK3~;6>qfQ#;!IXT zx=ca?Abw62lY(n&`qOBEd!mNo0mK_w3s+vetKM1wLT0qC83bZIV8XJV1|vu(aMgtM z_ipZzAZYe|p`lgf34lu9-#K6!Yaj{C9so@*+^6wx$a}HrOw;Pw6Yq8$G+vDbd7ov=;t^ZECqf;7 z;I_s(eNxX@Y)cOh7t z{KE~%fQ5zs|0LES^~Bo^Fw0E%q-JeeI>%r{ifk9{+jci+`e@X(As6y71 zRG})}xr^4sm;n$2USazt)AI$iBcC38dt6Ai5xq##YbKQ*Q z9Z_BALo##JZ*6kL2L`?34ELsbmW#2(722wC@Rg*sI1%fear=sU0gTxu_yXD@9`t2= zB6jUa3Mds`l8zhsQOj&_JeQ@i)r{f9Xa%nykDah{v%&&SR9rB3#bepmkG&+VFI{5t z>wlDfVxbJYzO?nco1iN6lK79n8=c0nF}t~(RBvhEGAfT&hOB)UF*1YoR0jmaGT7O> zGMc75!pPjA^s~7x9v`}*7-E=H^ngH(arvJtBXm#qjl05u^9hAq;ytg1RzS4I1_v%Y zKjtOL7bMf9ZR`^;s6$xOIieBnywW!QbyVKCR9zS>gjrRE$fxBAx-Om>o~(x{D5Q1s zE($}3?kICISBITR$i_G;8&~QxFED!iFnMs)S{oO^Ao9AiUfE}-90dD~y%h21AVEem zyckv>D*&%`kMT(SlG}#k%6Oyw+U_UiLq>OIdE{cI9%}zv%D%!N+%29)%{7fHvGu57 z04gffW|0(C|KVl!PBO{Pn2}w8+Zo=$^A|jaCOSHv2uaDSfah1C((T7}d{BzNp_7t! znt-<6CBTHL;WKPmj^uh8wn3Q40B`1Z53yV+R^;_uV8sd6;#Uh|2t7W-c}zlFQeR`s zy0OJ^GpD3nPt5P84N9BrWhw3+OGiNF8;o%y8fIF)O2qoKP$vF-B}Yu`YTRA#1Z>u` zB8mkDX>Bv*&grMGuhMg<-2#|?&hcSmVB`=()@Ia8g<0%XJ&bjx{o@%yFfF`4!h=et zW7B(}LSZ6}4^e>+!kpwKN)rosI4iln?8s+5UlAI6&U>?O%ue=OJUZR%OE#wJLz+En zN;i|rHMYF&WC=~Ba5aG@$p-G4mo1BUAaXj}r?wY=E&gamZ@b(5%>Yb-*y{;IRLIWQ+<%}?q zvzMNo^?Z&aajz}ep)Z*^t^L?Ue%>u6W}M9wR54B|A-;v_pn&HQB)j0ax~Z-O?GU6; zj!Jle?0|}qJ|#ii+j(?ejm+-|efl-17ziIu@3;n&jFREL4F7DNI%Lths5P=ckc2dV z8(M_3b_OK^8|Xd@RWbO7XbVrvdu}DPalJ?`zmNa*vh*gUCQeC(AZ8Gwk>WL5NA&m7 zW}|v*1DM?YvG(qI&V2-%VM9wrP5CD0Kk|m+a2L^Tau3s(>7EZ_av3N87OoyUa%lKG zV2YmjoirpWQV80?V?&ZZsw_9s7MXS|#KkMd;n#JA0!O12oMSKeuQQ1a<8&Bi!`0mF z!G)brTqhNLm@B=NlbHpmdy5j32CoaU^;v!&!2u#o0-7+1Zf7(U8UX|u*63_#@*M$# z7tgyo`$tmx-dyHr5Jbni#S|zpuEC)>_6HWOP%t^;!fZ0tgXUZ{(koxcq?5|mB!~+8 zjl|o#w)h;zE6Oa~e>hMd`W{O(jZ$nrUmhkSQqZ~^AaCYR(Qvy$`jLm#yrfLB^C-i8 zo2+-+v%Yk!RQ_{oJsT=t4rWdfpS`iE=@$?46bUyxMibNqp!Q$yT=dlU?C@+q&Upd= zaap<2-!$pFS%g2!y79yB=>K%jjE{S9e0oK*s&};WQYOlcywI5W1j@l{{Gz6y=3HM zZT$PLK=v;01(S{h;JX-nZAmp)AkBAT$b;>gdvUNHLV6I{g#GFNb+fZ#Q9-ESi(ds_ zRxnyiF^!RGF0q2b?=i1F~-g;L0}-M}k3;)O+n9|GLQDiXzt z(ZusHGFTKbt3k;e3h`r>RhyE{QjrMrbIj{~w)U1;o~w$gjpqCG`qA-M_Aht;4L@%J zIravn%P0S0fut39NXqhC2D0C&@$gOHdG#kVi0%}*2q%P3OBtk97o2xpu-9X z&zPjxGoQ}=a!Gta5Sf{`#(X>)iz*1Z@*|&MEb?t~gsA3r3DJt`n8axt3*Y6peIF|# zU2!*!j@mS5>J;LaFQk!l1+GC-(05p7VbfN{U%f;lt>^CJGWxPMvKa=c7J!~&t48G; zuEu|U%0+VzYmaX*>UZ z{d1c~qX`B1f*lT0&DtHmi!D?Br08*ZN=Lknf_)_F1%$05k-78=V@kev!Ju9ic$C+L{q3H z71|b<3ay?vl}rg_Y(}_rhp>;#B2t2`=N5kZ`8jE6tQIG}U!)y&HStFjn^4H4mP|t4 zje%BbPznHbRT}?SjYKfrWWx(&q%NJdyxkg192sf97G66)^t0XVj(jQ&?7|{gcuif7 zJ1*eooZ|!2xgYRZ-sfB3xp}D~&@Wwxf7`PbAI zOI0RAU8xMU?WES~tW?5PwrIHLv7DX1`O2mVo5{3c@r1FeW5WIz`^WN^KUmw!i*mis zPYn>?Yu>m1s>;6qotJ!;X*R;VkR!zp{hJ5Ril-YxS1Xwjrsi~ETGbLUEJ1|sXDI*% zpj%SZxL3$k1a1xuoP$y^aXL`&9-{fWIFH@z^hgvyhSH^$e}3=#jK<3Df#qujTb-&?1;TD!A0TgY>sw%A4pP?1{0_yYlQ{cF%#=r|e-BDZiNcpt@zU*A4pYobUVAojF zeL#ofhV}&ae|Dm-a5M8+{Te6{*B!2xaM4~R(RWk)tJG_M!mGZ8sxhN&ukMl7NF@e~ zu!~;JLAZ7cJZ3T{y@{G|d=sdDue@ji|l7=-S`J@cE(<;ySRN<`}`gwPD26c*GDpyzrQxm=pE(7J+} zU3Vz<0Y`3Z;_(t=WF;-g8ci@?lBn)c%Gyrr))bUxG#U0s0j?o)v$wZ?D%OB9;Oj(< z2C+3?Be3-Xi+n<0@%bd-9BcYw6y0k1sHOK2$x*veEeS0&oVi66_&AdAlXc^iTD4*#@)(cCOu(~H`0_dG)e#`r9aC#~m!@UfbU+n3&AJN0Oq=VO){ekb{hFJ7^ciuk=*IgP7;8g+9YVA3H>8 z_><(F84@a9H~vKwE`%=u@yaISWCWg{M~HK@xk*a=K;v9 z&^lG@%*;E4NX_26oi>DQ9Y0H2ZnpqLiZvnPr!YE$!(%0CBy^(~@hO>Stt@SI{oVe| zMe2k)eeTn@%11&qA^~utIAH$L6Wx7Wn~#2E#rOm)bnKd7t8NjCAg!a+m-d!;-;v_m zLa>`!g&doBHbqenK7b(;DWIsC2}xMy1p${C8UY#D6R-UcmRl%tbU#2xw67W8+EzK% zEC*43gxhG{BbBmSvdT3$`EcaGDVR@T@o%wf{^rqA5mV~_R1rlR6z}d6A^(JX8{T_w z5qK@~8!R}c_J-E*(nK7D;`n0tW*|E(G6U}+kBIFTUz1m@8^7K^iA0x!hs$nABJ}b- zVgOuc<@0gu!d1%AB5cxUZ(!kkcN6{} zD_%vpNst#nKHES4vVUfYGff{#F+KPMU`W7c?UuPCLIQr(x=(BRhU2%v)|c}wX?$@6E>PGDkVUZtfeo< zyzIIx#oip?Bh$g|?EM_L8VZ@d!Wv_fis}t(9`M0WjlVnl=wU*j_owxDgFtzg_pQGj zSX#s7_U~5z2p212@!@vsH`%@xx@R~&_5Gqf3gG!N?(4j+N80a~Q-17qQ_SxbIF_%K z@EPgWXj^vtNH}>fqIKPo-}*|p)caresw%lVO+oFhM{StxYs)G0%gu5-Jj(6-&9gqE zin@M?X{Z0D33*oP?Ej~; hQm=PV--S~#V@Ssl9}NGJ@i&Px7N*uF7$a!H{{yKCe^dYf diff --git a/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml b/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml index cd3bbf59c..484ed45b1 100644 --- a/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml +++ b/examples/benchmarks/TCTS/workflow_config_tcts_Alpha360.yaml @@ -22,9 +22,9 @@ data_handler_config: &data_handler_config - class: CSRankNorm kwargs: fields_group: label - label: ["Ref($close, -1) / $close - 1", - "Ref($close, -2) / Ref($close, -1) - 1", - "Ref($close, -3) / Ref($close, -2) - 1"] + label: ["Ref($close, -2) / Ref($close, -1) - 1", + "Ref($close, -3) / Ref($close, -1) - 1", + "Ref($close, -4) / Ref($close, -1) - 1"] port_analysis_config: &port_analysis_config strategy: class: TopkDropoutStrategy @@ -53,9 +53,8 @@ task: d_feat: 6 hidden_size: 64 num_layers: 2 - dropout: 0.0 + dropout: 0.3 n_epochs: 200 - lr: 1e-3 early_stop: 20 batch_size: 800 metric: loss @@ -64,12 +63,11 @@ task: fore_optimizer: adam weight_optimizer: adam output_dim: 3 - fore_lr: 5e-4 - weight_lr: 5e-4 + fore_lr: 2e-3 + weight_lr: 2e-3 steps: 3 - target_label: 1 + target_label: 0 lowest_valid_performance: 0.993 - seed: 0 dataset: class: DatasetH module_path: qlib.data.dataset @@ -93,8 +91,7 @@ task: kwargs: ana_long_short: False ann_scaler: 252 - label_col: 1 - class: PortAnaRecord module_path: qlib.workflow.record_temp kwargs: - config: *port_analysis_config + config: *port_analysis_config \ No newline at end of file diff --git a/qlib/contrib/model/pytorch_tcts.py b/qlib/contrib/model/pytorch_tcts.py index da7fda5f5..c0dae98e4 100644 --- a/qlib/contrib/model/pytorch_tcts.py +++ b/qlib/contrib/model/pytorch_tcts.py @@ -61,8 +61,9 @@ class TCTS(Model): weight_lr=5e-7, steps=3, GPU=0, - seed=None, target_label=0, + mode="soft", + seed=None, lowest_valid_performance=0.993, **kwargs ): @@ -87,6 +88,7 @@ class TCTS(Model): self.weight_lr = weight_lr self.steps = steps self.target_label = target_label + self.mode = mode self.lowest_valid_performance = lowest_valid_performance self._fore_optimizer = fore_optimizer self._weight_optimizer = weight_optimizer @@ -100,6 +102,8 @@ class TCTS(Model): "\nn_epochs : {}" "\nbatch_size : {}" "\nearly_stop : {}" + "\ntarget_label : {}" + "\nmode : {}" "\nloss_type : {}" "\nvisible_GPU : {}" "\nuse_GPU : {}" @@ -111,6 +115,8 @@ class TCTS(Model): n_epochs, batch_size, early_stop, + target_label, + mode, loss, GPU, self.use_gpu, @@ -120,9 +126,17 @@ class TCTS(Model): def loss_fn(self, pred, label, weight): - loc = torch.argmax(weight, 1) - loss = (pred - label[np.arange(weight.shape[0]), loc]) ** 2 - return torch.mean(loss) + if self.mode == "hard": + loc = torch.argmax(weight, 1) + loss = (pred - label[np.arange(weight.shape[0]), loc]) ** 2 + return torch.mean(loss) + + elif self.mode == "soft": + loss = (pred - label.transpose(0, 1)) ** 2 + return torch.mean(loss * weight.transpose(0, 1)) + + else: + raise NotImplementedError("mode {} is not supported!".format(self.mode)) def train_epoch(self, x_train, y_train, x_valid, y_valid): @@ -132,6 +146,10 @@ class TCTS(Model): indices = np.arange(len(x_train_values)) np.random.shuffle(indices) + task_embedding = torch.zeros([self.batch_size, self.output_dim]) + task_embedding[:, self.target_label] = 1 + task_embedding = task_embedding.to(self.device) + init_fore_model = copy.deepcopy(self.fore_model) for p in init_fore_model.parameters(): p.init_fore_model = False @@ -155,12 +173,13 @@ class TCTS(Model): init_pred = init_fore_model(feature) pred = self.fore_model(feature) - dis = init_pred - label.transpose(0, 1) - weight_feature = torch.cat((feature, dis.transpose(0, 1), label, init_pred.view(-1, 1)), 1) + weight_feature = torch.cat( + (feature, dis.transpose(0, 1), label, init_pred.view(-1, 1), task_embedding), 1 + ) weight = self.weight_model(weight_feature) - loss = self.loss_fn(pred, label, weight) # hard + loss = self.loss_fn(pred, label, weight) self.fore_optimizer.zero_grad() loss.backward() @@ -188,11 +207,11 @@ class TCTS(Model): pred = self.fore_model(feature) dis = pred - label.transpose(0, 1) - weight_feature = torch.cat((feature, dis.transpose(0, 1), label, pred.view(-1, 1)), 1) + weight_feature = torch.cat((feature, dis.transpose(0, 1), label, pred.view(-1, 1), task_embedding), 1) weight = self.weight_model(weight_feature) loc = torch.argmax(weight, 1) - valid_loss = torch.mean((pred - label[:, 0]) ** 2) - loss = torch.mean(-valid_loss * torch.log(weight[np.arange(weight.shape[0]), loc])) + valid_loss = torch.mean((pred - label[:, abs(self.target_label)]) ** 2) + loss = torch.mean(valid_loss * torch.log(weight[np.arange(weight.shape[0]), loc])) self.weight_optimizer.zero_grad() loss.backward() @@ -207,7 +226,6 @@ class TCTS(Model): self.fore_model.eval() - scores = [] losses = [] indices = np.arange(len(x_values)) @@ -277,7 +295,7 @@ class TCTS(Model): dropout=self.dropout, ) self.weight_model = MLPModel( - d_feat=360 + 2 * self.output_dim + 1, + d_feat=360 + 3 * self.output_dim + 1, hidden_size=self.hidden_size, num_layers=self.num_layers, dropout=self.dropout, @@ -303,8 +321,6 @@ class TCTS(Model): best_loss = np.inf best_epoch = 0 stop_round = 0 - fore_best_param = copy.deepcopy(self.fore_optimizer.state_dict()) - weight_best_param = copy.deepcopy(self.weight_optimizer.state_dict()) for epoch in range(self.n_epochs): print("Epoch:", epoch)