liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-09 05:50:59 +08:00
Code Activity

DDG-DA paper code (#743)

Browse Source 
* Merge data selection to main

* Update trainer for reweighter

* Typos fixed.

* update data selection interface

* successfully run exp after refactor some interface

* data selection share handler &  trainer

* fix meta model time series bug

* fix online workflow set_uri bug

* fix set_uri bug

* updawte ds docs and delay trainer bug

* docs

* resume reweighter

* add reweighting result

* fix qlib model import

* make recorder more friendly

* fix experiment workflow bug

* commit for merging master incase of conflictions

* Successful run DDG-DA with a single command

* remove unused code

* asdd more docs

* Update README.md

* Update & fix some bugs.

* Update configuration & remove debug functions

* Update README.md

* Modfify horizon from code rather than yaml

* Update performance in README.md

* fix part comments

* Remove unfinished TCTS.

* Fix some details.

* Update meta docs

* Update README.md of the benchmarks_dynamic

* Update README.md files

* Add README.md to the rolling_benchmark baseline.

* Refine the docs and link

* Rename README.md in benchmarks_dynamic.

* Remove comments.

* auto download data

Co-authored-by: wendili-cs <wendili.academic@qq.com>
Co-authored-by: demon143 <785696300@qq.com>
This commit is contained in:
you-n-g
2022-01-10 16:52:37 +08:00
committed by GitHub
parent 184ce34a34
commit cf35562e84
52 changed files with 2441 additions and 456 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
qlib/workflow/__init__.py
View File

@@ -2,7 +2,7 @@
# Licensed under the MIT License.
from contextlib import contextmanager
from typing import Any, Dict, Text, Optional
from typing import Text, Optional, Any, Dict, Text, Optional
from .expm import ExpManager
from .exp import Experiment
from .recorder import Recorder
@@ -15,7 +15,7 @@ class QlibRecorder:
A global system that helps to manage the experiments.
"""
def __init__(self, exp_manager):
def __init__(self, exp_manager: ExpManager):
self.exp_manager: ExpManager = exp_manager
def __repr__(self):
@@ -341,6 +341,10 @@ class QlibRecorder:
def set_uri(self, uri: Optional[Text]):
"""
Method to reset the current uri of current experiment manager.
NOTE:
- When the uri is refer to a file path, please using the absolute path instead of strings like "~/mlruns/"
The backend don't support strings like this.
"""
self.exp_manager.set_uri(uri)
@@ -501,13 +505,13 @@ class QlibRecorder:
raise ValueError(
"You can choose only one of `local_path`(save the files in a path) or `kwargs`(pass in the objects directly)"
)
self.get_exp().get_recorder().save_objects(local_path, artifact_path, **kwargs)
self.get_exp().get_recorder(start=True).save_objects(local_path, artifact_path, **kwargs)
def load_object(self, name: Text):
"""
Method for loading an object from artifacts in the experiment in the uri.
"""
return self.get_exp().get_recorder().load_object(name)
return self.get_exp().get_recorder(start=True).load_object(name)
def log_params(self, **kwargs):
"""
@@ -532,7 +536,7 @@ class QlibRecorder:
keyword argument:
name1=value1, name2=value2, ...
"""
self.get_exp().get_recorder().log_params(**kwargs)
self.get_exp().get_recorder(start=True).log_params(**kwargs)
def log_metrics(self, step=None, **kwargs):
"""
@@ -557,7 +561,7 @@ class QlibRecorder:
keyword argument:
name1=value1, name2=value2, ...
"""
self.get_exp().get_recorder().log_metrics(step, **kwargs)
self.get_exp().get_recorder(start=True).log_metrics(step, **kwargs)
def set_tags(self, **kwargs):
"""
@@ -582,7 +586,7 @@ class QlibRecorder:
keyword argument:
name1=value1, name2=value2, ...
"""
self.get_exp().get_recorder().set_tags(**kwargs)
self.get_exp().get_recorder(start=True).set_tags(**kwargs)
class RecorderWrapper(Wrapper):

40
qlib/workflow/exp.py
View File

@@ -1,7 +1,7 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
from typing import Dict, Union
from typing import Dict, List, Union
import mlflow, logging
from mlflow.entities import ViewType
from mlflow.exceptions import MlflowException
@@ -22,6 +22,7 @@ class Experiment:
self.id = id
self.name = name
self.active_recorder = None # only one recorder can running each time
self._default_rec_name = "abstract_recorder"
def __repr__(self):
return "{name}(id={id}, info={info})".format(name=self.__class__.__name__, id=self.id, info=self.info)
@@ -150,7 +151,7 @@ class Experiment:
create : boolean
create the recorder if it hasn't been created before.
start : boolean
start the new recorder if one is created.
start the new recorder if one is **created**.
Returns
-------
@@ -214,7 +215,10 @@ class Experiment:
"""
raise NotImplementedError(f"Please implement the `_get_recorder` method")
def list_recorders(self, **flt_kwargs) -> Dict[str, Recorder]:
RT_D = "dict" # return type dict
RT_L = "list" # return type list
def list_recorders(self, rtype: str = RT_D, **flt_kwargs) -> Union[List[Recorder], Dict[str, Recorder]]:
"""
List all the existing recorders of this experiment. Please first get the experiment instance before calling this method.
If user want to use the method `R.list_recorders()`, please refer to the related API document in `QlibRecorder`.
@@ -225,7 +229,11 @@ class Experiment:
Returns
-------
A dictionary (id -> recorder) of recorder information that being stored.
The return type depent on `rtype`
if `rtype` == "dict":
A dictionary (id -> recorder) of recorder information that being stored.
elif `rtype` == "list":
A list of Recorder.
"""
raise NotImplementedError(f"Please implement the `list_recorders` method.")
@@ -326,9 +334,16 @@ class MLflowExperiment(Experiment):
UNLIMITED = 50000 # FIXME: Mlflow can only list 50000 records at most!!!!!!!
def list_recorders(
self, max_results: int = UNLIMITED, status: Union[str, None] = None, filter_string: str = ""
) -> Dict[str, Recorder]:
self,
rtype=Experiment.RT_D,
max_results: int = UNLIMITED,
status: Union[str, None] = None,
filter_string: str = "",
):
"""
Quoting docs of search_runs
> The default ordering is to sort by start_time DESC, then run_id.
Parameters
----------
max_results : int
@@ -342,10 +357,17 @@ class MLflowExperiment(Experiment):
runs = self._client.search_runs(
self.id, run_view_type=ViewType.ACTIVE_ONLY, max_results=max_results, filter_string=filter_string
)
recorders = dict()
rids = []
recorders = []
for i in range(len(runs)):
recorder = MLflowRecorder(self.id, self._uri, mlflow_run=runs[i])
if status is None or recorder.status == status:
recorders[runs[i].info.run_id] = recorder
rids.append(runs[i].info.run_id)
recorders.append(recorder)
return recorders
if rtype == Experiment.RT_D:
return dict(zip(rids, recorders))
elif rtype == Experiment.RT_L:
return recorders
else:
raise NotImplementedError(f"This type of input is not supported")

10
qlib/workflow/expm.py
View File

@@ -17,7 +17,7 @@ from .recorder import Recorder
from ..log import get_module_logger
from ..utils.exceptions import ExpAlreadyExistError
logger = get_module_logger("workflow", logging.INFO)
logger = get_module_logger("workflow")
class ExpManager:
@@ -279,8 +279,9 @@ class ExpManager:
"""
if uri is None:
logger.info("No tracking URI is provided. Use the default tracking URI.")
self._current_uri = self.default_uri
if self._current_uri is None:
logger.debug("No tracking URI is provided. Use the default tracking URI.")
self._current_uri = self.default_uri
else:
# Temporarily re-set the current uri as the uri argument.
self._current_uri = uri
@@ -290,6 +291,7 @@ class ExpManager:
def _set_uri(self):
"""
Customized features for subclasses' set_uri function.
This method is designed for the underlying experiment backend storage.
"""
raise NotImplementedError(f"Please implement the `_set_uri` method.")
@@ -351,8 +353,6 @@ class MLflowExpManager(ExpManager):
if self.active_experiment is not None:
self.active_experiment.end(recorder_status)
self.active_experiment = None
# When an experiment end, we will release the current uri.
self._current_uri = None
def create_exp(self, experiment_name: Optional[Text] = None):
assert experiment_name is not None

107
qlib/workflow/record_temp.py
View File

@@ -14,8 +14,9 @@ from ..data.dataset import DatasetH
from ..data.dataset.handler import DataHandlerLP
from ..backtest import backtest as normal_backtest
from ..log import get_module_logger
from ..utils import flatten_dict, class_casting
from ..utils import fill_placeholder, flatten_dict, class_casting, get_date_by_shift
from ..utils.time import Freq
from ..utils.data import deepcopy_basic_type
from ..contrib.eva.alpha import calc_ic, calc_long_short_return, calc_long_short_prec
@@ -175,9 +176,10 @@ class SignalRecord(RecordTemp):
del params["data_key"]
# The backend handler should be DataHandler
raw_label = dataset.prepare(**params)
except AttributeError:
except AttributeError as e:
# The data handler is initialize with `drop_raw=True`...
# So raw_label is not available
logger.warning(f"Exception: {e}")
raw_label = None
return raw_label
@@ -203,6 +205,35 @@ class SignalRecord(RecordTemp):
return ["pred.pkl", "label.pkl"]
class ACRecordTemp(RecordTemp):
"""Automatically checking record template"""
def __init__(self, recorder, skip_existing=False):
self.skip_existing = skip_existing
super().__init__(recorder=recorder)
def generate(self, *args, **kwargs):
"""automatically checking the files and then run the concrete generating task"""
if self.skip_existing:
try:
self.check(include_self=True, parents=False)
except FileNotFoundError:
pass # continue to generating metrics
else:
logger.info("The results has previously generated, Generation skipped.")
return
try:
self.check()
except FileNotFoundError:
logger.warning("The dependent data does not exists. Generation skipped.")
return
return self._generate(*args, **kwargs)
def _generate(self, *args, **kwargs):
raise NotImplementedError(f"Please implement the `_generate` method")
class HFSignalRecord(SignalRecord):
"""
This is the Signal Analysis Record class that generates the analysis results such as IC and IR. This class inherits the ``RecordTemp`` class.
@@ -250,7 +281,7 @@ class HFSignalRecord(SignalRecord):
return ["ic.pkl", "ric.pkl", "long_pre.pkl", "short_pre.pkl", "long_short_r.pkl", "long_avg_r.pkl"]
class SigAnaRecord(RecordTemp):
class SigAnaRecord(ACRecordTemp):
"""
This is the Signal Analysis Record class that generates the analysis results such as IC and IR. This class inherits the ``RecordTemp`` class.
"""
@@ -259,39 +290,23 @@ class SigAnaRecord(RecordTemp):
depend_cls = SignalRecord
def __init__(self, recorder, ana_long_short=False, ann_scaler=252, label_col=0, skip_existing=False):
super().__init__(recorder=recorder)
super().__init__(recorder=recorder, skip_existing=skip_existing)
self.ana_long_short = ana_long_short
self.ann_scaler = ann_scaler
self.label_col = label_col
self.skip_existing = skip_existing
def generate(self, label: Optional[pd.DataFrame] = None, **kwargs):
def _generate(self, label: Optional[pd.DataFrame] = None, **kwargs):
"""
Parameters
----------
label : Optional[pd.DataFrame]
Label should be a dataframe.
"""
if self.skip_existing:
try:
self.check(include_self=True, parents=False)
except FileNotFoundError:
pass # continue to generating metrics
else:
logger.info("The results has previously generated, Generation skipped.")
return
try:
self.check()
except FileNotFoundError:
logger.warning("The dependent data does not exists. Generation skipped.")
return
pred = self.load("pred.pkl")
if label is None:
label = self.load("label.pkl")
if label is None or not isinstance(label, pd.DataFrame) or label.empty:
logger.warn(f"Empty label.")
logger.warning(f"Empty label.")
return
ic, ric = calc_ic(pred.iloc[:, 0], label.iloc[:, self.label_col])
metrics = {
@@ -328,7 +343,7 @@ class SigAnaRecord(RecordTemp):
return paths
class PortAnaRecord(RecordTemp):
class PortAnaRecord(ACRecordTemp):
"""
This is the Portfolio Analysis Record class that generates the analysis results such as those of backtest. This class inherits the ``RecordTemp`` class.
@@ -339,14 +354,35 @@ class PortAnaRecord(RecordTemp):
"""
artifact_path = "portfolio_analysis"
depend_cls = SignalRecord
def __init__(
self,
recorder,
config,
config: dict = { # Default config for daily trading
"strategy": {
"class": "TopkDropoutStrategy",
"module_path": "qlib.contrib.strategy",
"kwargs": {"signal": "<PRED>", "topk": 50, "n_drop": 5},
},
"backtest": {
"start_time": None,
"end_time": None,
"account": 100000000,
"benchmark": "SH000300",
"exchange_kwargs": {
"limit_threshold": 0.095,
"deal_price": "close",
"open_cost": 0.0005,
"close_cost": 0.0015,
"min_cost": 5,
},
},
},
risk_analysis_freq: Union[List, str] = None,
indicator_analysis_freq: Union[List, str] = None,
indicator_analysis_method=None,
skip_existing=False,
**kwargs,
):
"""
@@ -363,7 +399,12 @@ class PortAnaRecord(RecordTemp):
indicator_analysis_method : str, optional, default by None
the candidated values include 'mean', 'amount_weighted', 'value_weighted'
"""
super().__init__(recorder=recorder, **kwargs)
super().__init__(recorder=recorder, skip_existing=skip_existing, **kwargs)
# We only deepcopy_basic_type because
# - We don't want to affect the config outside.
# - We don't want to deepcopy complex object to avoid overhead
config = deepcopy_basic_type(config)
self.strategy_config = config["strategy"]
_default_executor_config = {
@@ -405,7 +446,21 @@ class PortAnaRecord(RecordTemp):
ret_freq.extend(self._get_report_freq(executor_config["kwargs"]["inner_executor"]))
return ret_freq
def generate(self, **kwargs):
def _generate(self, **kwargs):
pred = self.load("pred.pkl")
# replace the "<PRED>" with prediction saved before
placehorder_value = {"<PRED>": pred}
for k in "executor_config", "strategy_config":
setattr(self, k, fill_placeholder(getattr(self, k), placehorder_value))
# if the backtesting time range is not set, it will automatically extract time range from the prediction file
dt_values = pred.index.get_level_values("datetime")
if self.backtest_config["start_time"] is None:
self.backtest_config["start_time"] = dt_values.min()
if self.backtest_config["end_time"] is None:
self.backtest_config["end_time"] = get_date_by_shift(dt_values.max(), 1)
# custom strategy and get backtest
portfolio_metric_dict, indicator_dict = normal_backtest(
executor=self.executor_config, strategy=self.strategy_config, **self.backtest_config

5
qlib/workflow/recorder.py
View File

@@ -306,8 +306,9 @@ class MLflowRecorder(Recorder):
self.end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if self.status != Recorder.STATUS_S:
self.status = status
with TimeInspector.logt("waiting `async_log`"):
self.async_log.wait()
if self.async_log is not None:
with TimeInspector.logt("waiting `async_log`"):
self.async_log.wait()
self.async_log = None
def save_objects(self, local_path=None, artifact_path=None, **kwargs):

49
qlib/workflow/task/gen.py
View File

@@ -6,7 +6,7 @@ TaskGenerator module can generate many tasks based on TaskGen and some task temp
import abc
import copy
import pandas as pd
from typing import List, Union, Callable
from typing import Dict, List, Union, Callable
from qlib.utils import transform_end_date
from .utils import TimeAdjuster
@@ -119,14 +119,38 @@ def handler_mod(task: dict, rolling_gen):
pass
except TypeError:
# May be the handler is a string. `"handler.pkl"["kwargs"]` will raise TypeError
# e.g. a dumped file like file:///<file>/
pass
def trunc_segments(ta: TimeAdjuster, segments: Dict[str, pd.Timestamp], days, test_key="test"):
"""
To avoid the leakage of future information, the segments should be truncated according to the test start_time
NOTE:
This function will change segments **inplace**
"""
# adjust segment
test_start = min(t for t in segments[test_key] if t is not None)
for k in list(segments.keys()):
if k != test_key:
segments[k] = ta.truncate(segments[k], test_start, days)
class RollingGen(TaskGen):
ROLL_EX = TimeAdjuster.SHIFT_EX # fixed start date, expanding end date
ROLL_SD = TimeAdjuster.SHIFT_SD # fixed segments size, slide it from start date
def __init__(self, step: int = 40, rtype: str = ROLL_EX, ds_extra_mod_func: Union[None, Callable] = handler_mod):
def __init__(
self,
step: int = 40,
rtype: str = ROLL_EX,
ds_extra_mod_func: Union[None, Callable] = handler_mod,
test_key="test",
train_key="train",
trunc_days: int = None,
task_copy_func: Callable = copy.deepcopy,
):
"""
Generate tasks for rolling
@@ -139,14 +163,20 @@ class RollingGen(TaskGen):
ds_extra_mod_func: Callable
A method like: handler_mod(task: dict, rg: RollingGen)
Do some extra action after generating a task. For example, use ``handler_mod`` to modify the end time of the handler of a dataset.
trunc_days: int
trunc some data to avoid future information leakage
task_copy_func: Callable
the function to copy entire task. This is very useful when user want to share something between tasks
"""
self.step = step
self.rtype = rtype
self.ds_extra_mod_func = ds_extra_mod_func
self.ta = TimeAdjuster(future=True)
self.test_key = "test"
self.train_key = "train"
self.test_key = test_key
self.train_key = train_key
self.trunc_days = trunc_days
self.task_copy_func = task_copy_func
def _update_task_segs(self, task, segs):
# update segments of this task
@@ -191,7 +221,7 @@ class RollingGen(TaskGen):
break
prev_seg = segments
t = copy.deepcopy(task) # deepcopy is necessary to avoid modify task inplace
t = self.task_copy_func(task) # deepcopy is necessary to avoid replace task inplace
self._update_task_segs(t, segments)
yield t
@@ -247,7 +277,7 @@ class RollingGen(TaskGen):
"""
res = []
t = copy.deepcopy(task)
t = self.task_copy_func(task)
# calculate segments
@@ -258,6 +288,8 @@ class RollingGen(TaskGen):
# 2) and init test segments
test_start_idx = self.ta.align_idx(segments[self.test_key][0])
segments[self.test_key] = (self.ta.get(test_start_idx), self.ta.get(test_start_idx + self.step - 1))
if self.trunc_days is not None:
trunc_segments(self.ta, segments, self.trunc_days, self.test_key)
# update segments of this task
self._update_task_segs(t, segments)
@@ -313,10 +345,7 @@ class MultiHorizonGenBase(TaskGen):
# adjust segment
segments = self.ta.align_seg(t["dataset"]["kwargs"]["segments"])
test_start = min(t for t in segments[self.test_key] if t is not None)
for k in list(segments.keys()):
if k != self.test_key:
segments[k] = self.ta.truncate(segments[k], test_start, hr + self.label_leak_n)
trunc_segments(self.ta, segments, days=hr + self.label_leak_n, test_key=self.test_key)
t["dataset"]["kwargs"]["segments"] = segments
res.append(t)
return res

26
qlib/workflow/task/utils.py
View File

@@ -100,7 +100,7 @@ class TimeAdjuster:
idx : int
index of the calendar
"""
if idx >= len(self.cals):
if idx is None or idx >= len(self.cals):
return None
return self.cals[idx]
@@ -123,6 +123,9 @@ class TimeAdjuster:
-------
index : int
"""
if time_point is None:
# `None` indicates unbounded index/boarder
return None
time_point = pd.Timestamp(time_point)
if tp_type == "start":
idx = bisect.bisect_left(self.cals, time_point)
@@ -158,6 +161,8 @@ class TimeAdjuster:
Returns:
pd.Timestamp
"""
if time_point is None:
return None
return self.cals[self.align_idx(time_point, tp_type=tp_type)]
def align_seg(self, segment: Union[dict, tuple]) -> Union[dict, tuple]:
@@ -201,6 +206,10 @@ class TimeAdjuster:
days : int
The trading days to be truncated
the data in this segment may need 'days' data
`days` are based on the `test_start`.
For example, if the label contains the information of 2 days in the near future, the prediction horizon 1 day.
(e.g. the prediction target is `Ref($close, -2)/Ref($close, -1) - 1`)
the days should be 2 + 1 == 3 days.
Returns
---------
@@ -220,10 +229,17 @@ class TimeAdjuster:
SHIFT_SD = "sliding"
SHIFT_EX = "expanding"
def _add_step(self, index, step):
if index is None:
return None
return index + step
def shift(self, seg: tuple, step: int, rtype=SHIFT_SD) -> tuple:
"""
Shift the datatime of segment
If there are None (which indicates unbounded index) in the segment, this method will return None.
Parameters
----------
seg :
@@ -245,13 +261,13 @@ class TimeAdjuster:
if isinstance(seg, tuple):
start_idx, end_idx = self.align_idx(seg[0], tp_type="start"), self.align_idx(seg[1], tp_type="end")
if rtype == self.SHIFT_SD:
start_idx += step
end_idx += step
start_idx = self._add_step(start_idx, step)
end_idx = self._add_step(end_idx, step)
elif rtype == self.SHIFT_EX:
end_idx += step
end_idx = self._add_step(end_idx, step)
else:
raise NotImplementedError(f"This type of input is not supported")
if start_idx > len(self.cals):
if start_idx is not None and start_idx > len(self.cals):
raise KeyError("The segment is out of valid calendar")
return self.get(start_idx), self.get(end_idx)
else: