mirror of
https://github.com/microsoft/qlib.git
synced 2026-07-09 14:00:55 +08:00
Merge nested main (#597)
* MVP for Indian Stocks in qlib using yahooquery * cleaned with black * cleaned with black * add YahooNormalizeIN and YahooNormalizeIN1d * cleaned the code * added 1min for IN and also updated readme * update comments * fix comments * recorder support upload both raw file and directory * fix comments * Update README.md * Fix docs of QlibRecorder * sort index after loader (#538) make sure the fetch method is based on a index-sorted pd.DataFrame * refactor online serving rolling api * refactor TRA * format by black * fix horizon * fix TRA when use single head * clean up * improve pretrain * update README * fix tra when logdir is None * fix tra when logdir is None * Update strategy.py * Update README.md * Update README.md * Conda Suggestion * code standard docs * Update ensemble.py (#560) * Fix CI Bug (#575) Co-authored-by: yuxwang <anduinnn@foxmail.com> * Update gen.py (#576) * Fix multi-process loop calls (#574) * check lexsort in the 'lazy_sort_index' function (#566) * check lexsort * check lexsort * lexsort comment * lexsort comment * Delete .DS_Store * Update README.md * bug fix & use oracle transport pretrain * mend * Add `backend_freq_config` parameter, support multi-freq uri * Add sample_config to QlibDataLoader, support multi-freq * add multi-freq example * get_cls_kwargs renamed get_callable_kwargs * support multi-freq uri * Add inst_processors to D.features * Fix typo * Fix the index type of the multi-freq example * Fix duplicate mlflow directories in tests * Add DataPathManager to QlibConfig && modify inst_processors to supports list only * Modify the default value in the multi_freq example * Modify client-server mode and dataset-cache to disable inst_processor * Add wheel package to github CI * fix comment * Update FAQ.rst * Update README.md Fix wrong link * Update the docs of TaskManager (#586) * Update manage.py * update yaml * update run_all_model * Modify the Feature to be case sensitive (#589) * update README * remove verbose * fix spell bug * fix typos (#592) * Update Release Note * fix portfolio bug * Add calendar support for resample * add freq kwargs * test.yml: Remove redundant code (#595) * Supporting shared processor (#596) * Supporting shared processor * fix readonly reverse bug * remove pytests dependency * with fit bug * fix parameter error * fix comments * Fix undefined names in Python code (#599) * Update pytorch_tabnet.py $ `flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics` ``` ./qlib/qlib/contrib/model/pytorch_tabnet.py:567:38: F821 undefined name 'inp' self.independ.append(GLU(inp, out_dim, vbs=vbs)) ^ ./qlib/examples/model_rolling/task_manager_rolling.py:75:18: F821 undefined name 'task_train' run_task(task_train, self.task_pool, experiment_name=self.experiment_name) ^ 2 F821 undefined name 'task_train' 2 ``` * Fix undefined names in Python code * from qlib.model.trainer import task_train * update seed * fix some docstring * add comments * Fix SimpleDatasetCache * Update setup.py updated classifiers * Update setup.py change to matplotlib==3.3 * Update python-publish.yml added python 3.9 * updategrade version number * Update model list * fix the type of filter_pipe * fix comment * fix record_temp * update cvxpy version * Update code_standard.rst (#587) * Update code_standard.rst * Update docs/developer/code_standard.rst Co-authored-by: you-n-g <you-n-g@users.noreply.github.com> Co-authored-by: you-n-g <you-n-g@users.noreply.github.com> * Add file lock for MLflowExpManager (#619) * fix torch version * Share version number (#620) * Update initialization.rst (#622) * Update initialization.rst * Update docs/start/initialization.rst Co-authored-by: you-n-g <you-n-g@users.noreply.github.com> * Update docs/start/initialization.rst Co-authored-by: you-n-g <you-n-g@users.noreply.github.com> Co-authored-by: you-n-g <you-n-g@users.noreply.github.com> * fix bugs for running previous exmaple * fix deal amount bug * update change doc (#623) * Add files via upload * Update README.md * Update README.md * Update README.md * Delete change doc.gif * Add files via upload * Update README.md * Delete change doc.gif * Add files via upload * Delete change doc.gif * Add files via upload * Update README.md Co-authored-by: you-n-g <you-n-g@users.noreply.github.com> Co-authored-by: you-n-g <you-n-g@users.noreply.github.com> * update doc * simplify run all model * fix run all model bug * Fix Models (#483) * fix gat dataset * fix tft model * Update tft.py * Fix tft.py Co-authored-by: Pengrong Zhu <zhu.pengrong@foxmail.com> * type and skip empty exp * fix model yaml config * fix tft import bug * skip empty result * fix model and yaml bug * fix wrong generate parameter * Modify multi-freq example (#626) * modify the example of multi-freq * add Copyright * add a comment to average_ops.py * modify the example of multi-freq * add comment to multi_freq_handler.py * add the Ref expression description to multi_freq_handler.py * add expression description to multi_freq_handler.py * update images * fix workflow and update framework Co-authored-by: Gaurav <2796gaurav@gmail.com> Co-authored-by: 2796gaurav <17353992+2796gaurav@users.noreply.github.com> Co-authored-by: bxdd <bxd98@126.com> Co-authored-by: Young <afe.young@gmail.com> Co-authored-by: you-n-g <you-n-g@users.noreply.github.com> Co-authored-by: Dong Zhou <Zhou.Dong@microsoft.com> Co-authored-by: ZhangTP1996 <ztp18@mails.tsinghua.edu.cn> Co-authored-by: demon143 <59681577+demon143@users.noreply.github.com> Co-authored-by: Wangwuyi123 <51237097+Wangwuyi123@users.noreply.github.com> Co-authored-by: yuxwang <anduinnn@foxmail.com> Co-authored-by: Pengrong Zhu <zhu.pengrong@foxmail.com> Co-authored-by: Mark Zhao <50850474+markzhao98@users.noreply.github.com> Co-authored-by: cslwqxx <cslwqxx@users.noreply.github.com> Co-authored-by: Dong Zhou <evanzd@users.noreply.github.com> Co-authored-by: SaintMalik <37118134+saintmalik@users.noreply.github.com> Co-authored-by: Christian Clauss <cclauss@me.com> Co-authored-by: Anurag Kumar <mailanu98@gmail.com> Co-authored-by: demon143 <785696300@qq.com>
This commit is contained in:
wangwenxi-handsome
committed by
GitHub
GitHub
parent
163e3c6266
commit
3760a18a8d
@@ -18,6 +18,7 @@ from ...config import C
|
||||
from ...utils import parse_config, transform_end_date, init_instance_by_config
|
||||
from ...utils.serial import Serializable
|
||||
from .utils import fetch_df_by_index, fetch_df_by_col
|
||||
from ...utils import lazy_sort_index
|
||||
from pathlib import Path
|
||||
from .loader import DataLoader
|
||||
|
||||
@@ -146,7 +147,8 @@ class DataHandler(Serializable):
|
||||
# Setup data.
|
||||
# _data may be with multiple column index level. The outer level indicates the feature set name
|
||||
with TimeInspector.logt("Loading data"):
|
||||
self._data = self.data_loader.load(self.instruments, self.start_time, self.end_time)
|
||||
# make sure the fetch method is based on a index-sorted pd.DataFrame
|
||||
self._data = lazy_sort_index(self.data_loader.load(self.instruments, self.start_time, self.end_time))
|
||||
# TODO: cache
|
||||
|
||||
CS_ALL = "__all" # return all columns with single-level index column
|
||||
@@ -303,11 +305,14 @@ class DataHandlerLP(DataHandler):
|
||||
|
||||
# process type
|
||||
PTYPE_I = "independent"
|
||||
# - self._infer will be processed by infer_processors
|
||||
# - self._learn will be processed by learn_processors
|
||||
# - self._infer will be processed by shared_processors + infer_processors
|
||||
# - self._learn will be processed by shared_processors + learn_processors
|
||||
|
||||
# NOTE:
|
||||
PTYPE_A = "append"
|
||||
# - self._infer will be processed by infer_processors
|
||||
# - self._learn will be processed by infer_processors + learn_processors
|
||||
|
||||
# - self._infer will be processed by shared_processors + infer_processors
|
||||
# - self._learn will be processed by shared_processors + infer_processors + learn_processors
|
||||
# - (e.g. self._infer processed by learn_processors )
|
||||
|
||||
def __init__(
|
||||
@@ -316,8 +321,9 @@ class DataHandlerLP(DataHandler):
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
data_loader: Union[dict, str, DataLoader] = None,
|
||||
infer_processors=[],
|
||||
learn_processors=[],
|
||||
infer_processors: List = [],
|
||||
learn_processors: List = [],
|
||||
shared_processors: List = [],
|
||||
process_type=PTYPE_A,
|
||||
drop_raw=False,
|
||||
**kwargs,
|
||||
@@ -368,7 +374,8 @@ class DataHandlerLP(DataHandler):
|
||||
# Setup preprocessor
|
||||
self.infer_processors = [] # for lint
|
||||
self.learn_processors = [] # for lint
|
||||
for pname in "infer_processors", "learn_processors":
|
||||
self.shared_processors = [] # for lint
|
||||
for pname in "infer_processors", "learn_processors", "shared_processors":
|
||||
for proc in locals()[pname]:
|
||||
getattr(self, pname).append(
|
||||
init_instance_by_config(
|
||||
@@ -383,9 +390,12 @@ class DataHandlerLP(DataHandler):
|
||||
super().__init__(instruments, start_time, end_time, data_loader, **kwargs)
|
||||
|
||||
def get_all_processors(self):
|
||||
return self.infer_processors + self.learn_processors
|
||||
return self.shared_processors + self.infer_processors + self.learn_processors
|
||||
|
||||
def fit(self):
|
||||
"""
|
||||
fit data without processing the data
|
||||
"""
|
||||
for proc in self.get_all_processors():
|
||||
with TimeInspector.logt(f"{proc.__class__.__name__}"):
|
||||
proc.fit(self._data)
|
||||
@@ -398,30 +408,68 @@ class DataHandlerLP(DataHandler):
|
||||
"""
|
||||
self.process_data(with_fit=True)
|
||||
|
||||
@staticmethod
|
||||
def _run_proc_l(
|
||||
df: pd.DataFrame, proc_l: List[processor_module.Processor], with_fit: bool, check_for_infer: bool
|
||||
) -> pd.DataFrame:
|
||||
for proc in proc_l:
|
||||
if check_for_infer and not proc.is_for_infer():
|
||||
raise TypeError("Only processors usable for inference can be used in `infer_processors` ")
|
||||
with TimeInspector.logt(f"{proc.__class__.__name__}"):
|
||||
if with_fit:
|
||||
proc.fit(df)
|
||||
df = proc(df)
|
||||
return df
|
||||
|
||||
@staticmethod
|
||||
def _is_proc_readonly(proc_l: List[processor_module.Processor]):
|
||||
"""
|
||||
NOTE: it will return True if `len(proc_l) == 0`
|
||||
"""
|
||||
for p in proc_l:
|
||||
if not p.readonly():
|
||||
return False
|
||||
return True
|
||||
|
||||
def process_data(self, with_fit: bool = False):
|
||||
"""
|
||||
process_data data. Fun `processor.fit` if necessary
|
||||
|
||||
Notation: (data) [processor]
|
||||
|
||||
# data processing flow of self.process_type == DataHandlerLP.PTYPE_I
|
||||
(self._data)-[shared_processors]-(_shared_df)-[learn_processors]-(_learn_df)
|
||||
\
|
||||
-[infer_processors]-(_infer_df)
|
||||
|
||||
# data processing flow of self.process_type == DataHandlerLP.PTYPE_A
|
||||
(self._data)-[shared_processors]-(_shared_df)-[infer_processors]-(_infer_df)-[learn_processors]-(_learn_df)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
with_fit : bool
|
||||
The input of the `fit` will be the output of the previous processor
|
||||
"""
|
||||
# data for inference
|
||||
_infer_df = self._data
|
||||
if len(self.infer_processors) > 0 and not self.drop_raw: # avoid modifying the original data
|
||||
_infer_df = _infer_df.copy()
|
||||
# shared data processors
|
||||
# 1) assign
|
||||
_shared_df = self._data
|
||||
if not self._is_proc_readonly(self.shared_processors): # avoid modifying the original data
|
||||
_shared_df = _shared_df.copy()
|
||||
# 2) process
|
||||
_shared_df = self._run_proc_l(_shared_df, self.shared_processors, with_fit=with_fit, check_for_infer=True)
|
||||
|
||||
# data for inference
|
||||
# 1) assign
|
||||
_infer_df = _shared_df
|
||||
if not self._is_proc_readonly(self.infer_processors): # avoid modifying the original data
|
||||
_infer_df = _infer_df.copy()
|
||||
# 2) process
|
||||
_infer_df = self._run_proc_l(_infer_df, self.infer_processors, with_fit=with_fit, check_for_infer=True)
|
||||
|
||||
for proc in self.infer_processors:
|
||||
if not proc.is_for_infer():
|
||||
raise TypeError("Only processors usable for inference can be used in `infer_processors` ")
|
||||
with TimeInspector.logt(f"{proc.__class__.__name__}"):
|
||||
if with_fit:
|
||||
proc.fit(_infer_df)
|
||||
_infer_df = proc(_infer_df)
|
||||
self._infer = _infer_df
|
||||
|
||||
# data for learning
|
||||
# 1) assign
|
||||
if self.process_type == DataHandlerLP.PTYPE_I:
|
||||
_learn_df = self._data
|
||||
elif self.process_type == DataHandlerLP.PTYPE_A:
|
||||
@@ -429,14 +477,11 @@ class DataHandlerLP(DataHandler):
|
||||
_learn_df = _infer_df
|
||||
else:
|
||||
raise NotImplementedError(f"This type of input is not supported")
|
||||
|
||||
if len(self.learn_processors) > 0: # avoid modifying the original data
|
||||
if not self._is_proc_readonly(self.learn_processors): # avoid modifying the original data
|
||||
_learn_df = _learn_df.copy()
|
||||
for proc in self.learn_processors:
|
||||
with TimeInspector.logt(f"{proc.__class__.__name__}"):
|
||||
if with_fit:
|
||||
proc.fit(_learn_df)
|
||||
_learn_df = proc(_learn_df)
|
||||
# 2) process
|
||||
_learn_df = self._run_proc_l(_learn_df, self.learn_processors, with_fit=with_fit, check_for_infer=False)
|
||||
|
||||
self._learn = _learn_df
|
||||
|
||||
if self.drop_raw:
|
||||
|
||||
Reference in New Issue
Block a user