From 0be57d51bedfde919c99482689d52531a1117267 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 10 Dec 2020 09:00:00 +0000 Subject: [PATCH 01/20] support register custom feature ops easily --- qlib/data/ops.py | 100 ++++++++++++++++++++++------------------- qlib/utils/__init__.py | 19 +++++++- 2 files changed, 71 insertions(+), 48 deletions(-) diff --git a/qlib/data/ops.py b/qlib/data/ops.py index e17c0e4e6..737507a67 100644 --- a/qlib/data/ops.py +++ b/qlib/data/ops.py @@ -13,6 +13,7 @@ from scipy.stats import percentileofscore from .base import Expression, ExpressionOps from ..log import get_module_logger +from ..utils import OpsWrapper try: from ._libs.rolling import rolling_slope, rolling_rsquare, rolling_resi @@ -21,53 +22,6 @@ except ImportError as err: print("Do not import qlib package in the repository directory!") raise -__all__ = ( - "Ref", - "Max", - "Min", - "Sum", - "Mean", - "Std", - "Var", - "Skew", - "Kurt", - "Med", - "Mad", - "Slope", - "Rsquare", - "Resi", - "Rank", - "Quantile", - "Count", - "EMA", - "WMA", - "Corr", - "Cov", - "Delta", - "Abs", - "Sign", - "Log", - "Power", - "Add", - "Sub", - "Mul", - "Div", - "Greater", - "Less", - "And", - "Or", - "Not", - "Gt", - "Ge", - "Lt", - "Le", - "Eq", - "Ne", - "Mask", - "IdxMax", - "IdxMin", - "If", -) np.seterr(invalid="ignore") @@ -1430,3 +1384,55 @@ class Cov(PairRolling): def __init__(self, feature_left, feature_right, N): super(Cov, self).__init__(feature_left, feature_right, N, "cov") + +Operators = OpsWrapper() + +OpsList = [ + Ref, + Max, + Min, + Sum, + Mean, + Std, + Var, + Skew, + Kurt, + Med, + Mad, + Slope, + Rsquare, + Resi, + Rank, + Quantile, + Count, + EMA, + WMA, + Corr, + Cov, + Delta, + Abs, + Sign, + Log, + Power, + Add, + Sub, + Mul, + Div, + Greater, + Less, + And, + Or, + Not, + Gt, + Ge, + Lt, + Le, + Eq, + Ne, + Mask, + IdxMax, + IdxMin, + If, +] + +Operators.register(OpsList) \ No newline at end of file diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index ab67b67e3..18e621993 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -162,7 +162,7 @@ def parse_field(field): # - $open+$close -> Feature("open")+Feature("close") if not isinstance(field, str): field = str(field) - return re.sub(r"\$(\w+)", r'Feature("\1")', field) + return re.sub(r"\$(\w+)", r'Feature("\1")', re.sub(r"(\w+\s*)\(", r"Operators.\1(", field)) def get_module_by_module_path(module_path): @@ -728,3 +728,20 @@ def load_dataset(path_or_obj): elif extension == ".csv": return pd.read_csv(path_or_obj, parse_dates=True, index_col=[0, 1]) raise ValueError(f"unsupported file type `{extension}`") + +#################### Operator Wrapper ##################### + +class OpsWrapper(object): + """Ops Wrapper""" + + def __init__(self): + self._ops = {} + def register(self, ops_list): + + for operator in ops_list: + self._ops[operator.__name__] = operator + + def __getattr__(self, key): + if self._ops is {}: + raise AttributeError("Please run qlib.init() first using qlib to register ops") + return self._ops[key] \ No newline at end of file From 87cc52cd05f0e07fef7b641bc5d3f7f6710cadb7 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 10 Dec 2020 09:02:43 +0000 Subject: [PATCH 02/20] black format --- qlib/data/ops.py | 3 ++- qlib/utils/__init__.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/qlib/data/ops.py b/qlib/data/ops.py index 737507a67..6836b49b4 100644 --- a/qlib/data/ops.py +++ b/qlib/data/ops.py @@ -1385,6 +1385,7 @@ class Cov(PairRolling): def __init__(self, feature_left, feature_right, N): super(Cov, self).__init__(feature_left, feature_right, N, "cov") + Operators = OpsWrapper() OpsList = [ @@ -1435,4 +1436,4 @@ OpsList = [ If, ] -Operators.register(OpsList) \ No newline at end of file +Operators.register(OpsList) diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 18e621993..f7b406f58 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -729,13 +729,16 @@ def load_dataset(path_or_obj): return pd.read_csv(path_or_obj, parse_dates=True, index_col=[0, 1]) raise ValueError(f"unsupported file type `{extension}`") + #################### Operator Wrapper ##################### + class OpsWrapper(object): """Ops Wrapper""" def __init__(self): self._ops = {} + def register(self, ops_list): for operator in ops_list: @@ -744,4 +747,4 @@ class OpsWrapper(object): def __getattr__(self, key): if self._ops is {}: raise AttributeError("Please run qlib.init() first using qlib to register ops") - return self._ops[key] \ No newline at end of file + return self._ops[key] From 729b57e4a7143a19cb36c18ccbba9df810e76888 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 10 Dec 2020 09:11:12 +0000 Subject: [PATCH 03/20] add example script --- examples/workflow_with_custom_ops.py | 83 ++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 examples/workflow_with_custom_ops.py diff --git a/examples/workflow_with_custom_ops.py b/examples/workflow_with_custom_ops.py new file mode 100644 index 000000000..f75df6166 --- /dev/null +++ b/examples/workflow_with_custom_ops.py @@ -0,0 +1,83 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import sys +from pathlib import Path + +import qlib +import numpy as np +import pandas as pd +from qlib.data import D +from qlib.data.ops import Operators, ElemOperator, PairOperator +from qlib.config import REG_CN +from qlib.utils import exists_qlib_data + + +class Diff(ElemOperator): + """Feature First Difference + + Parameters + ---------- + feature : Expression + feature instance + + Returns + ---------- + Expression + a feature instance with first difference + """ + + def __init__(self, feature): + super(Diff, self).__init__(feature, "diff") + + def _load_internal(self, instrument, start_index, end_index, freq): + series = self.feature.load(instrument, start_index, end_index, freq) + return series.diff() + + def get_extended_window_size(self): + lft_etd, rght_etd = self.feature.get_extended_window_size() + return lft_etd + 1, rght_etd + + +class Distance(PairOperator): + """Feature Distance + + Parameters + ---------- + feature : Expression + feature instance + + Returns + ---------- + Expression + a feature instance with distance + """ + + def __init__(self, feature_left, feature_right): + super(Distance, self).__init__(feature_left, feature_right, "distance") + + def _load_internal(self, instrument, start_index, end_index, freq): + series_left = self.feature_left.load(instrument, start_index, end_index, freq) + series_right = self.feature_right.load(instrument, start_index, end_index, freq) + return np.abs(series_left - series_right) + + +if __name__ == "__main__": + + # register custom operators + OpsList = [Diff, Distance] + Operators.register(OpsList) + # use default data + provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir + if not exists_qlib_data(provider_uri): + print(f"Qlib data is not found in {provider_uri}") + sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts"))) + from get_data import GetData + + GetData().qlib_data(target_dir=provider_uri, region=REG_CN) + + qlib.init(provider_uri=provider_uri, region=REG_CN) + + instruments = ["SH600000"] + fields = ["Diff($close)", "Distance($close, Ref($close, 1))"] + print(D.features(instruments, fields, start_time="2010-01-01", end_time="2017-12-31", freq="day")) From 16450c2876f0b717f4226fec753168f6c461cdd5 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 10 Dec 2020 09:54:05 +0000 Subject: [PATCH 04/20] fix import --- qlib/data/cache.py | 2 +- qlib/data/data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qlib/data/cache.py b/qlib/data/cache.py index 3fab2b527..027ef89ef 100644 --- a/qlib/data/cache.py +++ b/qlib/data/cache.py @@ -32,7 +32,7 @@ from ..utils import ( from ..log import get_module_logger from .base import Feature -from .ops import * +from .ops import Operators class QlibCacheException(RuntimeError): diff --git a/qlib/data/data.py b/qlib/data/data.py index a4c3d63f2..c142b6b26 100644 --- a/qlib/data/data.py +++ b/qlib/data/data.py @@ -20,7 +20,7 @@ from multiprocessing import Pool from .cache import H from ..config import C -from .ops import * +from .ops import Operators from ..log import get_module_logger from ..utils import parse_field, read_bin, hash_args, normalize_cache_fields from .base import Feature From 2de812f26234717dbb8684fe862fdab8fa890996 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 10 Dec 2020 10:04:09 +0000 Subject: [PATCH 05/20] update ops docs --- docs/component/data.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/component/data.rst b/docs/component/data.rst index cb1103e72..d80887b12 100644 --- a/docs/component/data.rst +++ b/docs/component/data.rst @@ -195,6 +195,7 @@ Feature - `ExpressionOps` `ExpressionOps` will use operator for feature construction. To know more about ``Operator``, please refer to `Operator API <../reference/api.html#module-qlib.data.ops>`_. + Also, ``Qlib`` supports users define their own custom ``Operator``, an example has been given in ``qlib/examples/workflow_with_custom_ops.py``. To know more about ``Feature``, please refer to `Feature API <../reference/api.html#module-qlib.data.base>`_. From 0cdc5e125a5abdbe3800a7b5a70f6a9ccf96bc91 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 10 Dec 2020 10:08:29 +0000 Subject: [PATCH 06/20] update docs --- docs/component/data.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/component/data.rst b/docs/component/data.rst index d80887b12..c1e000b15 100644 --- a/docs/component/data.rst +++ b/docs/component/data.rst @@ -195,7 +195,7 @@ Feature - `ExpressionOps` `ExpressionOps` will use operator for feature construction. To know more about ``Operator``, please refer to `Operator API <../reference/api.html#module-qlib.data.ops>`_. - Also, ``Qlib`` supports users define their own custom ``Operator``, an example has been given in ``qlib/examples/workflow_with_custom_ops.py``. + Also, ``Qlib`` supports users to define their own custom ``Operator``, an example has been given in ``qlib/examples/workflow_with_custom_ops.py``. To know more about ``Feature``, please refer to `Feature API <../reference/api.html#module-qlib.data.base>`_. From 7d97fd39ce70c75369ac2324e710c16697ee4c60 Mon Sep 17 00:00:00 2001 From: bxdd Date: Mon, 21 Dec 2020 12:06:42 +0000 Subject: [PATCH 07/20] update ops register --- qlib/data/ops.py | 22 +++++++++- qlib/utils/__init__.py | 20 --------- .../test_register_custom_ops.py | 41 +++++++++---------- 3 files changed, 40 insertions(+), 43 deletions(-) rename examples/workflow_with_custom_ops.py => tests/test_register_custom_ops.py (66%) diff --git a/qlib/data/ops.py b/qlib/data/ops.py index 6836b49b4..861a69bf9 100644 --- a/qlib/data/ops.py +++ b/qlib/data/ops.py @@ -13,7 +13,6 @@ from scipy.stats import percentileofscore from .base import Expression, ExpressionOps from ..log import get_module_logger -from ..utils import OpsWrapper try: from ._libs.rolling import rolling_slope, rolling_rsquare, rolling_resi @@ -1386,8 +1385,27 @@ class Cov(PairRolling): super(Cov, self).__init__(feature_left, feature_right, N, "cov") -Operators = OpsWrapper() +class OpsWrapper(object): + """Ops Wrapper""" + def __init__(self): + self._ops = {} + + def register(self, ops_list): + for operator in ops_list: + if operator.__name__ in self._ops: + get_module_logger(self.__class__.__name__).warning( + "The custom operator [{}] will override the qlib default definition".format(operator.__name__) + ) + self._ops[operator.__name__] = operator + + def __getattr__(self, key): + if key not in self._ops: + raise AttributeError("The operator [{}] is not registered".format(key)) + return self._ops[key] + + +Operators = OpsWrapper() OpsList = [ Ref, Max, diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index f7b406f58..b08f9426d 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -728,23 +728,3 @@ def load_dataset(path_or_obj): elif extension == ".csv": return pd.read_csv(path_or_obj, parse_dates=True, index_col=[0, 1]) raise ValueError(f"unsupported file type `{extension}`") - - -#################### Operator Wrapper ##################### - - -class OpsWrapper(object): - """Ops Wrapper""" - - def __init__(self): - self._ops = {} - - def register(self, ops_list): - - for operator in ops_list: - self._ops[operator.__name__] = operator - - def __getattr__(self, key): - if self._ops is {}: - raise AttributeError("Please run qlib.init() first using qlib to register ops") - return self._ops[key] diff --git a/examples/workflow_with_custom_ops.py b/tests/test_register_custom_ops.py similarity index 66% rename from examples/workflow_with_custom_ops.py rename to tests/test_register_custom_ops.py index f75df6166..e109fffae 100644 --- a/examples/workflow_with_custom_ops.py +++ b/tests/test_register_custom_ops.py @@ -1,16 +1,18 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. import sys +import unittest +import numpy as np +import pandas as pd from pathlib import Path import qlib -import numpy as np -import pandas as pd from qlib.data import D from qlib.data.ops import Operators, ElemOperator, PairOperator from qlib.config import REG_CN from qlib.utils import exists_qlib_data +from qlib.tests import TestAutoData class Diff(ElemOperator): @@ -61,23 +63,20 @@ class Distance(PairOperator): series_right = self.feature_right.load(instrument, start_index, end_index, freq) return np.abs(series_left - series_right) +class TestRegiterCustomOps(TestAutoData): + + def test_regiter_custom_ops(self): + OpsList = [Diff, Distance] + Operators.register(OpsList) + instruments = ["SH600000"] + fields = ["Diff($close)", "Distance($close, Ref($close, 1))"] + print(D.features(instruments, fields, start_time="2010-01-01", end_time="2017-12-31", freq="day")) + if __name__ == "__main__": + unittest.main(verbosity=10) - # register custom operators - OpsList = [Diff, Distance] - Operators.register(OpsList) - # use default data - provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir - if not exists_qlib_data(provider_uri): - print(f"Qlib data is not found in {provider_uri}") - sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts"))) - from get_data import GetData - - GetData().qlib_data(target_dir=provider_uri, region=REG_CN) - - qlib.init(provider_uri=provider_uri, region=REG_CN) - - instruments = ["SH600000"] - fields = ["Diff($close)", "Distance($close, Ref($close, 1))"] - print(D.features(instruments, fields, start_time="2010-01-01", end_time="2017-12-31", freq="day")) + # User could use following code to run test when using line_profiler + # td = TestDataset() + # td.setUpClass() + # td.testTSDataset() From 132df027a5fd5c38749b1d2ebde20fbb998e093e Mon Sep 17 00:00:00 2001 From: bxdd Date: Mon, 21 Dec 2020 12:09:25 +0000 Subject: [PATCH 08/20] update format --- tests/test_register_custom_ops.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_register_custom_ops.py b/tests/test_register_custom_ops.py index e109fffae..dac8f0910 100644 --- a/tests/test_register_custom_ops.py +++ b/tests/test_register_custom_ops.py @@ -63,8 +63,8 @@ class Distance(PairOperator): series_right = self.feature_right.load(instrument, start_index, end_index, freq) return np.abs(series_left - series_right) + class TestRegiterCustomOps(TestAutoData): - def test_regiter_custom_ops(self): OpsList = [Diff, Distance] Operators.register(OpsList) @@ -75,8 +75,3 @@ class TestRegiterCustomOps(TestAutoData): if __name__ == "__main__": unittest.main(verbosity=10) - - # User could use following code to run test when using line_profiler - # td = TestDataset() - # td.setUpClass() - # td.testTSDataset() From 53f501ac197e03e07f005b6b580eb8994b96a10f Mon Sep 17 00:00:00 2001 From: bxdd Date: Mon, 21 Dec 2020 12:44:27 +0000 Subject: [PATCH 09/20] del import --- tests/test_register_custom_ops.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_register_custom_ops.py b/tests/test_register_custom_ops.py index dac8f0910..1114e7cf0 100644 --- a/tests/test_register_custom_ops.py +++ b/tests/test_register_custom_ops.py @@ -4,14 +4,11 @@ import sys import unittest import numpy as np -import pandas as pd -from pathlib import Path import qlib from qlib.data import D from qlib.data.ops import Operators, ElemOperator, PairOperator from qlib.config import REG_CN -from qlib.utils import exists_qlib_data from qlib.tests import TestAutoData @@ -74,4 +71,4 @@ class TestRegiterCustomOps(TestAutoData): if __name__ == "__main__": - unittest.main(verbosity=10) + unittest.main() From b11712fa54b9708169082fce452d58fa3aa26abc Mon Sep 17 00:00:00 2001 From: bxdd Date: Wed, 23 Dec 2020 16:39:17 +0000 Subject: [PATCH 10/20] fix cant find ops error on Windows --- tests/test_register_custom_ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_register_custom_ops.py b/tests/test_register_custom_ops.py index 1114e7cf0..e47104c3a 100644 --- a/tests/test_register_custom_ops.py +++ b/tests/test_register_custom_ops.py @@ -60,11 +60,11 @@ class Distance(PairOperator): series_right = self.feature_right.load(instrument, start_index, end_index, freq) return np.abs(series_left - series_right) +OpsList = [Diff, Distance] +Operators.register(OpsList) class TestRegiterCustomOps(TestAutoData): def test_regiter_custom_ops(self): - OpsList = [Diff, Distance] - Operators.register(OpsList) instruments = ["SH600000"] fields = ["Diff($close)", "Distance($close, Ref($close, 1))"] print(D.features(instruments, fields, start_time="2010-01-01", end_time="2017-12-31", freq="day")) From cb3b6c5bde51917e993f00b8265fc3164d31c030 Mon Sep 17 00:00:00 2001 From: bxdd Date: Wed, 23 Dec 2020 16:41:32 +0000 Subject: [PATCH 11/20] black format --- tests/test_register_custom_ops.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_register_custom_ops.py b/tests/test_register_custom_ops.py index e47104c3a..7c11b45a3 100644 --- a/tests/test_register_custom_ops.py +++ b/tests/test_register_custom_ops.py @@ -60,9 +60,11 @@ class Distance(PairOperator): series_right = self.feature_right.load(instrument, start_index, end_index, freq) return np.abs(series_left - series_right) + OpsList = [Diff, Distance] Operators.register(OpsList) + class TestRegiterCustomOps(TestAutoData): def test_regiter_custom_ops(self): instruments = ["SH600000"] From d8f36df7f4cd944fc816834506705e621e0e2a20 Mon Sep 17 00:00:00 2001 From: bxdd Date: Wed, 23 Dec 2020 18:28:05 +0000 Subject: [PATCH 12/20] debug on macos --- qlib/data/ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qlib/data/ops.py b/qlib/data/ops.py index 861a69bf9..f80ca7450 100644 --- a/qlib/data/ops.py +++ b/qlib/data/ops.py @@ -1401,7 +1401,7 @@ class OpsWrapper(object): def __getattr__(self, key): if key not in self._ops: - raise AttributeError("The operator [{}] is not registered".format(key)) + raise AttributeError("The operator [{0}] is not registered, and all dict is {1}".format(key, self._ops)) return self._ops[key] From ecdfe49fd17504db3c6ad9ed9e3a5562f772eb20 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 19 Jan 2021 20:39:15 +0900 Subject: [PATCH 13/20] del custom ops test for check the CI status --- tests/test_register_custom_ops.py | 76 ------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 tests/test_register_custom_ops.py diff --git a/tests/test_register_custom_ops.py b/tests/test_register_custom_ops.py deleted file mode 100644 index 7c11b45a3..000000000 --- a/tests/test_register_custom_ops.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -import sys -import unittest -import numpy as np - -import qlib -from qlib.data import D -from qlib.data.ops import Operators, ElemOperator, PairOperator -from qlib.config import REG_CN -from qlib.tests import TestAutoData - - -class Diff(ElemOperator): - """Feature First Difference - - Parameters - ---------- - feature : Expression - feature instance - - Returns - ---------- - Expression - a feature instance with first difference - """ - - def __init__(self, feature): - super(Diff, self).__init__(feature, "diff") - - def _load_internal(self, instrument, start_index, end_index, freq): - series = self.feature.load(instrument, start_index, end_index, freq) - return series.diff() - - def get_extended_window_size(self): - lft_etd, rght_etd = self.feature.get_extended_window_size() - return lft_etd + 1, rght_etd - - -class Distance(PairOperator): - """Feature Distance - - Parameters - ---------- - feature : Expression - feature instance - - Returns - ---------- - Expression - a feature instance with distance - """ - - def __init__(self, feature_left, feature_right): - super(Distance, self).__init__(feature_left, feature_right, "distance") - - def _load_internal(self, instrument, start_index, end_index, freq): - series_left = self.feature_left.load(instrument, start_index, end_index, freq) - series_right = self.feature_right.load(instrument, start_index, end_index, freq) - return np.abs(series_left - series_right) - - -OpsList = [Diff, Distance] -Operators.register(OpsList) - - -class TestRegiterCustomOps(TestAutoData): - def test_regiter_custom_ops(self): - instruments = ["SH600000"] - fields = ["Diff($close)", "Distance($close, Ref($close, 1))"] - print(D.features(instruments, fields, start_time="2010-01-01", end_time="2017-12-31", freq="day")) - - -if __name__ == "__main__": - unittest.main() From 4fcfde7cfbb1dda3a7c5fa88ebe9b768bf0f89e8 Mon Sep 17 00:00:00 2001 From: zhupr Date: Wed, 20 Jan 2021 14:41:50 +0800 Subject: [PATCH 14/20] Initialization is split into: set_config and config_based_on_C --- qlib/__init__.py | 70 +++++++++--------------------------------- qlib/config.py | 3 ++ qlib/data/data.py | 9 +++--- qlib/utils/__init__.py | 56 +++++++++++++++++++++++++++++++-- 4 files changed, 76 insertions(+), 62 deletions(-) diff --git a/qlib/__init__.py b/qlib/__init__.py index f79b8c4f5..1f0c73949 100644 --- a/qlib/__init__.py +++ b/qlib/__init__.py @@ -6,88 +6,48 @@ __version__ = "0.6.0.dev" import os -import re -import sys -import copy import yaml import logging import platform import subprocess -from pathlib import Path -from .utils import can_use_cache, init_instance_by_config, get_module_by_module_path -from .workflow.utils import experiment_exit_handler +from .utils import set_config, config_based_on_c + # init qlib def init(default_conf="client", **kwargs): - from .config import C, REG_CN, REG_US, QlibConfig - from .data.data import register_all_wrappers - from .log import get_module_logger, set_log_with_config + from .config import C + from .log import get_module_logger from .data.cache import H - from .workflow import R, QlibRecorder - C.reset() H.clear() - _logging_config = C.logging_config - if "logging_config" in kwargs: - _logging_config = kwargs["logging_config"] - - # set global config - if _logging_config: - set_log_with_config(_logging_config) - # FIXME: this logger ignored the level in config - LOG = get_module_logger("Initialization", level=logging.INFO) - LOG.info(f"default_conf: {default_conf}.") + logger = get_module_logger("Initialization", level=logging.INFO) - C.set_mode(default_conf) - C.set_region(kwargs.get("region", C["region"] if "region" in C else REG_CN)) - - for k, v in kwargs.items(): - C[k] = v - if k not in C: - LOG.warning("Unrecognized config %s" % k) - - C.resolve_path() - - if not (C["expression_cache"] is None and C["dataset_cache"] is None): - # check redis - if not can_use_cache(): - LOG.warning( - f"redis connection failed(host={C['redis_host']} port={C['redis_port']}), cache will not be used!" - ) - C["expression_cache"] = None - C["dataset_cache"] = None + set_config(C, default_conf, **kwargs) # check path if server/local - if C.get_uri_type() == QlibConfig.LOCAL_URI: + if C.get_uri_type() == C.LOCAL_URI: if not os.path.exists(C["provider_uri"]): if C["auto_mount"]: - LOG.error( + logger.error( f"Invalid provider uri: {C['provider_uri']}, please check if a valid provider uri has been set. This path does not exist." ) else: - LOG.warning(f"auto_path is False, please make sure {C['mount_path']} is mounted") - elif C.get_uri_type() == QlibConfig.NFS_URI: + logger.warning(f"auto_path is False, please make sure {C['mount_path']} is mounted") + elif C.get_uri_type() == C.NFS_URI: _mount_nfs_uri(C) else: raise NotImplementedError(f"This type of URI is not supported") - LOG.info("qlib successfully initialized based on %s settings." % default_conf) - register_all_wrappers() - - LOG.info(f"data_path={C.get_data_path()}") - if "flask_server" in C: - LOG.info(f"flask_server={C['flask_server']}, flask_port={C['flask_port']}") + logger.info(f"flask_server={C['flask_server']}, flask_port={C['flask_port']}") - # set up QlibRecorder - exp_manager = init_instance_by_config(C["exp_manager"]) - qr = QlibRecorder(exp_manager) - R.register(qr) - # clean up experiment when python program ends - experiment_exit_handler() + config_based_on_c(C) + + logger.info("qlib successfully initialized based on %s settings." % default_conf) + logger.info(f"data_path={C.get_data_path()}") def _mount_nfs_uri(C): diff --git a/qlib/config.py b/qlib/config.py index 869ea99c9..c757fdb32 100644 --- a/qlib/config.py +++ b/qlib/config.py @@ -59,6 +59,9 @@ class Config: def update(self, *args, **kwargs): self.__dict__["_config"].update(*args, **kwargs) + def set_conf_from_C(self, config_c): + self.update(**config_c.__dict__["_config"]) + # REGION CONST REG_CN = "cn" diff --git a/qlib/data/data.py b/qlib/data/data.py index c142b6b26..89a2502f9 100644 --- a/qlib/data/data.py +++ b/qlib/data/data.py @@ -25,7 +25,7 @@ from ..log import get_module_logger from ..utils import parse_field, read_bin, hash_args, normalize_cache_fields from .base import Feature from .cache import DiskDatasetCache, DiskExpressionCache -from ..utils import Wrapper, init_instance_by_config, register_wrapper, get_module_by_module_path +from ..utils import Wrapper, init_instance_by_config, register_wrapper, get_module_by_module_path, config_based_on_c class CalendarProvider(abc.ABC): @@ -481,11 +481,10 @@ class DatasetProvider(abc.ABC): """ # FIXME: Windows OS or MacOS using spawn: https://docs.python.org/3.8/library/multiprocessing.html?highlight=spawn#contexts-and-start-methods - global C - C = g_config # NOTE: This place is compatible with windows, windows multi-process is spawn if getattr(ExpressionD, "_provider", None) is None: - register_all_wrappers() + C.set_conf_from_C(g_config) + config_based_on_c(g_config) obj = dict() for field in column_names: @@ -1043,7 +1042,7 @@ DatasetD = Wrapper() D = Wrapper() -def register_all_wrappers(): +def register_all_wrappers(C): """register_all_wrappers""" logger = get_module_logger("data") module = get_module_by_module_path("qlib.data") diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index b08f9426d..17a12ae9c 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -15,6 +15,7 @@ import bisect import shutil import difflib import hashlib +import logging import datetime import requests import tempfile @@ -26,8 +27,9 @@ import pandas as pd from pathlib import Path from typing import Union, Tuple -from ..config import C -from ..log import get_module_logger +from ..config import C, REG_CN +from ..log import get_module_logger, set_log_with_config + log = get_module_logger("utils") @@ -728,3 +730,53 @@ def load_dataset(path_or_obj): elif extension == ".csv": return pd.read_csv(path_or_obj, parse_dates=True, index_col=[0, 1]) raise ValueError(f"unsupported file type `{extension}`") + + +def set_config(config_c, default_conf="client", **kwargs): + + config_c.reset() + + _logging_config = config_c.logging_config + if "logging_config" in kwargs: + _logging_config = kwargs["logging_config"] + + # set global config + if _logging_config: + set_log_with_config(_logging_config) + + # FIXME: this logger ignored the level in config + logger = get_module_logger("Initialization", level=logging.INFO) + logger.info(f"default_conf: {default_conf}.") + + config_c.set_mode(default_conf) + config_c.set_region(kwargs.get("region", config_c["region"] if "region" in config_c else REG_CN)) + + for k, v in kwargs.items(): + if k not in config_c: + logger.warning("Unrecognized config %s" % k) + config_c[k] = v + + config_c.resolve_path() + + if not (config_c["expression_cache"] is None and config_c["dataset_cache"] is None): + # check redis + if not can_use_cache(): + logger.warning( + f"redis connection failed(host={config_c['redis_host']} port={config_c['redis_port']}), cache will not be used!" + ) + config_c["expression_cache"] = None + config_c["dataset_cache"] = None + + +def config_based_on_c(config_c): + from ..data.data import register_all_wrappers + from ..workflow import R, QlibRecorder + from ..workflow.utils import experiment_exit_handler + + register_all_wrappers(config_c) + # set up QlibRecorder + exp_manager = init_instance_by_config(config_c["exp_manager"]) + qr = QlibRecorder(exp_manager) + R.register(qr) + # clean up experiment when python program ends + experiment_exit_handler() From 6daaa7951987c6a8b4ba3af20ceb08bee0653ec5 Mon Sep 17 00:00:00 2001 From: bxdd Date: Wed, 20 Jan 2021 18:44:53 +0900 Subject: [PATCH 15/20] add register ops config --- qlib/config.py | 2 + qlib/data/ops.py | 54 +++++++++++++++---------- qlib/utils/__init__.py | 2 + tests/test_register_ops.py | 82 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 119 insertions(+), 21 deletions(-) create mode 100644 tests/test_register_ops.py diff --git a/qlib/config.py b/qlib/config.py index 1737c5b37..eb68a504d 100644 --- a/qlib/config.py +++ b/qlib/config.py @@ -187,6 +187,8 @@ MODE_CONF = { "timeout": 100, "logging_level": "INFO", "region": REG_CN, + ## Custom Operator + "custom_ops": [], }, } diff --git a/qlib/data/ops.py b/qlib/data/ops.py index c988db3b5..91f7349d2 100644 --- a/qlib/data/ops.py +++ b/qlib/data/ops.py @@ -1396,27 +1396,6 @@ class Cov(PairRolling): super(Cov, self).__init__(feature_left, feature_right, N, "cov") -class OpsWrapper(object): - """Ops Wrapper""" - - def __init__(self): - self._ops = {} - - def register(self, ops_list): - for operator in ops_list: - if operator.__name__ in self._ops: - get_module_logger(self.__class__.__name__).warning( - "The custom operator [{}] will override the qlib default definition".format(operator.__name__) - ) - self._ops[operator.__name__] = operator - - def __getattr__(self, key): - if key not in self._ops: - raise AttributeError("The operator [{0}] is not registered, and all dict is {1}".format(key, self._ops)) - return self._ops[key] - - -Operators = OpsWrapper() OpsList = [ Ref, Max, @@ -1465,4 +1444,37 @@ OpsList = [ If, ] + +class OpsWrapper(object): + """Ops Wrapper""" + + def __init__(self): + self._ops = {} + + def register(self, ops_list): + for operator in ops_list: + if not issubclass(operator, ExpressionOps): + raise TypeError("operator must be subclass of ExpressionOps, not {}".format(operator)) + + if operator.__name__ in self._ops: + get_module_logger(self.__class__.__name__).warning( + "The custom operator [{}] will override the qlib default definition".format(operator.__name__) + ) + self._ops[operator.__name__] = operator + + def __getattr__(self, key): + if key not in self._ops: + raise AttributeError("The operator [{0}] is not registered".format(key)) + return self._ops[key] + + +Operators = OpsWrapper() Operators.register(OpsList) + + +def register_custom_ops(C): + """register custom operator""" + logger = get_module_logger("ops") + if getattr(C, "custom_ops", None) is not None: + Operators.register(C.custom_ops) + logger.debug("register custom operator {}".format(C.custom_ops)) diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 7e029d05e..9e2e92a2b 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -783,10 +783,12 @@ def set_config(config_c, default_conf="client", **kwargs): def config_based_on_c(config_c): + from ..data.ops import register_custom_ops from ..data.data import register_all_wrappers from ..workflow import R, QlibRecorder from ..workflow.utils import experiment_exit_handler + register_custom_ops(config_c) register_all_wrappers(config_c) # set up QlibRecorder exp_manager = init_instance_by_config(config_c["exp_manager"]) diff --git a/tests/test_register_ops.py b/tests/test_register_ops.py new file mode 100644 index 000000000..b147f916f --- /dev/null +++ b/tests/test_register_ops.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import sys +import unittest +import numpy as np + +import qlib +from qlib.data import D +from qlib.data.ops import ElemOperator, PairOperator +from qlib.config import REG_CN +from qlib.utils import exists_qlib_data +from qlib.tests import TestAutoData +from qlib.tests.data import GetData + + +class Diff(ElemOperator): + """Feature First Difference + Parameters + ---------- + feature : Expression + feature instance + Returns + ---------- + Expression + a feature instance with first difference + """ + + def __init__(self, feature): + super(Diff, self).__init__(feature, "diff") + + def _load_internal(self, instrument, start_index, end_index, freq): + series = self.feature.load(instrument, start_index, end_index, freq) + return series.diff() + + def get_extended_window_size(self): + lft_etd, rght_etd = self.feature.get_extended_window_size() + return lft_etd + 1, rght_etd + + +class Distance(PairOperator): + """Feature Distance + Parameters + ---------- + feature : Expression + feature instance + Returns + ---------- + Expression + a feature instance with distance + """ + + def __init__(self, feature_left, feature_right): + super(Distance, self).__init__(feature_left, feature_right, "distance") + + def _load_internal(self, instrument, start_index, end_index, freq): + series_left = self.feature_left.load(instrument, start_index, end_index, freq) + series_right = self.feature_right.load(instrument, start_index, end_index, freq) + return np.abs(series_left - series_right) + + +class TestRegiterCustomOps(TestAutoData): + @classmethod + def setUpClass(cls) -> None: + # use default data + provider_uri = "~/.qlib/qlib_data/cn_data_simple" # target_dir + if not exists_qlib_data(provider_uri): + print(f"Qlib data is not found in {provider_uri}") + + GetData().qlib_data( + name="qlib_data_simple", region="cn", version="latest", interval="1d", target_dir=provider_uri + ) + qlib.init(provider_uri=provider_uri, custom_ops=[Diff, Distance], region=REG_CN) + + def test_regiter_custom_ops(self): + instruments = ["SH600000"] + fields = ["Diff($close)", "Distance($close, Ref($close, 1))"] + print(D.features(instruments, fields, start_time="2010-01-01", end_time="2017-12-31", freq="day")) + + +if __name__ == "__main__": + unittest.main() From c622d3f6f8dba3c75b6a4b3947923e6f27445f21 Mon Sep 17 00:00:00 2001 From: bxdd <45119470+bxdd@users.noreply.github.com> Date: Wed, 20 Jan 2021 18:55:30 +0800 Subject: [PATCH 16/20] Update data.rst --- docs/component/data.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/component/data.rst b/docs/component/data.rst index abb655f9b..530c97d1f 100644 --- a/docs/component/data.rst +++ b/docs/component/data.rst @@ -195,7 +195,7 @@ Feature - `ExpressionOps` `ExpressionOps` will use operator for feature construction. To know more about ``Operator``, please refer to `Operator API <../reference/api.html#module-qlib.data.ops>`_. - Also, ``Qlib`` supports users to define their own custom ``Operator``, an example has been given in ``qlib/examples/workflow_with_custom_ops.py``. + Also, ``Qlib`` supports users to define their own custom ``Operator``, an example has been given in ``qlib/tests/test_register_ops.py``. To know more about ``Feature``, please refer to `Feature API <../reference/api.html#module-qlib.data.base>`_. From a762248d986241f90da1173d525de4f7466690dc Mon Sep 17 00:00:00 2001 From: bxdd Date: Fri, 22 Jan 2021 01:06:32 +0900 Subject: [PATCH 17/20] update test&docs --- docs/component/data.rst | 2 +- tests/test_register_ops.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/component/data.rst b/docs/component/data.rst index 530c97d1f..854ab1c27 100644 --- a/docs/component/data.rst +++ b/docs/component/data.rst @@ -195,7 +195,7 @@ Feature - `ExpressionOps` `ExpressionOps` will use operator for feature construction. To know more about ``Operator``, please refer to `Operator API <../reference/api.html#module-qlib.data.ops>`_. - Also, ``Qlib`` supports users to define their own custom ``Operator``, an example has been given in ``qlib/tests/test_register_ops.py``. + Also, ``Qlib`` supports users to define their own custom ``Operator``, an example has been given in ``tests/test_register_ops.py``. To know more about ``Feature``, please refer to `Feature API <../reference/api.html#module-qlib.data.base>`_. diff --git a/tests/test_register_ops.py b/tests/test_register_ops.py index b147f916f..d497d0b89 100644 --- a/tests/test_register_ops.py +++ b/tests/test_register_ops.py @@ -63,13 +63,11 @@ class TestRegiterCustomOps(TestAutoData): @classmethod def setUpClass(cls) -> None: # use default data - provider_uri = "~/.qlib/qlib_data/cn_data_simple" # target_dir + provider_uri = "~/.qlib/qlib_data/cn_data_simple_1" # target_dir if not exists_qlib_data(provider_uri): print(f"Qlib data is not found in {provider_uri}") - GetData().qlib_data( - name="qlib_data_simple", region="cn", version="latest", interval="1d", target_dir=provider_uri - ) + GetData().qlib_data(name="qlib_data_simple", region="cn", interval="1d", target_dir=provider_uri) qlib.init(provider_uri=provider_uri, custom_ops=[Diff, Distance], region=REG_CN) def test_regiter_custom_ops(self): From 3f9f295a87e1c494d1d04a7988200af1f2be01bf Mon Sep 17 00:00:00 2001 From: zhupr Date: Sat, 23 Jan 2021 16:43:20 +0800 Subject: [PATCH 18/20] add register in config --- qlib/__init__.py | 6 ++-- qlib/config.py | 69 ++++++++++++++++++++++++++++++++++++++++-- qlib/data/data.py | 6 ++-- qlib/utils/__init__.py | 52 ------------------------------- 4 files changed, 71 insertions(+), 62 deletions(-) diff --git a/qlib/__init__.py b/qlib/__init__.py index 9f3bc693a..f82848b90 100644 --- a/qlib/__init__.py +++ b/qlib/__init__.py @@ -11,8 +11,6 @@ import logging import platform import subprocess -from .utils import set_config, config_based_on_c - # init qlib def init(default_conf="client", **kwargs): @@ -25,7 +23,7 @@ def init(default_conf="client", **kwargs): # FIXME: this logger ignored the level in config logger = get_module_logger("Initialization", level=logging.INFO) - set_config(C, default_conf, **kwargs) + C.set(default_conf, **kwargs) # check path if server/local if C.get_uri_type() == C.LOCAL_URI: @@ -44,7 +42,7 @@ def init(default_conf="client", **kwargs): if "flask_server" in C: logger.info(f"flask_server={C['flask_server']}, flask_port={C['flask_port']}") - config_based_on_c(C) + C.register() logger.info("qlib successfully initialized based on %s settings." % default_conf) logger.info(f"data_path={C.get_data_path()}") diff --git a/qlib/config.py b/qlib/config.py index eb68a504d..a65d41041 100644 --- a/qlib/config.py +++ b/qlib/config.py @@ -11,11 +11,12 @@ Two modes are supported """ -import copy -from pathlib import Path -import re import os +import re +import copy +import logging import multiprocessing +from pathlib import Path class Config: @@ -212,6 +213,10 @@ class QlibConfig(Config): LOCAL_URI = "local" NFS_URI = "nfs" + def __init__(self, default_conf): + super().__init__(default_conf) + self._registered = False + def set_mode(self, mode): # raise KeyError self.update(MODE_CONF[mode]) @@ -248,6 +253,64 @@ class QlibConfig(Config): else: raise NotImplementedError(f"This type of uri is not supported") + def set(self, default_conf="client", **kwargs): + from .utils import set_log_with_config, get_module_logger, can_use_cache + + self.reset() + + _logging_config = self.logging_config + if "logging_config" in kwargs: + _logging_config = kwargs["logging_config"] + + # set global config + if _logging_config: + set_log_with_config(_logging_config) + + # FIXME: this logger ignored the level in config + logger = get_module_logger("Initialization", level=logging.INFO) + logger.info(f"default_conf: {default_conf}.") + + self.set_mode(default_conf) + self.set_region(kwargs.get("region", self["region"] if "region" in self else REG_CN)) + + for k, v in kwargs.items(): + if k not in self: + logger.warning("Unrecognized config %s" % k) + self[k] = v + + self.resolve_path() + + if not (self["expression_cache"] is None and self["dataset_cache"] is None): + # check redis + if not can_use_cache(): + logger.warning( + f"redis connection failed(host={self['redis_host']} port={self['redis_port']}), cache will not be used!" + ) + self["expression_cache"] = None + self["dataset_cache"] = None + + def register(self): + from .utils import init_instance_by_config + from .data.ops import register_custom_ops + from .data.data import register_all_wrappers + from .workflow import R, QlibRecorder + from .workflow.utils import experiment_exit_handler + + register_custom_ops(self) + register_all_wrappers(self) + # set up QlibRecorder + exp_manager = init_instance_by_config(self["exp_manager"]) + qr = QlibRecorder(exp_manager) + R.register(qr) + # clean up experiment when python program ends + experiment_exit_handler() + + self._registered = True + + @property + def registered(self): + return self._registered + # global config C = QlibConfig(_default_config) diff --git a/qlib/data/data.py b/qlib/data/data.py index cfb6db1dc..d95728199 100644 --- a/qlib/data/data.py +++ b/qlib/data/data.py @@ -25,7 +25,7 @@ from ..log import get_module_logger from ..utils import parse_field, read_bin, hash_args, normalize_cache_fields from .base import Feature from .cache import DiskDatasetCache, DiskExpressionCache -from ..utils import Wrapper, init_instance_by_config, register_wrapper, get_module_by_module_path, config_based_on_c +from ..utils import Wrapper, init_instance_by_config, register_wrapper, get_module_by_module_path class CalendarProvider(abc.ABC): @@ -486,9 +486,9 @@ class DatasetProvider(abc.ABC): """ # FIXME: Windows OS or MacOS using spawn: https://docs.python.org/3.8/library/multiprocessing.html?highlight=spawn#contexts-and-start-methods # NOTE: This place is compatible with windows, windows multi-process is spawn - if getattr(ExpressionD, "_provider", None) is None: + if not C.registered: C.set_conf_from_C(g_config) - config_based_on_c(g_config) + C.register() obj = dict() for field in column_names: diff --git a/qlib/utils/__init__.py b/qlib/utils/__init__.py index 9e2e92a2b..2d9d11de2 100644 --- a/qlib/utils/__init__.py +++ b/qlib/utils/__init__.py @@ -744,55 +744,3 @@ def load_dataset(path_or_obj): elif extension == ".csv": return pd.read_csv(path_or_obj, parse_dates=True, index_col=[0, 1]) raise ValueError(f"unsupported file type `{extension}`") - - -def set_config(config_c, default_conf="client", **kwargs): - - config_c.reset() - - _logging_config = config_c.logging_config - if "logging_config" in kwargs: - _logging_config = kwargs["logging_config"] - - # set global config - if _logging_config: - set_log_with_config(_logging_config) - - # FIXME: this logger ignored the level in config - logger = get_module_logger("Initialization", level=logging.INFO) - logger.info(f"default_conf: {default_conf}.") - - config_c.set_mode(default_conf) - config_c.set_region(kwargs.get("region", config_c["region"] if "region" in config_c else REG_CN)) - - for k, v in kwargs.items(): - if k not in config_c: - logger.warning("Unrecognized config %s" % k) - config_c[k] = v - - config_c.resolve_path() - - if not (config_c["expression_cache"] is None and config_c["dataset_cache"] is None): - # check redis - if not can_use_cache(): - logger.warning( - f"redis connection failed(host={config_c['redis_host']} port={config_c['redis_port']}), cache will not be used!" - ) - config_c["expression_cache"] = None - config_c["dataset_cache"] = None - - -def config_based_on_c(config_c): - from ..data.ops import register_custom_ops - from ..data.data import register_all_wrappers - from ..workflow import R, QlibRecorder - from ..workflow.utils import experiment_exit_handler - - register_custom_ops(config_c) - register_all_wrappers(config_c) - # set up QlibRecorder - exp_manager = init_instance_by_config(config_c["exp_manager"]) - qr = QlibRecorder(exp_manager) - R.register(qr) - # clean up experiment when python program ends - experiment_exit_handler() From 2016ebbbb2b3cb911b517dc842b7793d5cdad477 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 26 Jan 2021 08:47:07 +0000 Subject: [PATCH 19/20] update tests --- qlib/tests/__init__.py | 5 ++++- tests/test_register_ops.py | 9 ++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/qlib/tests/__init__.py b/qlib/tests/__init__.py index a1b33a2a2..bdc6297da 100644 --- a/qlib/tests/__init__.py +++ b/qlib/tests/__init__.py @@ -7,6 +7,9 @@ from ..config import REG_CN class TestAutoData(unittest.TestCase): + + _setup_kwargs = {} + @classmethod def setUpClass(cls) -> None: # use default data @@ -17,4 +20,4 @@ class TestAutoData(unittest.TestCase): GetData().qlib_data( name="qlib_data_simple", region="cn", version="latest", interval="1d", target_dir=provider_uri ) - init(provider_uri=provider_uri, region=REG_CN) + init(provider_uri=provider_uri, region=REG_CN, **cls._setup_kwargs) diff --git a/tests/test_register_ops.py b/tests/test_register_ops.py index d497d0b89..ec2ec3d93 100644 --- a/tests/test_register_ops.py +++ b/tests/test_register_ops.py @@ -62,13 +62,8 @@ class Distance(PairOperator): class TestRegiterCustomOps(TestAutoData): @classmethod def setUpClass(cls) -> None: - # use default data - provider_uri = "~/.qlib/qlib_data/cn_data_simple_1" # target_dir - if not exists_qlib_data(provider_uri): - print(f"Qlib data is not found in {provider_uri}") - - GetData().qlib_data(name="qlib_data_simple", region="cn", interval="1d", target_dir=provider_uri) - qlib.init(provider_uri=provider_uri, custom_ops=[Diff, Distance], region=REG_CN) + cls._setup_kwargs.update({"custom_ops":[Diff, Distance]}) + super().setUpClass() def test_regiter_custom_ops(self): instruments = ["SH600000"] From ee94634b2393930043e0021c02eff388767e29c2 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 26 Jan 2021 08:47:53 +0000 Subject: [PATCH 20/20] black --- tests/test_register_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_register_ops.py b/tests/test_register_ops.py index ec2ec3d93..cb172b2bb 100644 --- a/tests/test_register_ops.py +++ b/tests/test_register_ops.py @@ -62,7 +62,7 @@ class Distance(PairOperator): class TestRegiterCustomOps(TestAutoData): @classmethod def setUpClass(cls) -> None: - cls._setup_kwargs.update({"custom_ops":[Diff, Distance]}) + cls._setup_kwargs.update({"custom_ops": [Diff, Distance]}) super().setUpClass() def test_regiter_custom_ops(self):