From ef11a9d95c06965680b0e886b6c6657687895c69 Mon Sep 17 00:00:00 2001 From: zhupr Date: Fri, 28 May 2021 14:57:06 +0800 Subject: [PATCH] modify the default value of exists_skip in the GetData.qlib_data parameter to False --- examples/highfreq/workflow.py | 2 +- examples/hyperparameter/LightGBM/hyperparameter_158.py | 2 +- examples/hyperparameter/LightGBM/hyperparameter_360.py | 2 +- examples/model_interpreter/feature.py | 2 +- examples/rolling_process_data/workflow.py | 2 +- examples/run_all_model.py | 2 +- examples/workflow_by_code.py | 2 +- qlib/tests/__init__.py | 1 + qlib/tests/data.py | 8 ++++++-- tests/test_get_data.py | 4 +++- 10 files changed, 17 insertions(+), 10 deletions(-) diff --git a/examples/highfreq/workflow.py b/examples/highfreq/workflow.py index 856885b25..7bf5fd09a 100644 --- a/examples/highfreq/workflow.py +++ b/examples/highfreq/workflow.py @@ -85,7 +85,7 @@ class HighfreqWorkflow: # use yahoo_cn_1min data QLIB_INIT_CONFIG = {**HIGH_FREQ_CONFIG, **self.SPEC_CONF} provider_uri = QLIB_INIT_CONFIG.get("provider_uri") - GetData().qlib_data(target_dir=provider_uri, interval="1min", region=REG_CN) + GetData().qlib_data(target_dir=provider_uri, interval="1min", region=REG_CN, exists_skip=True) qlib.init(**QLIB_INIT_CONFIG) def _prepare_calender_cache(self): diff --git a/examples/hyperparameter/LightGBM/hyperparameter_158.py b/examples/hyperparameter/LightGBM/hyperparameter_158.py index 9e4557ed5..89cc10cc6 100644 --- a/examples/hyperparameter/LightGBM/hyperparameter_158.py +++ b/examples/hyperparameter/LightGBM/hyperparameter_158.py @@ -37,7 +37,7 @@ def objective(trial): if __name__ == "__main__": provider_uri = "~/.qlib/qlib_data/cn_data" - GetData().qlib_data(target_dir=provider_uri, region=REG_CN) + GetData().qlib_data(target_dir=provider_uri, region=REG_CN, exists_skip=True) qlib.init(provider_uri=provider_uri, region="cn") dataset = init_instance_by_config(CSI300_DATASET_CONFIG) diff --git a/examples/hyperparameter/LightGBM/hyperparameter_360.py b/examples/hyperparameter/LightGBM/hyperparameter_360.py index a8127014b..bc0cc245d 100644 --- a/examples/hyperparameter/LightGBM/hyperparameter_360.py +++ b/examples/hyperparameter/LightGBM/hyperparameter_360.py @@ -40,7 +40,7 @@ def objective(trial): if __name__ == "__main__": provider_uri = "~/.qlib/qlib_data/cn_data" - GetData().qlib_data(target_dir=provider_uri, region=REG_CN) + GetData().qlib_data(target_dir=provider_uri, region=REG_CN, exists_skip=True) qlib.init(provider_uri=provider_uri, region=REG_CN) dataset = init_instance_by_config(DATASET_CONFIG) diff --git a/examples/model_interpreter/feature.py b/examples/model_interpreter/feature.py index 1c29fda6e..a1288e07d 100644 --- a/examples/model_interpreter/feature.py +++ b/examples/model_interpreter/feature.py @@ -14,7 +14,7 @@ if __name__ == "__main__": # use default data provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir - GetData().qlib_data(target_dir=provider_uri, region=REG_CN) + GetData().qlib_data(target_dir=provider_uri, region=REG_CN, exists_skip=True) qlib.init(provider_uri=provider_uri, region=REG_CN) diff --git a/examples/rolling_process_data/workflow.py b/examples/rolling_process_data/workflow.py index bfa2d1ec4..387d5cde7 100644 --- a/examples/rolling_process_data/workflow.py +++ b/examples/rolling_process_data/workflow.py @@ -23,7 +23,7 @@ class RollingDataWorkflow: """initialize qlib""" # use yahoo_cn_1min data provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir - GetData().qlib_data(target_dir=provider_uri, region=REG_CN) + GetData().qlib_data(target_dir=provider_uri, region=REG_CN, exists_skip=True) qlib.init(provider_uri=provider_uri, region=REG_CN) def _dump_pre_handler(self, path): diff --git a/examples/run_all_model.py b/examples/run_all_model.py index 8875b9aa1..c79fee004 100644 --- a/examples/run_all_model.py +++ b/examples/run_all_model.py @@ -37,7 +37,7 @@ exp_manager = { }, } -GetData().qlib_data(target_dir=provider_uri, region=REG_CN) +GetData().qlib_data(target_dir=provider_uri, region=REG_CN, exists_skip=True) qlib.init(provider_uri=provider_uri, region=REG_CN, exp_manager=exp_manager) # decorator to check the arguments diff --git a/examples/workflow_by_code.py b/examples/workflow_by_code.py index 2e84cadc2..1cdf2ac80 100644 --- a/examples/workflow_by_code.py +++ b/examples/workflow_by_code.py @@ -14,7 +14,7 @@ if __name__ == "__main__": # use default data provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir - GetData().qlib_data(target_dir=provider_uri, region=REG_CN) + GetData().qlib_data(target_dir=provider_uri, region=REG_CN, exists_skip=True) qlib.init(provider_uri=provider_uri, region=REG_CN) port_analysis_config = { diff --git a/qlib/tests/__init__.py b/qlib/tests/__init__.py index e72f000ba..7f43cd99a 100644 --- a/qlib/tests/__init__.py +++ b/qlib/tests/__init__.py @@ -19,5 +19,6 @@ class TestAutoData(unittest.TestCase): interval="1d", target_dir=cls.provider_uri, delete_old=False, + exists_skip=True, ) init(provider_uri=cls.provider_uri, region=REG_CN, **cls._setup_kwargs) diff --git a/qlib/tests/data.py b/qlib/tests/data.py index 0f226c6b1..2bfe43590 100644 --- a/qlib/tests/data.py +++ b/qlib/tests/data.py @@ -113,7 +113,7 @@ class GetData: interval="1d", region="cn", delete_old=True, - exists_skip=True, + exists_skip=False, ): """download cn qlib data from remote @@ -132,7 +132,7 @@ class GetData: delete_old: bool delete an existing directory, by default True exists_skip: bool - exists skip, by default True + exists skip, by default False Examples --------- @@ -145,6 +145,10 @@ class GetData: """ if exists_skip and exists_qlib_data(target_dir): + logger.warning( + f"Data already exists: {target_dir}, the data download will be skipped\n" + f"\tIf downloading is required: `exists_skip=False` or `change target_dir`" + ) return qlib_version = ".".join(re.findall(r"(\d+)\.+", qlib.__version__)) diff --git a/tests/test_get_data.py b/tests/test_get_data.py index 55a2c3318..93a852f55 100644 --- a/tests/test_get_data.py +++ b/tests/test_get_data.py @@ -34,7 +34,9 @@ class TestGetData(unittest.TestCase): def test_0_qlib_data(self): - GetData().qlib_data(name="qlib_data_simple", target_dir=QLIB_DIR, region="cn", interval="1d", delete_old=False) + GetData().qlib_data( + name="qlib_data_simple", target_dir=QLIB_DIR, region="cn", interval="1d", delete_old=False, exists_skip=True + ) df = D.features(D.instruments("csi300"), self.FIELDS) self.assertListEqual(list(df.columns), self.FIELDS, "get qlib data failed") self.assertFalse(df.dropna().empty, "get qlib data failed")