liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-13 15:56:57 +08:00
Code Activity

fix get data error

Browse Source
This commit is contained in:
Linlang
2024-05-23 06:57:59 +08:00
parent 63021018d6
commit 117f67d6e1
2 changed files with 8 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
examples/orderbook_data/README.md
View File

@@ -16,7 +16,7 @@ Current version of script with default value tries to connect localhost **via de
Run following command to install necessary libraries Run following command to install necessary libraries
``` ```
pip install pytest coverage pip install pytest coverage gdown
pip install arctic # NOTE: pip may fail to resolve the right package dependency !!! Please make sure the dependency are satisfied. pip install arctic # NOTE: pip may fail to resolve the right package dependency !!! Please make sure the dependency are satisfied.
``` ```
@@ -27,7 +27,8 @@ pip install arctic # NOTE: pip may fail to resolve the right package dependency
2. Please follow following steps to download example data 2. Please follow following steps to download example data
```bash ```bash
cd examples/orderbook_data/ cd examples/orderbook_data/
python ../../scripts/get_data.py download_data --target_dir . --file_name highfreq_orderbook_example_data.zip gdown https://drive.google.com/uc?id=15nZF7tFT_eKVZAcMFL1qPS4jGyJflH7e # Proxies may be necessary here.
python ../../scripts/get_data.py _unzip --file_path highfreq_orderbook_example_data.zip --target_dir .
``` ```
3. Please import the example data to your mongo db 3. Please import the example data to your mongo db

14
qlib/tests/data.py
View File

@@ -12,15 +12,11 @@ import datetime
from tqdm import tqdm from tqdm import tqdm
from pathlib import Path from pathlib import Path
from loguru import logger from loguru import logger
from cryptography.fernet import Fernet
from qlib.utils import exists_qlib_data from qlib.utils import exists_qlib_data
class GetData: class GetData:
REMOTE_URL = "https://qlibpublic.blob.core.windows.net/data/default/stock_data" REMOTE_URL = "https://github.com/SunsetWolf/qlib_dataset/releases/download"
# "?" is not included in the token.
TOKEN = b"gAAAAABkmDhojHc0VSCDdNK1MqmRzNLeDFXe5hy8obHpa6SDQh4de6nW5gtzuD-fa6O_WZb0yyqYOL7ndOfJX_751W3xN5YB4-n-P22jK-t6ucoZqhT70KPD0Lf0_P328QPJVZ1gDnjIdjhi2YLOcP4BFTHLNYO0mvzszR8TKm9iT5AKRvuysWnpi8bbYwGU9zAcJK3x9EPL43hOGtxliFHcPNGMBoJW4g_ercdhi0-Qgv5_JLsV-29_MV-_AhuaYvJuN2dEywBy"
KEY = "EYcA8cgorA8X9OhyMwVfuFxn_1W3jGk6jCbs3L2oPoA="
def __init__(self, delete_zip_file=False): def __init__(self, delete_zip_file=False):
""" """
@@ -33,9 +29,7 @@ class GetData:
self.delete_zip_file = delete_zip_file self.delete_zip_file = delete_zip_file
def merge_remote_url(self, file_name: str): def merge_remote_url(self, file_name: str):
fernet = Fernet(self.KEY) return f"{self.REMOTE_URL}/{file_name}"
token = fernet.decrypt(self.TOKEN).decode()
return f"{self.REMOTE_URL}/{file_name}?{token}"
def download_data(self, file_name: str, target_dir: [Path, str], delete_old: bool = True): def download_data(self, file_name: str, target_dir: [Path, str], delete_old: bool = True):
""" """
@@ -99,7 +93,9 @@ class GetData:
return status return status
@staticmethod @staticmethod
def _unzip(file_path: Path, target_dir: Path, delete_old: bool = True): def _unzip(file_path: [Path, str], target_dir: [Path, str], delete_old: bool = True):
file_path = Path(file_path)
target_dir = Path(target_dir)
if delete_old: if delete_old:
logger.warning( logger.warning(
f"will delete the old qlib data directory(features, instruments, calendars, features_cache, dataset_cache): {target_dir}" f"will delete the old qlib data directory(features, instruments, calendars, features_cache, dataset_cache): {target_dir}"