liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-15 08:46:56 +08:00
Code Activity

download orderbook data (#1754)

Browse Source 
* download orderbook data

* fix CI error

* fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* test fix CI error

* optimize get_data code

* optimize get_data code

* optimize get_data code

* optimize README

---------

Co-authored-by: Linlang <v-linlanglv@microsoft.com>
This commit is contained in:
Linlang
2024-03-07 14:41:21 +08:00
committed by GitHub
parent 98f569eed2
commit 39f88daaa7
14 changed files with 30 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
examples/benchmarks/TRA/src/model.py
View File

@@ -324,7 +324,6 @@ class TRAModel(Model):
class LSTM(nn.Module): class LSTM(nn.Module):
"""LSTM Model """LSTM Model
Args: Args:
@@ -414,7 +413,6 @@ class PositionalEncoding(nn.Module):
class Transformer(nn.Module): class Transformer(nn.Module):
"""Transformer Model """Transformer Model
Args: Args:
@@ -475,7 +473,6 @@ class Transformer(nn.Module):
class TRA(nn.Module): class TRA(nn.Module):
"""Temporal Routing Adaptor (TRA) """Temporal Routing Adaptor (TRA)
TRA takes historical prediction errors & latent representation as inputs, TRA takes historical prediction errors & latent representation as inputs,

5
examples/orderbook_data/README.md
View File

@@ -27,13 +27,11 @@ pip install arctic # NOTE: pip may fail to resolve the right package dependency
2. Please follow following steps to download example data 2. Please follow following steps to download example data
```bash ```bash
cd examples/orderbook_data/ cd examples/orderbook_data/
wget http://fintech.msra.cn/stock_data/downloads/highfreq_orderboook_example_data.tar.bz2 python ../../scripts/get_data.py download_data --target_dir . --file_name highfreq_orderbook_example_data.zip
tar xf highfreq_orderboook_example_data.tar.bz2
``` ```
3. Please import the example data to your mongo db 3. Please import the example data to your mongo db
```bash ```bash
cd examples/orderbook_data/
python create_dataset.py initialize_library # Initialization Libraries python create_dataset.py initialize_library # Initialization Libraries
python create_dataset.py import_data # Initialization Libraries python create_dataset.py import_data # Initialization Libraries
``` ```
@@ -42,7 +40,6 @@ python create_dataset.py import_data # Initialization Libraries
After importing these data, you run `example.py` to create some high-frequency features. After importing these data, you run `example.py` to create some high-frequency features.
```bash ```bash
cd examples/orderbook_data/
pytest -s --disable-warnings example.py # If you want run all examples pytest -s --disable-warnings example.py # If you want run all examples
pytest -s --disable-warnings example.py::TestClass::test_exp_10 # If you want to run specific example pytest -s --disable-warnings example.py::TestClass::test_exp_10 # If you want to run specific example
``` ```

16
qlib/backtest/__init__.py
View File

@@ -162,13 +162,15 @@ def create_account_instance(
init_cash=init_cash, init_cash=init_cash,
position_dict=position_dict, position_dict=position_dict,
pos_type=pos_type, pos_type=pos_type,
benchmark_config={} benchmark_config=(
if benchmark is None {}
else { if benchmark is None
"benchmark": benchmark, else {
"start_time": start_time, "benchmark": benchmark,
"end_time": end_time, "start_time": start_time,
}, "end_time": end_time,
}
),
) )

8
qlib/backtest/report.py
View File

@@ -622,9 +622,11 @@ class Indicator:
print( print(
"[Indicator({}) {}]: FFR: {}, PA: {}, POS: {}".format( "[Indicator({}) {}]: FFR: {}, PA: {}, POS: {}".format(
freq, freq,
trade_start_time (
if isinstance(trade_start_time, str) trade_start_time
else trade_start_time.strftime("%Y-%m-%d %H:%M:%S"), if isinstance(trade_start_time, str)
else trade_start_time.strftime("%Y-%m-%d %H:%M:%S")
),
fulfill_rate, fulfill_rate,
price_advantage, price_advantage,
positive_rate, positive_rate,

1
qlib/contrib/eva/alpha.py
View File

@@ -3,6 +3,7 @@ Here is a batch of evaluation functions.
The interface should be redesigned carefully in the future. The interface should be redesigned carefully in the future.
""" """
import pandas as pd import pandas as pd
from typing import Tuple from typing import Tuple
from qlib import get_module_logger from qlib import get_module_logger

3
qlib/contrib/model/pytorch_tra.py
View File

@@ -511,7 +511,6 @@ class TRAModel(Model):
class RNN(nn.Module): class RNN(nn.Module):
"""RNN Model """RNN Model
Args: Args:
@@ -601,7 +600,6 @@ class PositionalEncoding(nn.Module):
class Transformer(nn.Module): class Transformer(nn.Module):
"""Transformer Model """Transformer Model
Args: Args:
@@ -649,7 +647,6 @@ class Transformer(nn.Module):
class TRA(nn.Module): class TRA(nn.Module):
"""Temporal Routing Adaptor (TRA) """Temporal Routing Adaptor (TRA)
TRA takes historical prediction errors & latent representation as inputs, TRA takes historical prediction errors & latent representation as inputs,

1
qlib/contrib/strategy/signal_strategy.py
View File

@@ -373,7 +373,6 @@ class WeightStrategyBase(BaseSignalStrategy):
class EnhancedIndexingStrategy(WeightStrategyBase): class EnhancedIndexingStrategy(WeightStrategyBase):
"""Enhanced Indexing Strategy """Enhanced Indexing Strategy
Enhanced indexing combines the arts of active management and passive management, Enhanced indexing combines the arts of active management and passive management,

2
qlib/model/ens/ensemble.py
View File

@@ -30,7 +30,6 @@ class Ensemble:
class SingleKeyEnsemble(Ensemble): class SingleKeyEnsemble(Ensemble):
""" """
Extract the object if there is only one key and value in the dict. Make the result more readable. Extract the object if there is only one key and value in the dict. Make the result more readable.
{Only key: Only value} -> Only value {Only key: Only value} -> Only value
@@ -64,7 +63,6 @@ class SingleKeyEnsemble(Ensemble):
class RollingEnsemble(Ensemble): class RollingEnsemble(Ensemble):
"""Merge a dict of rolling dataframe like `prediction` or `IC` into an ensemble. """Merge a dict of rolling dataframe like `prediction` or `IC` into an ensemble.
NOTE: The values of dict must be pd.DataFrame, and have the index "datetime". NOTE: The values of dict must be pd.DataFrame, and have the index "datetime".

4
qlib/model/riskmodel/shrink.py
View File

@@ -247,9 +247,7 @@ class ShrinkCovEstimator(RiskModel):
v1 = y.T.dot(z) / t - cov_mkt[:, None] * S v1 = y.T.dot(z) / t - cov_mkt[:, None] * S
roff1 = np.sum(v1 * cov_mkt[:, None].T) / var_mkt - np.sum(np.diag(v1) * cov_mkt) / var_mkt roff1 = np.sum(v1 * cov_mkt[:, None].T) / var_mkt - np.sum(np.diag(v1) * cov_mkt) / var_mkt
v3 = z.T.dot(z) / t - var_mkt * S v3 = z.T.dot(z) / t - var_mkt * S
roff3 = ( roff3 = np.sum(v3 * np.outer(cov_mkt, cov_mkt)) / var_mkt**2 - np.sum(np.diag(v3) * cov_mkt**2) / var_mkt**2
np.sum(v3 * np.outer(cov_mkt, cov_mkt)) / var_mkt**2 - np.sum(np.diag(v3) * cov_mkt**2) / var_mkt**2
)
roff = 2 * roff1 - roff3 roff = 2 * roff1 - roff3
rho = rdiag + roff rho = rdiag + roff

1
qlib/workflow/online/strategy.py
View File

@@ -90,7 +90,6 @@ class OnlineStrategy:
class RollingStrategy(OnlineStrategy): class RollingStrategy(OnlineStrategy):
""" """
This example strategy always uses the latest rolling model sas online models. This example strategy always uses the latest rolling model sas online models.
""" """

4
scripts/dump_bin.py
View File

@@ -146,9 +146,7 @@ class DumpDataBase:
return ( return (
self._include_fields self._include_fields
if self._include_fields if self._include_fields
else set(df_columns) - set(self._exclude_fields) else set(df_columns) - set(self._exclude_fields) if self._exclude_fields else df_columns
if self._exclude_fields
else df_columns
) )
@staticmethod @staticmethod

8
scripts/dump_pit.py
View File

@@ -132,9 +132,11 @@ class DumpPitData:
return ( return (
set(self._include_fields) set(self._include_fields)
if self._include_fields if self._include_fields
else set(df[self.field_column_name]) - set(self._exclude_fields) else (
if self._exclude_fields set(df[self.field_column_name]) - set(self._exclude_fields)
else set(df[self.field_column_name]) if self._exclude_fields
else set(df[self.field_column_name])
)
) )
def get_filenames(self, symbol, field, interval): def get_filenames(self, symbol, field, interval):

2
setup.py
View File

@@ -65,6 +65,8 @@ REQUIRED = [
# To ensure stable operation of the experiment manager, we have limited the version of mlflow, # To ensure stable operation of the experiment manager, we have limited the version of mlflow,
# and we need to verify whether version 2.0 of mlflow can serve qlib properly. # and we need to verify whether version 2.0 of mlflow can serve qlib properly.
"mlflow>=1.12.1, <=1.30.0", "mlflow>=1.12.1, <=1.30.0",
# mlflow 1.30.0 requires packaging<22, so we limit the packaging version, otherwise the CI will fail.
"packaging<22",
"tqdm", "tqdm",
"loguru", "loguru",
"lightgbm>=3.3.0", "lightgbm>=3.3.0",

6
tests/test_workflow.py
View File

@@ -9,7 +9,9 @@ from qlib.tests import TestAutoData
class WorkflowTest(TestAutoData): class WorkflowTest(TestAutoData):
TMP_PATH = Path("./.mlruns_tmp/") # Creating the directory manually doesn't work with mlflow,
# so we add a subfolder named .trash when we create the directory.
TMP_PATH = Path("./.mlruns_tmp/.trash")
def tearDown(self) -> None: def tearDown(self) -> None:
if self.TMP_PATH.exists(): if self.TMP_PATH.exists():
@@ -17,6 +19,8 @@ class WorkflowTest(TestAutoData):
def test_get_local_dir(self): def test_get_local_dir(self):
""" """ """ """
self.TMP_PATH.mkdir(parents=True, exist_ok=True)
with R.start(uri=str(self.TMP_PATH)): with R.start(uri=str(self.TMP_PATH)):
pass pass