liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-09 22:10:56 +08:00
Code Activity

Reindex files.

Browse Source
This commit is contained in:
Charles Young
2021-03-04 22:30:38 +08:00
parent 2bff6eb781
commit 83c6e74783
12 changed files with 793 additions and 1038 deletions
Download Patch File Download Diff File Expand all files Collapse all files

282
tests/test_enhanced_indexing.py
View File

@@ -1,282 +0,0 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
import sys
import math
import shutil
import unittest
import numpy as np
import pandas as pd
from tqdm import tqdm
from pathlib import Path
import qlib
from qlib.config import C
from qlib.utils import init_instance_by_config, flatten_dict
from qlib.workflow import R
from qlib.config import REG_CN
from qlib.workflow.record_temp import SignalRecord, SigAnaRecord
from qlib.tests import TestAutoData
from qlib.portfolio.optimizer import EnhancedIndexingOptimizer
from qlib.model.riskmodel import StructuredCovEstimator
from qlib.data.dataset.loader import QlibDataLoader
from qlib.data.dataset.handler import DataHandler
from qlib.data import D
from qlib.utils import exists_qlib_data, init_instance_by_config
market = "all"
trade_gap = 21
label_config = "Ref($close, -{}) / Ref($close, -1) - 1".format(trade_gap) # reconstruct portfolio once a month
provider_uri = "~/.qlib_ei/qlib_data/cn_data" # target_dir
if not exists_qlib_data(provider_uri):
print(f"Qlib data is not found in {provider_uri}")
sys.path.append(str(Path.cwd().parent.joinpath("scripts")))
from get_data import GetData
GetData().qlib_data(target_dir=provider_uri, region=REG_CN)
qlib.init(provider_uri=provider_uri, region=REG_CN)
###################################
# train model
###################################
data_handler_config = {
"start_time": "2008-01-01",
"end_time": "2020-08-01",
"fit_start_time": "2008-01-01",
"fit_end_time": "2014-11-30",
"instruments": market,
"label": [label_config]
}
task = {
"model": {
"class": "LGBModel",
"module_path": "qlib.contrib.model.gbdt",
"kwargs": {
"loss": "mse",
"colsample_bytree": 0.8879,
"learning_rate": 0.0421,
"subsample": 0.8789,
"lambda_l1": 205.6999,
"lambda_l2": 580.9768,
"max_depth": 8,
"num_leaves": 210,
"num_threads": 32,
},
},
"dataset": {
"class": "DatasetH",
"module_path": "qlib.data.dataset",
"kwargs": {
"handler": {
"class": "Alpha158",
"module_path": "qlib.contrib.data.handler",
"kwargs": data_handler_config,
},
"segments": {
"train": ("2008-01-01", "2014-11-30"),
"valid": ("2015-01-01", "2016-11-30"),
"test": ("2017-01-01", "2018-01-01"),
},
},
},
}
class CSI300:
"""Simulate CSI300 as the Benchmark for Enhanced Indexing to Track"""
def __init__(self):
# provider_uri = '/nfs_data/qlib_data/ycz_daily/qlib'
# qlib.init(provider_uri=provider_uri, region=REG_CN, dataset_cache=None, expression_cache=None)
self.csi_weight = D.features(D.instruments('csi300'), ['$csi300_weight'])
def __call__(self, pd_index, trade_date):
weights = np.zeros(len(pd_index))
for idx, instrument in enumerate(pd_index):
if (instrument, trade_date) in self.csi_weight.index:
weight = self.csi_weight.loc[(instrument, trade_date)].values[0]
if not math.isnan(weight):
weights[idx] = weight
assert weights.sum() > 0, ' Fetch CSI Weights Error!'
weights = weights / weights.sum()
return weights
class EnhancedIndexingStrategy:
"""Enhanced Indexing Strategy"""
def __init__(self):
self.benchmark = CSI300()
provider_uri = "~/.qlib_ei/qlib_data/cn_data"
qlib.init(provider_uri=provider_uri, region=REG_CN)
self.data_handler = DataHandler(market, "2015-01-01", "2019-01-01", QlibDataLoader(["$close"]))
self.label_handler = DataHandler(market, "2015-01-01", "2019-01-01", QlibDataLoader([label_config]))
self.cov_estimator = StructuredCovEstimator()
self.optimizer = EnhancedIndexingOptimizer(lamb=0.1, delta=0.4, bench_dev=0.03, max_iters=50000)
def update(self, score_series, current, pred_date):
"""
Parameters
-----------
score_series : pd.Series
stock_id , score.
current : Position()
current of account.
trade_exchange : Exchange()
exchange.
trade_date : pd.Timestamp
date.
"""
print(score_series)
score_series = score_series.dropna()
# portfolio init weight
init_weight = current.reindex(score_series.index, fill_value=0).values.squeeze()
init_weight_sum = init_weight.sum()
if init_weight_sum > 0:
init_weight /= init_weight_sum
# covariance estimation
selector = (self.data_handler.get_range_selector(pred_date, 252), score_series.index)
price = self.data_handler.fetch(selector, level=None, squeeze=True)
F, cov_b, var_u = self.cov_estimator.predict(price, return_decomposed_components=True)
# optimize target portfolio
w_bench = self.benchmark(score_series.index, pred_date)
passed_init_weight = init_weight if init_weight_sum > 0 else None
# print(F)
# print(cov_b)
# print(var_u)
# print(passed_init_weight)
# print(w_bench)
target_weight = self.optimizer(score_series.values, F, cov_b, var_u, passed_init_weight, w_bench)
# print(target_weight)
target = pd.DataFrame(data=target_weight, index=score_series.index)
active_weights = target_weight - w_bench
selector = (self.label_handler.get_range_selector(pred_date, 1), score_series.index)
label = self.label_handler.fetch(selector, level=None, squeeze=True)
alpha = 0
for instrument, weight in zip(score_series.index, active_weights):
delta = label.loc[(pred_date, instrument)]
alpha += weight * (0 if math.isnan(delta) else delta)
print(alpha)
return alpha, target
def train():
"""train model
Returns
-------
pred_score: pandas.DataFrame
predict scores
performance: dict
model performance
"""
# model initiation
model = init_instance_by_config(task["model"])
dataset = init_instance_by_config(task["dataset"])
# start exp
with R.start(experiment_name="workflow"):
R.log_params(**flatten_dict(task))
model.fit(dataset)
# prediction
recorder = R.get_recorder()
rid = recorder.id
sr = SignalRecord(model, dataset, recorder)
sr.generate()
pred_score = sr.load()
# calculate ic and ric
sar = SigAnaRecord(recorder)
sar.generate()
ic = sar.load(sar.get_path("ic.pkl"))
ric = sar.load(sar.get_path("ric.pkl"))
return pred_score, {"ic": ic, "ric": ric}, rid
def backtest_analysis(scores):
"""backtest enhanced indexing
Parameters
----------
scores: pandas.DataFrame
predict scores
Returns
-------
sharpe_ratio: floating-point
sharpe ratio of the enhanced indexing portfolio
"""
# backtest and analysis
with R.start(experiment_name="backtest_analysis"):
strategy = EnhancedIndexingStrategy()
dates = scores.index.get_level_values(0).unique()
alphas = []
current = pd.DataFrame()
gap_between_next_trade = 0
for date in tqdm(dates):
if gap_between_next_trade == 0:
score_series = scores.loc[date]
alpha, current = strategy.update(score_series, current, date)
alphas.append(alpha)
gap_between_next_trade = trade_gap
else:
gap_between_next_trade -= 1
alphas = np.array(alphas)
sharpe_ratio = alphas.mean() / np.std(alphas)
print('Sharpe:', sharpe_ratio)
return sharpe_ratio
class TestAllFlow(TestAutoData):
PRED_SCORE = None
REPORT_NORMAL = None
POSITIONS = None
RID = None
@classmethod
def tearDownClass(cls) -> None:
shutil.rmtree(str(Path(C["exp_manager"]["kwargs"]["uri"].strip("file:")).resolve()))
def test_0_train(self):
TestAllFlow.PRED_SCORE, ic_ric, TestAllFlow.RID = train()
self.assertGreaterEqual(ic_ric["ic"].all(), 0, "train failed")
self.assertGreaterEqual(ic_ric["ric"].all(), 0, "train failed")
def test_1_backtest(self):
sharpe_ratio = backtest_analysis(TestAllFlow.PRED_SCORE)
self.assertGreaterEqual(
sharpe_ratio,
0.90,
"backtest failed",
)
def suite():
_suite = unittest.TestSuite()
_suite.addTest(TestAllFlow("test_0_train"))
_suite.addTest(TestAllFlow("test_1_backtest"))
return _suite
if __name__ == "__main__":
runner = unittest.TextTestRunner()
runner.run(suite())