mirror of
https://github.com/microsoft/qlib.git
synced 2026-07-07 04:50:56 +08:00
283 lines
8.7 KiB
Python
283 lines
8.7 KiB
Python
# Copyright (c) Microsoft Corporation.
|
|
# Licensed under the MIT License.
|
|
|
|
import sys
|
|
import math
|
|
import shutil
|
|
import unittest
|
|
import numpy as np
|
|
import pandas as pd
|
|
from tqdm import tqdm
|
|
from pathlib import Path
|
|
|
|
import qlib
|
|
from qlib.config import C
|
|
from qlib.utils import init_instance_by_config, flatten_dict
|
|
from qlib.workflow import R
|
|
from qlib.config import REG_CN
|
|
from qlib.workflow.record_temp import SignalRecord, SigAnaRecord
|
|
from qlib.tests import TestAutoData
|
|
from qlib.portfolio.optimizer import EnhancedIndexingOptimizer
|
|
from qlib.model.riskmodel import StructuredCovEstimator
|
|
from qlib.data.dataset.loader import QlibDataLoader
|
|
from qlib.data.dataset.handler import DataHandler
|
|
from qlib.data import D
|
|
from qlib.utils import exists_qlib_data, init_instance_by_config
|
|
|
|
market = "all"
|
|
trade_gap = 21
|
|
label_config = "Ref($close, -{}) / Ref($close, -1) - 1".format(trade_gap) # reconstruct portfolio once a month
|
|
|
|
provider_uri = "~/.qlib_ei/qlib_data/cn_data" # target_dir
|
|
if not exists_qlib_data(provider_uri):
|
|
print(f"Qlib data is not found in {provider_uri}")
|
|
sys.path.append(str(Path.cwd().parent.joinpath("scripts")))
|
|
from get_data import GetData
|
|
GetData().qlib_data(target_dir=provider_uri, region=REG_CN)
|
|
qlib.init(provider_uri=provider_uri, region=REG_CN)
|
|
|
|
###################################
|
|
# train model
|
|
###################################
|
|
data_handler_config = {
|
|
"start_time": "2008-01-01",
|
|
"end_time": "2020-08-01",
|
|
"fit_start_time": "2008-01-01",
|
|
"fit_end_time": "2014-11-30",
|
|
"instruments": market,
|
|
"label": [label_config]
|
|
}
|
|
|
|
task = {
|
|
"model": {
|
|
"class": "LGBModel",
|
|
"module_path": "qlib.contrib.model.gbdt",
|
|
"kwargs": {
|
|
"loss": "mse",
|
|
"colsample_bytree": 0.8879,
|
|
"learning_rate": 0.0421,
|
|
"subsample": 0.8789,
|
|
"lambda_l1": 205.6999,
|
|
"lambda_l2": 580.9768,
|
|
"max_depth": 8,
|
|
"num_leaves": 210,
|
|
"num_threads": 32,
|
|
},
|
|
},
|
|
"dataset": {
|
|
"class": "DatasetH",
|
|
"module_path": "qlib.data.dataset",
|
|
"kwargs": {
|
|
"handler": {
|
|
"class": "Alpha158",
|
|
"module_path": "qlib.contrib.data.handler",
|
|
"kwargs": data_handler_config,
|
|
},
|
|
"segments": {
|
|
"train": ("2008-01-01", "2014-11-30"),
|
|
"valid": ("2015-01-01", "2016-11-30"),
|
|
"test": ("2017-01-01", "2018-01-01"),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
class CSI300:
|
|
"""Simulate CSI300 as the Benchmark for Enhanced Indexing to Track"""
|
|
|
|
def __init__(self):
|
|
# provider_uri = '/nfs_data/qlib_data/ycz_daily/qlib'
|
|
# qlib.init(provider_uri=provider_uri, region=REG_CN, dataset_cache=None, expression_cache=None)
|
|
self.csi_weight = D.features(D.instruments('csi300'), ['$csi300_weight'])
|
|
|
|
def __call__(self, pd_index, trade_date):
|
|
weights = np.zeros(len(pd_index))
|
|
|
|
for idx, instrument in enumerate(pd_index):
|
|
if (instrument, trade_date) in self.csi_weight.index:
|
|
weight = self.csi_weight.loc[(instrument, trade_date)].values[0]
|
|
if not math.isnan(weight):
|
|
weights[idx] = weight
|
|
|
|
assert weights.sum() > 0, ' Fetch CSI Weights Error!'
|
|
weights = weights / weights.sum()
|
|
|
|
return weights
|
|
|
|
|
|
class EnhancedIndexingStrategy:
|
|
"""Enhanced Indexing Strategy"""
|
|
|
|
def __init__(self):
|
|
self.benchmark = CSI300()
|
|
|
|
provider_uri = "~/.qlib_ei/qlib_data/cn_data"
|
|
qlib.init(provider_uri=provider_uri, region=REG_CN)
|
|
|
|
self.data_handler = DataHandler(market, "2015-01-01", "2019-01-01", QlibDataLoader(["$close"]))
|
|
self.label_handler = DataHandler(market, "2015-01-01", "2019-01-01", QlibDataLoader([label_config]))
|
|
self.cov_estimator = StructuredCovEstimator()
|
|
self.optimizer = EnhancedIndexingOptimizer(lamb=0.1, delta=0.4, bench_dev=0.03, max_iters=50000)
|
|
|
|
def update(self, score_series, current, pred_date):
|
|
"""
|
|
Parameters
|
|
-----------
|
|
score_series : pd.Series
|
|
stock_id , score.
|
|
current : Position()
|
|
current of account.
|
|
trade_exchange : Exchange()
|
|
exchange.
|
|
trade_date : pd.Timestamp
|
|
date.
|
|
"""
|
|
print(score_series)
|
|
score_series = score_series.dropna()
|
|
|
|
# portfolio init weight
|
|
init_weight = current.reindex(score_series.index, fill_value=0).values.squeeze()
|
|
init_weight_sum = init_weight.sum()
|
|
if init_weight_sum > 0:
|
|
init_weight /= init_weight_sum
|
|
|
|
# covariance estimation
|
|
selector = (self.data_handler.get_range_selector(pred_date, 252), score_series.index)
|
|
price = self.data_handler.fetch(selector, level=None, squeeze=True)
|
|
F, cov_b, var_u = self.cov_estimator.predict(price, return_decomposed_components=True)
|
|
|
|
# optimize target portfolio
|
|
w_bench = self.benchmark(score_series.index, pred_date)
|
|
passed_init_weight = init_weight if init_weight_sum > 0 else None
|
|
# print(F)
|
|
# print(cov_b)
|
|
# print(var_u)
|
|
# print(passed_init_weight)
|
|
# print(w_bench)
|
|
target_weight = self.optimizer(score_series.values, F, cov_b, var_u, passed_init_weight, w_bench)
|
|
# print(target_weight)
|
|
target = pd.DataFrame(data=target_weight, index=score_series.index)
|
|
|
|
active_weights = target_weight - w_bench
|
|
selector = (self.label_handler.get_range_selector(pred_date, 1), score_series.index)
|
|
label = self.label_handler.fetch(selector, level=None, squeeze=True)
|
|
alpha = 0
|
|
for instrument, weight in zip(score_series.index, active_weights):
|
|
delta = label.loc[(pred_date, instrument)]
|
|
alpha += weight * (0 if math.isnan(delta) else delta)
|
|
|
|
print(alpha)
|
|
|
|
return alpha, target
|
|
|
|
|
|
def train():
|
|
"""train model
|
|
|
|
Returns
|
|
-------
|
|
pred_score: pandas.DataFrame
|
|
predict scores
|
|
performance: dict
|
|
model performance
|
|
"""
|
|
|
|
# model initiation
|
|
model = init_instance_by_config(task["model"])
|
|
dataset = init_instance_by_config(task["dataset"])
|
|
|
|
# start exp
|
|
with R.start(experiment_name="workflow"):
|
|
R.log_params(**flatten_dict(task))
|
|
model.fit(dataset)
|
|
|
|
# prediction
|
|
recorder = R.get_recorder()
|
|
rid = recorder.id
|
|
sr = SignalRecord(model, dataset, recorder)
|
|
sr.generate()
|
|
pred_score = sr.load()
|
|
|
|
# calculate ic and ric
|
|
sar = SigAnaRecord(recorder)
|
|
sar.generate()
|
|
ic = sar.load(sar.get_path("ic.pkl"))
|
|
ric = sar.load(sar.get_path("ric.pkl"))
|
|
|
|
return pred_score, {"ic": ic, "ric": ric}, rid
|
|
|
|
|
|
def backtest_analysis(scores):
|
|
"""backtest enhanced indexing
|
|
|
|
Parameters
|
|
----------
|
|
scores: pandas.DataFrame
|
|
predict scores
|
|
|
|
Returns
|
|
-------
|
|
sharpe_ratio: floating-point
|
|
sharpe ratio of the enhanced indexing portfolio
|
|
"""
|
|
|
|
# backtest and analysis
|
|
with R.start(experiment_name="backtest_analysis"):
|
|
strategy = EnhancedIndexingStrategy()
|
|
dates = scores.index.get_level_values(0).unique()
|
|
|
|
alphas = []
|
|
current = pd.DataFrame()
|
|
gap_between_next_trade = 0
|
|
for date in tqdm(dates):
|
|
if gap_between_next_trade == 0:
|
|
score_series = scores.loc[date]
|
|
alpha, current = strategy.update(score_series, current, date)
|
|
alphas.append(alpha)
|
|
gap_between_next_trade = trade_gap
|
|
else:
|
|
gap_between_next_trade -= 1
|
|
|
|
alphas = np.array(alphas)
|
|
sharpe_ratio = alphas.mean() / np.std(alphas)
|
|
print('Sharpe:', sharpe_ratio)
|
|
|
|
return sharpe_ratio
|
|
|
|
|
|
class TestAllFlow(TestAutoData):
|
|
PRED_SCORE = None
|
|
REPORT_NORMAL = None
|
|
POSITIONS = None
|
|
RID = None
|
|
|
|
@classmethod
|
|
def tearDownClass(cls) -> None:
|
|
shutil.rmtree(str(Path(C["exp_manager"]["kwargs"]["uri"].strip("file:")).resolve()))
|
|
|
|
def test_0_train(self):
|
|
TestAllFlow.PRED_SCORE, ic_ric, TestAllFlow.RID = train()
|
|
self.assertGreaterEqual(ic_ric["ic"].all(), 0, "train failed")
|
|
self.assertGreaterEqual(ic_ric["ric"].all(), 0, "train failed")
|
|
|
|
def test_1_backtest(self):
|
|
sharpe_ratio = backtest_analysis(TestAllFlow.PRED_SCORE)
|
|
self.assertGreaterEqual(
|
|
sharpe_ratio,
|
|
0.90,
|
|
"backtest failed",
|
|
)
|
|
|
|
|
|
def suite():
|
|
_suite = unittest.TestSuite()
|
|
_suite.addTest(TestAllFlow("test_0_train"))
|
|
_suite.addTest(TestAllFlow("test_1_backtest"))
|
|
return _suite
|
|
|
|
|
|
if __name__ == "__main__":
|
|
runner = unittest.TextTestRunner()
|
|
runner.run(suite())
|