liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-07 04:50:56 +08:00
Code Activity

fix comments & add VAStrategy & add trade indicator

Browse Source
This commit is contained in:
bxdd
2021-06-14 21:32:18 +08:00
parent 46d253b457 40416d8c30
commit f78e90171b
31 changed files with 1536 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
qlib/backtest/__init__.py
View File

@@ -4,8 +4,8 @@
from .account import Account
from .exchange import Exchange
from .executor import BaseExecutor
from .backtest import backtest as backtest_func
from .backtest import collect_data as data_generator
from .backtest import backtest_loop
from .backtest import collect_data_loop
from .utils import CommonInfrastructure
from .order import Order
@@ -116,7 +116,7 @@ def backtest(start_time, end_time, strategy, executor, benchmark="SH000300", acc
trade_strategy, trade_executor = get_strategy_executor(
start_time, end_time, strategy, executor, benchmark, account, exchange_kwargs
)
report_dict = backtest_func(start_time, end_time, trade_strategy, trade_executor)
report_dict = backtest_loop(start_time, end_time, trade_strategy, trade_executor)
return report_dict
@@ -126,6 +126,6 @@ def collect_data(start_time, end_time, strategy, executor, benchmark="SH000300",
trade_strategy, trade_executor = get_strategy_executor(
start_time, end_time, strategy, executor, benchmark, account, exchange_kwargs
)
report_dict = yield from data_generator(start_time, end_time, trade_strategy, trade_executor)
report_dict = yield from collect_data_loop(start_time, end_time, trade_strategy, trade_executor)
return report_dict

3
qlib/backtest/account.py
View File

@@ -7,7 +7,7 @@ import warnings
import pandas as pd
from .position import Position
from .report import Report
from .report import Report, Indicator
from .order import Order
@@ -42,6 +42,7 @@ class Account:
def reset_report(self, freq, benchmark_config):
self.report = Report(freq, benchmark_config)
self.indicator = Indicator()
self.positions = {}
self.rtn = 0
self.ct = 0

29
qlib/backtest/backtest.py
View File

@@ -2,8 +2,25 @@
# Licensed under the MIT License.
def backtest(start_time, end_time, trade_strategy, trade_executor):
def backtest_loop(start_time, end_time, trade_strategy, trade_executor):
"""backtest funciton for the interaction of the outermost strategy and executor in the nested decison execution
Parameters
----------
start_time : pd.Timestamp|str
closed start time for backtest
end_time : pd.Timestamp|str
closed end time for backtest
trade_strategy : BaseStrategy
the outermost portfolio strategy
trade_executor : BaseExecutor
the outermost executor
Returns
-------
report: Report
it records the trading report information
"""
trade_executor.reset(start_time=start_time, end_time=end_time)
level_infra = trade_executor.get_level_infra()
trade_strategy.reset(level_infra=level_infra)
@@ -16,8 +33,14 @@ def backtest(start_time, end_time, trade_strategy, trade_executor):
return trade_executor.get_report()
def collect_data(start_time, end_time, trade_strategy, trade_executor):
def collect_data_loop(start_time, end_time, trade_strategy, trade_executor):
"""Generator for collecting the trade decision data for rl training
Yields
-------
object
trade decision
"""
trade_executor.reset(start_time=start_time, end_time=end_time)
level_infra = trade_executor.get_level_infra()
trade_strategy.reset(level_infra=level_infra)
@@ -26,5 +49,3 @@ def collect_data(start_time, end_time, trade_strategy, trade_executor):
while not trade_executor.finished():
_trade_decision = trade_strategy.generate_trade_decision(_execute_result)
_execute_result = yield from trade_executor.collect_data(_trade_decision)
return trade_executor.get_report()

5
qlib/backtest/exchange.py
View File

@@ -342,7 +342,10 @@ class Exchange:
return -deal_amount
def generate_order_for_target_amount_position(self, target_position, current_position, start_time, end_time):
"""Parameter:
"""
Note: some future information is used in this function
Parameter:
target_position : dict { stock_id : amount }
current_postion : dict { stock_id : amount}
trade_unit : trade_unit

105
qlib/backtest/executor.py
View File

@@ -3,14 +3,14 @@ import warnings
import pandas as pd
from typing import Union
from ..utils import init_instance_by_config
from ..utils.resam import parse_freq
from .order import Order
from .exchange import Exchange
from .utils import TradeCalendarManager, CommonInfrastructure, LevelInfrastructure
from ..utils import init_instance_by_config
from ..utils.resam import parse_freq
from ..strategy.base import BaseStrategy
class BaseExecutor:
"""Base executor for trading"""
@@ -20,6 +20,7 @@ class BaseExecutor:
time_per_step: str,
start_time: Union[str, pd.Timestamp] = None,
end_time: Union[str, pd.Timestamp] = None,
show_indicator: bool = False,
generate_report: bool = False,
verbose: bool = False,
track_data: bool = False,
@@ -31,12 +32,14 @@ class BaseExecutor:
----------
time_per_step : str
trade time per trading step, used for genreate the trade calendar
show_indicator: bool, optional
whether to show indicators, such as FFR/PA/POS, .etc
generate_report : bool, optional
whether to generate report, by default False
verbose : bool, optional
whether to print trading info, by default False
track_data : bool, optional
whether to generate trade_decision, will be used when making data for multi-level training
whether to generate trade_decision, will be used when training rl agent
- If `self.track_data` is true, when making data for training, the input `trade_decision` of `execute` will be generated by `collect_data`
- Else, `trade_decision` will not be generated
common_infra : CommonInfrastructure, optional:
@@ -48,6 +51,7 @@ class BaseExecutor:
"""
self.time_per_step = time_per_step
self.show_indicator = show_indicator
self.generate_report = generate_report
self.verbose = verbose
self.track_data = track_data
@@ -103,11 +107,27 @@ class BaseExecutor:
Returns
----------
execute_result : List[object]
the executed result for trade decison
the executed result for trade decision
"""
raise NotImplementedError("execute is not implemented!")
def collect_data(self, trade_decision):
"""Generator for collecting the trade decision data for rl training
Parameters
----------
trade_decision : object
Returns
----------
execute_result : List[object]
the executed result for trade decision
Yields
-------
object
trade decision
"""
if self.track_data:
yield trade_decision
return self.execute(trade_decision)
@@ -122,6 +142,9 @@ class BaseExecutor:
"""Return all executors"""
return [self]
def get_trade_indicator(self):
return self.trade_account.indicator.trade_indicator
class NestedExecutor(BaseExecutor):
"""
@@ -129,8 +152,6 @@ class NestedExecutor(BaseExecutor):
- At each time `execute` is called, it will call the inner strategy and executor to execute the `trade_decision` in a higher frequency env.
"""
from ..strategy.base import BaseStrategy
def __init__(
self,
time_per_step: str,
@@ -138,6 +159,7 @@ class NestedExecutor(BaseExecutor):
inner_strategy: Union[BaseStrategy, dict],
start_time: Union[str, pd.Timestamp] = None,
end_time: Union[str, pd.Timestamp] = None,
show_indicator: bool = False,
generate_report: bool = False,
verbose: bool = False,
track_data: bool = False,
@@ -161,13 +183,14 @@ class NestedExecutor(BaseExecutor):
inner_executor, common_infra=common_infra, accept_types=BaseExecutor
)
self.inner_strategy = init_instance_by_config(
inner_strategy, common_infra=common_infra, accept_types=self.BaseStrategy
inner_strategy, common_infra=common_infra, accept_types=BaseStrategy
)
super(NestedExecutor, self).__init__(
time_per_step=time_per_step,
start_time=start_time,
end_time=end_time,
show_indicator=show_indicator,
generate_report=generate_report,
verbose=verbose,
track_data=track_data,
@@ -199,7 +222,7 @@ class NestedExecutor(BaseExecutor):
sub_level_infra = self.inner_executor.get_level_infra()
self.inner_strategy.reset(level_infra=sub_level_infra, outer_trade_decision=trade_decision)
def _update_trade_account(self):
def _update_trade_account(self, inner_indicators):
trade_step = self.trade_calendar.get_trade_step()
trade_start_time, trade_end_time = self.trade_calendar.get_step_time(trade_step)
self.trade_account.update_bar_count()
@@ -210,33 +233,44 @@ class NestedExecutor(BaseExecutor):
trade_exchange=self.trade_exchange,
)
self.trade_account.indicator.clear()
self.trade_account.indicator.agg_report_info(inner_indicators=inner_indicators)
self.trade_account.indicator.agg_FFR()
self.trade_account.indicator.agg_PA(inner_indicators=inner_indicators)
if self.show_indicator:
FFR_value = self.trade_account.indicator.get_statistics_FFR(method="value_weighted")
PA_value = self.trade_account.indicator.get_statistics_PA(method="value_weighted")
POS_values = self.trade_account.indicator.get_statistics_POS()
print(
"[Indicator({}) {:%Y-%m-%d}]: FFR: {}, PA: {}, POS: {}".format(
self.time_per_step, trade_start_time, FFR_value, PA_value, POS_values
)
)
def execute(self, trade_decision):
self._init_sub_trading(trade_decision)
execute_result = []
_inner_execute_result = None
while not self.inner_executor.finished():
_inner_trade_decision = self.inner_strategy.generate_trade_decision(_inner_execute_result)
_inner_execute_result = self.inner_executor.execute(trade_decision=_inner_trade_decision)
execute_result.extend(_inner_execute_result)
if hasattr(self, "trade_account"):
self._update_trade_account()
self.trade_calendar.step()
return execute_result
for _data in self.collect_data(trade_decision):
pass
return self._execute_result
def collect_data(self, trade_decision):
if self.track_data:
yield trade_decision
self.trade_calendar.step()
self._init_sub_trading(trade_decision)
execute_result = []
inner_indicators = []
_inner_execute_result = None
while not self.inner_executor.finished():
_inner_trade_decision = self.inner_strategy.generate_trade_decision(_inner_execute_result)
_inner_execute_result = yield from self.inner_executor.collect_data(trade_decision=_inner_trade_decision)
execute_result.extend(_inner_execute_result)
if hasattr(self, "trade_account"):
self._update_trade_account()
inner_indicators.append(self.inner_executor.get_trade_indicator())
if hasattr(self, "trade_account"):
self._update_trade_account(inner_indicators=inner_indicators)
self.trade_calendar.step()
self._execute_result = execute_result
return execute_result
def get_report(self):
@@ -261,6 +295,7 @@ class SimulatorExecutor(BaseExecutor):
time_per_step: str,
start_time: Union[str, pd.Timestamp] = None,
end_time: Union[str, pd.Timestamp] = None,
show_indicator: bool = False,
generate_report: bool = False,
verbose: bool = False,
track_data: bool = False,
@@ -279,6 +314,7 @@ class SimulatorExecutor(BaseExecutor):
time_per_step=time_per_step,
start_time=start_time,
end_time=end_time,
show_indicator=show_indicator,
generate_report=generate_report,
verbose=verbose,
track_data=track_data,
@@ -337,7 +373,7 @@ class SimulatorExecutor(BaseExecutor):
else:
if self.verbose:
print("[W {:%Y-%m-%d}]: {} wrong.".format(trade_start_time, order.stock_id))
print("[W {:%Y-%m-%d %H:%M:%S}]: {} wrong.".format(trade_start_time, order.stock_id))
# do nothing
pass
@@ -349,6 +385,25 @@ class SimulatorExecutor(BaseExecutor):
trade_end_time=trade_end_time,
trade_exchange=self.trade_exchange,
)
self.trade_account.indicator.clear()
self.trade_account.indicator.update_trade_info(trade_info=execute_result)
self.trade_account.indicator.update_FFR()
self.trade_account.indicator.update_PA(
freq=self.time_per_step, trade_start_time=trade_start_time, trade_end_time=trade_end_time
)
self.trade_account.indicator.record(trade_start_time=trade_start_time)
if self.show_indicator:
FFR_value = self.trade_account.indicator.get_statistics_FFR(method="value_weighted")
PA_value = self.trade_account.indicator.get_statistics_PA(method="value_weighted")
POS_values = self.trade_account.indicator.get_statistics_POS()
print(
"[Indicator({}) {:%Y-%m-%d %H:%M:%S}]: FFR: {}, PA: {}, POS: {}".format(
self.time_per_step, trade_start_time, FFR_value, PA_value, POS_values
)
)
self.trade_calendar.step()
return execute_result

195
qlib/backtest/report.py
View File

@@ -7,10 +7,11 @@ from logging import warning
import pandas as pd
import pathlib
import warnings
from pandas.core import groupby
from pandas.core.frame import DataFrame
from ..utils.resam import parse_freq, resam_ts_data
from ..utils.resam import parse_freq, resam_ts_data, get_higher_freq_feature
from ..data import D
from ..tests.config import CSI300_BENCH
@@ -79,19 +80,7 @@ class Report:
raise ValueError("benchmark freq can't be None!")
_codes = benchmark if isinstance(benchmark, list) else [benchmark]
fields = ["$close/Ref($close,1)-1"]
try:
_temp_result = D.features(_codes, fields, start_time, end_time, freq=freq, disk_cache=1)
except (ValueError, KeyError):
_, norm_freq = parse_freq(freq)
if norm_freq in ["month", "week", "day"]:
try:
_temp_result = D.features(_codes, fields, start_time, end_time, freq="day", disk_cache=1)
except (ValueError, KeyError):
_temp_result = D.features(_codes, fields, start_time, end_time, freq="1min", disk_cache=1)
elif norm_freq == "minute":
_temp_result = D.features(_codes, fields, start_time, end_time, freq="1min", disk_cache=1)
else:
raise ValueError(f"benchmark freq {freq} is not supported")
_temp_result, _ = get_higher_freq_feature(_codes, fields, start_time, end_time, freq=freq)
if len(_temp_result) == 0:
raise ValueError(f"The benchmark {_codes} does not exist. Please provide the right benchmark")
return _temp_result.groupby(level="datetime")[_temp_result.columns.tolist()[0]].mean().fillna(0)
@@ -122,11 +111,11 @@ class Report:
turnover_rate=None,
cost_rate=None,
stock_value=None,
bench_value=None,
):
# check data
if None in [
trade_start_time,
trade_end_time,
account_value,
cash,
return_rate,
@@ -135,8 +124,14 @@ class Report:
stock_value,
]:
raise ValueError(
"None in [trade_start_time, trade_end_time, account_value, cash, return_rate, turnover_rate, cost_rate, stock_value]"
"None in [trade_start_time, account_value, cash, return_rate, turnover_rate, cost_rate, stock_value]"
)
if trade_end_time is None and bench_value is None:
raise ValueError("Both trade_end_time and bench_value is None, benchmark is not usable.")
elif bench_value is None:
bench_value = self._sample_benchmark(self.bench, trade_start_time, trade_end_time)
# update report data
self.accounts[trade_start_time] = account_value
self.returns[trade_start_time] = return_rate
@@ -144,7 +139,7 @@ class Report:
self.costs[trade_start_time] = cost_rate
self.values[trade_start_time] = stock_value
self.cashes[trade_start_time] = cash
self.benches[trade_start_time] = self._sample_benchmark(self.bench, trade_start_time, trade_end_time)
self.benches[trade_start_time] = bench_value
# update latest_report_date
self.latest_report_time = trade_start_time
# finish daily report update
@@ -178,14 +173,162 @@ class Report:
index = r.index
self.init_vars()
for trade_time in index:
for trade_start_time in index:
self.update_report_record(
trade_time=trade_time,
account_value=r.loc[trade_time]["account"],
cash=r.loc[trade_time]["cash"],
return_rate=r.loc[trade_time]["return"],
turnover_rate=r.loc[trade_time]["turnover"],
cost_rate=r.loc[trade_time]["cost"],
stock_value=r.loc[trade_time]["value"],
bench_value=r.loc[trade_time]["bench"],
trade_start_time=trade_start_time,
account_value=r.loc[trade_start_time]["account"],
cash=r.loc[trade_start_time]["cash"],
return_rate=r.loc[trade_start_time]["return"],
turnover_rate=r.loc[trade_start_time]["turnover"],
cost_rate=r.loc[trade_start_time]["cost"],
stock_value=r.loc[trade_start_time]["value"],
bench_value=r.loc[trade_start_time]["bench"],
)
class Indicator:
def __init__(self):
self.indicator_his = dict()
self.trade_indicator = dict()
def __getitem__(self, key):
return self.trade_indicator[key]
def __setitem__(self, key, value):
self.trade_indicator[key] = value
def __contains__(self, key):
return key in self.trade_indicator
def clear(self):
self.trade_indicator = dict()
def record(self, trade_start_time):
self.indicator_his[trade_start_time] = pd.DataFrame(self.trade_indicator)
def update_trade_info(self, trade_info: list):
amount = dict()
deal_amount = dict()
trade_price = dict()
trade_cost = dict()
for order, _trade_val, _trade_cost, _trade_price in trade_info:
amount[order.stock_id] = order.amount * (order.direction * 2 - 1)
deal_amount[order.stock_id] = order.deal_amount * (order.direction * 2 - 1)
trade_price[order.stock_id] = _trade_price
trade_cost[order.stock_id] = _trade_cost
self["amount"] = pd.Series(amount)
self["deal_amount"] = pd.Series(deal_amount)
self["trade_price"] = pd.Series(trade_price)
self["trade_cost"] = pd.Series(trade_cost)
def update_FFR(self):
self["fulfill_rate"] = self["deal_amount"] / self["amount"]
def update_PA(self, freq, trade_start_time, trade_end_time, base_price="twap"):
base_price = base_price.lower()
instruments = list(self["amount"].index)
if base_price == "twap":
# too slow
# price_info, _ = get_higher_freq_feature(instruments, fields=["$close"], start_time=trade_start_time, end_time=trade_end_time, freq=freq)
# price_info = price_info.astype(float)
# self["base_price"] = price_info["$close"].groupby(level="instrument").mean()
self["base_price"] = self["trade_price"]
elif base_price == "vwap":
# too slow
price_info, _ = get_higher_freq_feature(
instruments,
fields=["$close", "$volume"],
start_time=trade_start_time,
end_time=trade_end_time,
freq=freq,
)
price_info = price_info.astype(float)
self["base_price"] = price_info.groupby(level="instrument").apply(
lambda x: (x["$close"] * x["$volume"]).sum() / x["$volume"].sum()
)
self["volume"] = price_info["$volume"].groupby(level="instrument").sum()
else:
raise ValueError(f"base_price {base_price} is not supported!")
self["pa"] = (self["trade_price"] - self["base_price"]) / self["base_price"]
def agg_report_info(self, inner_indicators):
amount = pd.Series()
deal_amount = pd.Series()
trade_price = pd.Series()
trade_cost = pd.Series()
for inner_indicator in inner_indicators:
amount = amount.add(inner_indicator["amount"], fill_value=0)
deal_amount = deal_amount.add(inner_indicator["deal_amount"], fill_value=0)
trade_price = trade_price.add(inner_indicator["trade_price"] * inner_indicator["deal_amount"], fill_value=0)
trade_cost = trade_cost.add(inner_indicator["trade_cost"], fill_value=0)
self["amount"] = amount
self["deal_amount"] = deal_amount
trade_price /= self["deal_amount"]
self["trade_price"] = trade_price
self["trade_cost"] = trade_cost
def agg_FFR(self):
self["fulfill_rate"] = self["deal_amount"] / self["amount"]
def agg_PA(self, inner_indicators, base_price="twap"):
base_price = base_price.lower()
if base_price == "twap":
base_price = pd.Series()
price_count = pd.Series()
for inner_indicator in inner_indicators:
base_price = base_price.add(inner_indicator["base_price"], fill_value=0)
price_count = price_count.add(pd.Series(1, index=inner_indicator["base_price"].index), fill_value=0)
base_price /= price_count
self["base_price"] = base_price
elif base_price == "vwap":
base_price = pd.Series()
volume = pd.Series()
for inner_indicator in inner_indicators:
base_price = base_price.add(inner_indicator["base_price"] * inner_indicator["volume"], fill_value=0)
volume = volume.add(inner_indicator["volume"], fill_value=0)
base_price /= volume
self["base_price"] = base_price
self["volume"] = volume
else:
raise ValueError(f"base_price {base_price} is not supported!")
self["pa"] = (self["trade_price"] - self["base_price"]) / self["base_price"]
def get_statistics_FFR(self, method="mean"):
if method == "mean":
return self["fulfill_rate"].mean()
elif method == "amount_weighted":
weights = self["deal_amount"].abs()
return (self["fulfill_rate"] * weights).sum() / weights.sum()
elif method == "value_weighted":
weights = (self["deal_amount"] * self["trade_price"]).abs()
return (self["fulfill_rate"] * weights).sum() / weights.sum()
else:
raise ValueError(f"method {method} is not supported!")
def get_statistics_PA(self, method="mean"):
pa_order = self["pa"] * (self["amount"] < 0).astype(int)
if method == "mean":
return pa_order.mean()
elif method == "amount_weighted":
weights = self["deal_amount"].abs()
return (pa_order * weights).sum() / weights.sum()
elif method == "value_weighted":
weights = (self["deal_amount"] * self["trade_price"]).abs()
return (pa_order * weights).sum() / weights.sum()
else:
raise ValueError(f"method {method} is not supported!")
def get_statistics_POS(self):
pa_order = self["pa"] * (self["amount"] < 0).astype(int)
return (pa_order > 1e-8).astype(int).sum() / len(pa_order)

7
qlib/backtest/utils.py
View File

@@ -74,7 +74,12 @@ class TradeCalendarManager:
def get_step_time(self, trade_step=0, shift=0):
"""
Get the time range of trading step
Get the left and right endpoints of the trade_step'th trading interval
About the endpoints:
- Qlib uses the closed interval in time-series data selection, which has the same performance as pandas.Series.loc
- The returned right endpoints should minus 1 seconds becasue of the closed interval representation in Qlib.
Note: Qlib supports up to minutely decision execution, so 1 seconds is less than any trading time interval.
Parameters
----------