liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-15 08:46:56 +08:00
Code Activity

add cash settlement mechanism

Browse Source
This commit is contained in:
Young
2021-07-26 17:05:33 +00:00
parent 4924717276
commit fcca242807
8 changed files with 170 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
qlib/backtest/__init__.py
View File

@@ -184,7 +184,7 @@ def backtest(
exchange_kwargs={},
pos_type: str = "Position",
):
"""initialize the strategy and executor, then backtest funciton for the interaction of the outermost strategy and executor in the nested decision execution
"""initialize the strategy and executor, then backtest function for the interaction of the outermost strategy and executor in the nested decision execution
Parameters
----------

15
qlib/backtest/account.py
View File

@@ -1,9 +1,8 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
from __future__ import annotations
import copy
from typing import Dict, List, Tuple
from typing import Dict, List, Tuple, TYPE_CHECKING
from qlib.utils import init_instance_by_config
import warnings
import pandas as pd
@@ -11,7 +10,9 @@ import pandas as pd
from .position import BasePosition, InfPosition, Position
from .report import Report, Indicator
from .order import BaseTradeDecision, Order
from .exchange import Exchange
if TYPE_CHECKING:
from .exchange import Exchange
"""
rtn & earning in the Account
@@ -105,8 +106,6 @@ class Account:
"kwargs": {
"cash": init_cash,
"position_dict": position_dict,
"start_time": benchmark_config["start_time"],
"freq": freq,
},
"module_path": "qlib.backtest.position",
}
@@ -122,7 +121,7 @@ class Account:
self.report = Report(freq, benchmark_config)
self.positions = {}
# trading related matric(e.g. high-frequency trading)
# trading related metrics(e.g. high-frequency trading)
self.indicator = Indicator()
def reset(self, freq=None, benchmark_config=None, init_report=False, port_metr_enabled: bool = None):
@@ -302,7 +301,7 @@ class Account:
if atomic is True and trade_info is None:
raise ValueError("trade_info is necessary in atomic executor")
elif atomic is False and inner_order_indicators is None:
raise ValueError("inner_order_indicators is necessary in unatomic executor")
raise ValueError("inner_order_indicators is necessary in un-atomic executor")
# TODO: `update_bar_count` and `update_current` should placed in Position and be merged.
self.update_bar_count()

39
qlib/backtest/exchange.py
View File

@@ -1,8 +1,12 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .account import Account
from qlib.backtest.position import Position
from qlib.backtest.position import BasePosition, Position
import random
import logging
from typing import List, Tuple, Union, Callable, Iterable
@@ -278,7 +282,7 @@ class Exchange:
else:
return True
def deal_order(self, order, trade_account=None, position=None):
def deal_order(self, order, trade_account: Account = None, position: BasePosition = None):
"""
Deal order when the actual transaction
@@ -289,13 +293,12 @@ class Exchange:
:param position: position to be updated after dealing the order.
:return: trade_val, trade_cost, trade_price
"""
# need to check order first
# TODO: check the order unit limit in the exchange!!!!
# The order limit is related to the adj factor and the cur_amount.
# factor = self.quote[(order.stock_id, order.trade_date)]['$factor']
# cur_amount = trade_account.current.get_stock_amount(order.stock_id)
# check order first.
if self.check_order(order) is False:
raise AttributeError("need to check order first")
order.deal_amount = 0.0
# using np.nan instead of None to make it more convenient to should the value in format string
return 0.0, 0.0, np.nan
if trade_account is not None and position is not None:
raise ValueError("trade_account and position can only choose one")
@@ -304,14 +307,18 @@ class Exchange:
trade_val, trade_cost = self._calc_trade_info_by_order(
order, trade_account.current if trade_account else position
)
# update account
if order.deal_amount > 1e-5:
# If the order can only be deal 0 aomount. Nothing to be updated
# Otherwise, it will result some stock with 0 amount in the position
# If the order can only be deal 0 amount. Nothing to be updated
# Otherwise, it will result in
# 1) some stock with 0 amount in the position
# 2) `trade_unit` of trade_cost will be lost in user account
if trade_account:
trade_account.update_order(order=order, trade_val=trade_val, cost=trade_cost, trade_price=trade_price)
elif position:
position.update_order(order=order, trade_val=trade_val, cost=trade_cost, trade_price=trade_price)
else:
# if dealing is not successful, the trade_cost should be zero
trade_cost = 0
return trade_val, trade_cost, trade_price
@@ -346,7 +353,7 @@ class Exchange:
`None`: if the stock is suspended `None` may be returned
`float`: return factor if the factor exists
"""
assert (start_time is not None and end_time is not None, "the time range must be given")
assert start_time is not None and end_time is not None, "the time range must be given"
if stock_id not in self.quote.get_all_stock():
return None
return self.quote.get_data(stock_id, start_time, end_time, fields="$factor", method=ts_data_last)
@@ -509,7 +516,7 @@ class Exchange:
)
return value
def _get_factor_or_raise_erorr(self, factor: float = None, stock_id: str = None, start_time=None, end_time=None):
def _get_factor_or_raise_error(self, factor: float = None, stock_id: str = None, start_time=None, end_time=None):
"""Please refer to the docs of get_amount_of_trade_unit"""
if factor is None:
if stock_id is not None and start_time is not None and end_time is not None:
@@ -537,7 +544,7 @@ class Exchange:
the end time of trading range
"""
if not self.trade_w_adj_price and self.trade_unit is not None:
factor = self._get_factor_or_raise_erorr(
factor = self._get_factor_or_raise_error(
factor=factor, stock_id=stock_id, start_time=start_time, end_time=end_time
)
return self.trade_unit / factor
@@ -556,7 +563,7 @@ class Exchange:
"""
if not self.trade_w_adj_price and self.trade_unit is not None:
# the minimal amount is 1. Add 0.1 for solving precision problem.
factor = self._get_factor_or_raise_erorr(
factor = self._get_factor_or_raise_error(
factor=factor, stock_id=stock_id, start_time=start_time, end_time=end_time
)
return (deal_amount * factor + 0.1) // self.trade_unit * self.trade_unit / factor
@@ -626,7 +633,7 @@ class Exchange:
order.stock_id, order.start_time, order.end_time, order.deal_amount
)
trade_val = order.deal_amount * trade_price
trade_cost = trade_val * self.open_cost
trade_cost = max(trade_val * self.open_cost, self.min_cost)
else:
raise NotImplementedError("order type {} error".format(order.type))

63
qlib/backtest/executor.py
View File

@@ -1,5 +1,6 @@
from abc import abstractclassmethod, abstractmethod
import copy
from qlib.backtest.position import BasePosition
from qlib.log import get_module_logger
from types import GeneratorType
from qlib.backtest.account import Account
@@ -32,6 +33,7 @@ class BaseExecutor:
track_data: bool = False,
trade_exchange: Exchange = None,
common_infra: CommonInfrastructure = None,
settle_type=BasePosition.ST_NO,
**kwargs,
):
"""
@@ -95,6 +97,8 @@ class BaseExecutor:
- trade_exchange : Exchange, optional
exchange that provides market info
settle_type : str
Please refer to the docs of BasePosition.settle_start
"""
self.time_per_step = time_per_step
self.indicator_config = indicator_config
@@ -104,6 +108,7 @@ class BaseExecutor:
self._trade_exchange = trade_exchange
self.level_infra = LevelInfrastructure()
self.level_infra.reset_infra(common_infra=common_infra)
self._settle_type = settle_type
self.reset(start_time=start_time, end_time=end_time, common_infra=common_infra)
if common_infra is None:
get_module_logger("BaseExecutor").warning(f"`common_infra` is not set for {self}")
@@ -235,6 +240,9 @@ class BaseExecutor:
if atomic and trade_decision.get_range_limit(default_value=None) is not None:
raise ValueError("atomic executor doesn't support specify `range_limit`")
if self._settle_type != BasePosition.ST_NO:
self.trade_account.current.settle_start(self._settle_type)
obj = self._collect_data(trade_decision=trade_decision, level=level)
if isinstance(obj, GeneratorType):
@@ -256,6 +264,10 @@ class BaseExecutor:
)
self.trade_calendar.step()
if self._settle_type != BasePosition.ST_NO:
self.trade_account.current.settle_commit()
if return_value is not None:
return_value.update({"execute_result": res})
return res
@@ -366,7 +378,7 @@ class NestedExecutor(BaseExecutor):
trade_decision = self._update_trade_decision(trade_decision)
if trade_decision.empty() and self._skip_empty_decision:
# give one chance for outer stategy to update the strategy
# give one chance for outer strategy to update the strategy
# - For updating some information in the sub executor(the strategy have no knowledge of the inner
# executor when generating the decision)
break
@@ -409,6 +421,9 @@ class NestedExecutor(BaseExecutor):
class SimulatorExecutor(BaseExecutor):
"""Executor that simulate the true market"""
# TODO: TT_SERIAL & TT_PARAL will be replaced by feature fix_pos now.
# Please remove them in the future.
# available trade_types
TT_SERIAL = "serial"
## The orders will be executed serially in a sequence
@@ -486,34 +501,22 @@ class SimulatorExecutor(BaseExecutor):
execute_result = []
for order in self._get_order_iterator(trade_decision):
if self.trade_exchange.check_order(order) is True:
# execute the order.
# NOTE: The trade_account will be changed in this function
trade_val, trade_cost, trade_price = self.trade_exchange.deal_order(
order, trade_account=self.trade_account
)
execute_result.append((order, trade_val, trade_cost, trade_price))
if self.verbose:
if order.direction == Order.SELL: # sell
action = "sell"
else:
action = "buy"
print(
"[I {:%Y-%m-%d %H:%M:%S}]: {} {}, price {:.2f}, amount {}, deal_amount {}, factor {}, value {:.2f}, cach {:.2f}.".format(
trade_start_time,
action,
order.stock_id,
trade_price,
order.amount,
order.deal_amount,
order.factor,
trade_val,
self.trade_account.get_cash(),
)
# execute the order.
# NOTE: The trade_account will be changed in this function
trade_val, trade_cost, trade_price = self.trade_exchange.deal_order(order, trade_account=self.trade_account)
execute_result.append((order, trade_val, trade_cost, trade_price))
if self.verbose:
print(
"[I {:%Y-%m-%d %H:%M:%S}]: {} {}, price {:.2f}, amount {}, deal_amount {}, factor {}, value {:.2f}, cash {:.2f}.".format(
trade_start_time,
"sell" if order.direction == Order.SELL else "buy",
order.stock_id,
trade_price,
order.amount,
order.deal_amount,
order.factor,
trade_val,
self.trade_account.get_cash(),
)
else:
if self.verbose:
print("[W {:%Y-%m-%d %H:%M:%S}]: {} wrong.".format(trade_start_time, order.stock_id))
# do nothing
pass
)
return execute_result, {"trade_info": execute_result}

10
qlib/backtest/order.py
View File

@@ -58,12 +58,19 @@ class Order:
# 3) results
# - users should not care about these values
# - they are set by the backtest system after finishing the results.
# What the value should be about in all kinds of cases
# - not tradable: the deal_amount == 0 , factor is None
# - the stock is suspended and the entire order fails. No cost for this order
# - dealed or partially dealed: deal_amount >= 0 and factor is not None
deal_amount: float = field(init=False) # `deal_amount` is a non-negative value
factor: float = field(init=False)
# TODO:
# a status field to indicate the dealing result of the order
# FIXME:
# for compatible now.
# Plese remove them in the future
# Please remove them in the future
SELL: ClassVar[OrderDir] = OrderDir.SELL
BUY: ClassVar[OrderDir] = OrderDir.BUY
@@ -71,6 +78,7 @@ class Order:
if self.direction not in {Order.SELL, Order.BUY}:
raise NotImplementedError("direction not supported, `Order.SELL` for sell, `Order.BUY` for buy")
self.deal_amount = 0
self.factor = None
@property
def amount_delta(self) -> float:

134
qlib/backtest/position.py
View File

@@ -20,8 +20,8 @@ class BasePosition:
Please refer to the `Position` class for the position
"""
def __init__(self, cash=0.0, *args, **kwargs) -> None:
pass
def __init__(self, cash=0.0, *args, **kwargs):
self._settle_type = self.ST_NO
def skip_update(self) -> bool:
"""
@@ -124,13 +124,16 @@ class BasePosition:
"""
raise NotImplementedError(f"Please implement the `get_stock_amount` method")
def get_cash(self) -> float:
def get_cash(self, include_settle: bool = False) -> float:
"""
Returns
-------
float:
the cash in position
the available(tradable) cash in position
include_settle:
will the unsettled(delayed) cash included
Default: not include those unavailable cash
"""
raise NotImplementedError(f"Please implement the `get_cash` method")
@@ -188,6 +191,37 @@ class BasePosition:
"""
raise NotImplementedError(f"Please implement the `add_count_all` method")
ST_CASH = "cash"
ST_NO = None
def settle_start(self, settle_type: str):
"""
settlement start
It will act like start and commit a transaction
Parameters
----------
settle_type : str
Should we make delay the settlement in each execution (each execution will make the executor a step forward)
- "cash": make the cash settlement delayed.
- The cash you get can't be used in current step (e.g. you can't sell a stock to get cash to buy another
stock)
- None: not settlement mechanism
- TODO: other assets will be supported in the future.
"""
raise NotImplementedError(f"Please implement the `settle_conf` method")
def settle_commit(self):
"""
settlement commit
Parameters
----------
settle_type : str
please refer to the documents of Executor
"""
raise NotImplementedError(f"Please implement the `settle_commit` method")
class Position(BasePosition):
"""Position
@@ -203,7 +237,7 @@ class Position(BasePosition):
}
"""
def __init__(self, start_time, freq, cash: float = 0, position_dict: Dict[str, Dict[str, float]] = {}):
def __init__(self, cash: float = 0, position_dict: Dict[str, Dict[str, float]] = {}):
"""Init position by cash and position_dict.
Parameters
@@ -217,11 +251,12 @@ class Position(BasePosition):
if there is no price key in the dict of stocks, it will be filled by _fill_stock_value.
by default {}.
"""
super().__init__()
# NOTE: The position dict must be copied!!!
# Otherwise the initial value
self.init_cash = cash
self.position = self._fill_stock_value(position_dict.copy(), start_time, freq)
self.position = position_dict.copy()
self.position["cash"] = cash
self.position["now_account_value"] = self.calculate_value()
@@ -312,7 +347,13 @@ class Position(BasePosition):
elif abs(self.position[stock_id]["amount"]) <= 1e-5:
self._del_stock(stock_id)
self.position["cash"] += trade_val - cost
new_cash = trade_val - cost
if self._settle_type == self.ST_CASH:
self.position["cash_delay"] += new_cash
elif self._settle_type == self.ST_NO:
self.position["cash"] += new_cash
else:
raise NotImplementedError(f"This type of input is not supported")
def _del_stock(self, stock_id):
del self.position[stock_id]
@@ -340,9 +381,6 @@ class Position(BasePosition):
def update_stock_weight(self, stock_id, weight):
self.position[stock_id]["weight"] = weight
def update_cash(self, cash):
self.position["cash"] = cash
def calculate_stock_value(self):
stock_list = self.get_stock_list()
value = 0
@@ -352,11 +390,11 @@ class Position(BasePosition):
def calculate_value(self):
value = self.calculate_stock_value()
value += self.position["cash"]
value += self.position["cash"] + self.position.get("cash_delay", 0.0)
return value
def get_stock_list(self):
stock_list = list(set(self.position.keys()) - {"cash", "now_account_value"})
stock_list = list(set(self.position.keys()) - {"cash", "now_account_value", "cash_delay"})
return stock_list
def get_stock_price(self, code):
@@ -375,8 +413,11 @@ class Position(BasePosition):
def get_stock_weight(self, code):
return self.position[code]["weight"]
def get_cash(self):
return self.position["cash"]
def get_cash(self, include_settle=False):
cash = self.position["cash"]
if include_settle:
cash += self.position.get("cash_delay", 0.0)
return cash
def get_stock_amount_dict(self):
"""generate stock amount dict {stock_id : amount of stock}"""
@@ -388,7 +429,7 @@ class Position(BasePosition):
def get_stock_weight_dict(self, only_stock=False):
"""get_stock_weight_dict
generate stock weight fict {stock_id : value weight of stock in the position}
generate stock weight dict {stock_id : value weight of stock in the position}
it is meaningful in the beginning or the end of each trade date
:param only_stock: If only_stock=True, the weight of each stock in total stock will be returned
@@ -417,49 +458,20 @@ class Position(BasePosition):
for stock_code, weight in weight_dict.items():
self.update_stock_weight(stock_code, weight)
def save_position(self, path):
path = pathlib.Path(path)
p = copy.deepcopy(self.position)
cash = pd.Series(dtype=float)
cash["init_cash"] = self.init_cash
cash["cash"] = p["cash"]
cash["now_account_value"] = p["now_account_value"]
del p["cash"]
del p["now_account_value"]
positions = pd.DataFrame.from_dict(p, orient="index")
with pd.ExcelWriter(path) as writer:
positions.to_excel(writer, sheet_name="position")
cash.to_excel(writer, sheet_name="info")
def settle_start(self, settle_type):
assert self._settle_type == self.ST_NO, "Currently, settlement can't be nested!!!!!"
self._settle_type = settle_type
if settle_type == self.ST_CASH:
self.position["cash_delay"] = 0.0
def load_position(self, path):
"""load position information from a file
should have format below
sheet "position"
columns: ['stock', f'count_{bar}', 'amount', 'price', 'weight']
f'count_{bar}': <how many bars the security has been hold>,
'amount': <the amount of the security>,
'price': <the close price of security in the last trading day>,
'weight': <the security weight of total position value>,
sheet "cash"
index: ['init_cash', 'cash', 'now_account_value']
'init_cash': <inital cash when account was created>,
'cash': <current cash in account>,
'now_account_value': <current total account value, should equal to sum(price[stock]*amount[stock])>
"""
path = pathlib.Path(path)
positions = pd.read_excel(open(path, "rb"), sheet_name="position", index_col=0)
cash_record = pd.read_excel(open(path, "rb"), sheet_name="info", index_col=0)
positions = positions.to_dict(orient="index")
init_cash = cash_record.loc["init_cash"].values[0]
cash = cash_record.loc["cash"].values[0]
now_account_value = cash_record.loc["now_account_value"].values[0]
# assign values
self.position = {}
self.init_cash = init_cash
self.position = positions
self.position["cash"] = cash
self.position["now_account_value"] = now_account_value
def settle_commit(self):
if self._settle_type != self.ST_NO:
if self._settle_type == self.ST_CASH:
self.position["cash"] += self.position["cash_delay"]
del self.position["cash_delay"]
else:
raise NotImplementedError(f"This type of input is not supported")
self._settle_type = self.ST_NO
class InfPosition(BasePosition):
@@ -502,7 +514,7 @@ class InfPosition(BasePosition):
def get_stock_amount(self, code) -> float:
return np.inf
def get_cash(self) -> float:
def get_cash(self, include_settle=False) -> float:
return np.inf
def get_stock_amount_dict(self) -> Dict:
@@ -516,3 +528,9 @@ class InfPosition(BasePosition):
def update_weight_all(self):
raise NotImplementedError(f"InfPosition doesn't support update_weight_all")
def settle_start(self, settle_type: str):
pass
def settle_commit(self):
pass

20
qlib/contrib/strategy/rule_strategy.py
View File

@@ -18,7 +18,12 @@ from qlib.backtest.utils import get_start_end_idx
class TWAPStrategy(BaseStrategy):
"""TWAP Strategy for trading"""
"""TWAP Strategy for trading
NOTE:
- This TWAP strategy will celling round when trading. This will make the TWAP trading strategy produce the order
ealier when the total trade unit of amount is less than the trading step
"""
def reset(self, outer_trade_decision: BaseTradeDecision = None, **kwargs):
"""
@@ -583,7 +588,11 @@ class FileOrderStrategy(BaseStrategy):
"""
def __init__(
self, file: Union[IO, str, Path], trade_range: Union[Tuple[int, int], TradeRange] = None, *args, **kwargs
self,
file: Union[IO, str, Path, pd.DataFrame],
trade_range: Union[Tuple[int, int], TradeRange] = None,
*args,
**kwargs,
):
"""
@@ -611,8 +620,11 @@ class FileOrderStrategy(BaseStrategy):
"""
super().__init__(*args, **kwargs)
with get_io_object(file) as f:
self.order_df = pd.read_csv(f, dtype={"datetime": np.str})
if isinstance(file, pd.DataFrame):
self.order_df = file
else:
with get_io_object(file) as f:
self.order_df = pd.read_csv(f, dtype={"datetime": np.str})
self.order_df["datetime"] = self.order_df["datetime"].apply(pd.Timestamp)
self.order_df = self.order_df.set_index(["datetime", "instrument"])

7
qlib/utils/exceptions.py
View File

@@ -1,17 +1,20 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
# Base exception class
class QlibException(Exception):
def __init__(self, message):
super(QlibException, self).__init__(message)
# Error type for reinitialization when starting an experiment
class RecorderInitializationError(QlibException):
"""Error type for re-initialization when starting an experiment"""
pass
# Error type for Recorder when can not load object
class LoadObjectError(QlibException):
"""Error type for Recorder when can not load object"""
pass