From fcca242807c18c4ec5053876232145a6234c9b78 Mon Sep 17 00:00:00 2001 From: Young Date: Mon, 26 Jul 2021 17:05:33 +0000 Subject: [PATCH] add cash settlement mechanism --- qlib/backtest/__init__.py | 2 +- qlib/backtest/account.py | 15 ++- qlib/backtest/exchange.py | 39 ++++--- qlib/backtest/executor.py | 63 ++++++------ qlib/backtest/order.py | 10 +- qlib/backtest/position.py | 134 ++++++++++++++----------- qlib/contrib/strategy/rule_strategy.py | 20 +++- qlib/utils/exceptions.py | 7 +- 8 files changed, 170 insertions(+), 120 deletions(-) diff --git a/qlib/backtest/__init__.py b/qlib/backtest/__init__.py index 23b8ec9c5..a97841da7 100644 --- a/qlib/backtest/__init__.py +++ b/qlib/backtest/__init__.py @@ -184,7 +184,7 @@ def backtest( exchange_kwargs={}, pos_type: str = "Position", ): - """initialize the strategy and executor, then backtest funciton for the interaction of the outermost strategy and executor in the nested decision execution + """initialize the strategy and executor, then backtest function for the interaction of the outermost strategy and executor in the nested decision execution Parameters ---------- diff --git a/qlib/backtest/account.py b/qlib/backtest/account.py index 03e51c740..773e1a037 100644 --- a/qlib/backtest/account.py +++ b/qlib/backtest/account.py @@ -1,9 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. - - +from __future__ import annotations import copy -from typing import Dict, List, Tuple +from typing import Dict, List, Tuple, TYPE_CHECKING from qlib.utils import init_instance_by_config import warnings import pandas as pd @@ -11,7 +10,9 @@ import pandas as pd from .position import BasePosition, InfPosition, Position from .report import Report, Indicator from .order import BaseTradeDecision, Order -from .exchange import Exchange + +if TYPE_CHECKING: + from .exchange import Exchange """ rtn & earning in the Account @@ -105,8 +106,6 @@ class Account: "kwargs": { "cash": init_cash, "position_dict": position_dict, - "start_time": benchmark_config["start_time"], - "freq": freq, }, "module_path": "qlib.backtest.position", } @@ -122,7 +121,7 @@ class Account: self.report = Report(freq, benchmark_config) self.positions = {} - # trading related matric(e.g. high-frequency trading) + # trading related metrics(e.g. high-frequency trading) self.indicator = Indicator() def reset(self, freq=None, benchmark_config=None, init_report=False, port_metr_enabled: bool = None): @@ -302,7 +301,7 @@ class Account: if atomic is True and trade_info is None: raise ValueError("trade_info is necessary in atomic executor") elif atomic is False and inner_order_indicators is None: - raise ValueError("inner_order_indicators is necessary in unatomic executor") + raise ValueError("inner_order_indicators is necessary in un-atomic executor") # TODO: `update_bar_count` and `update_current` should placed in Position and be merged. self.update_bar_count() diff --git a/qlib/backtest/exchange.py b/qlib/backtest/exchange.py index e73510743..9044179e0 100644 --- a/qlib/backtest/exchange.py +++ b/qlib/backtest/exchange.py @@ -1,8 +1,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +from __future__ import annotations +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from .account import Account -from qlib.backtest.position import Position +from qlib.backtest.position import BasePosition, Position import random import logging from typing import List, Tuple, Union, Callable, Iterable @@ -278,7 +282,7 @@ class Exchange: else: return True - def deal_order(self, order, trade_account=None, position=None): + def deal_order(self, order, trade_account: Account = None, position: BasePosition = None): """ Deal order when the actual transaction @@ -289,13 +293,12 @@ class Exchange: :param position: position to be updated after dealing the order. :return: trade_val, trade_cost, trade_price """ - # need to check order first - # TODO: check the order unit limit in the exchange!!!! - # The order limit is related to the adj factor and the cur_amount. - # factor = self.quote[(order.stock_id, order.trade_date)]['$factor'] - # cur_amount = trade_account.current.get_stock_amount(order.stock_id) + # check order first. if self.check_order(order) is False: - raise AttributeError("need to check order first") + order.deal_amount = 0.0 + # using np.nan instead of None to make it more convenient to should the value in format string + return 0.0, 0.0, np.nan + if trade_account is not None and position is not None: raise ValueError("trade_account and position can only choose one") @@ -304,14 +307,18 @@ class Exchange: trade_val, trade_cost = self._calc_trade_info_by_order( order, trade_account.current if trade_account else position ) - # update account if order.deal_amount > 1e-5: - # If the order can only be deal 0 aomount. Nothing to be updated - # Otherwise, it will result some stock with 0 amount in the position + # If the order can only be deal 0 amount. Nothing to be updated + # Otherwise, it will result in + # 1) some stock with 0 amount in the position + # 2) `trade_unit` of trade_cost will be lost in user account if trade_account: trade_account.update_order(order=order, trade_val=trade_val, cost=trade_cost, trade_price=trade_price) elif position: position.update_order(order=order, trade_val=trade_val, cost=trade_cost, trade_price=trade_price) + else: + # if dealing is not successful, the trade_cost should be zero + trade_cost = 0 return trade_val, trade_cost, trade_price @@ -346,7 +353,7 @@ class Exchange: `None`: if the stock is suspended `None` may be returned `float`: return factor if the factor exists """ - assert (start_time is not None and end_time is not None, "the time range must be given") + assert start_time is not None and end_time is not None, "the time range must be given" if stock_id not in self.quote.get_all_stock(): return None return self.quote.get_data(stock_id, start_time, end_time, fields="$factor", method=ts_data_last) @@ -509,7 +516,7 @@ class Exchange: ) return value - def _get_factor_or_raise_erorr(self, factor: float = None, stock_id: str = None, start_time=None, end_time=None): + def _get_factor_or_raise_error(self, factor: float = None, stock_id: str = None, start_time=None, end_time=None): """Please refer to the docs of get_amount_of_trade_unit""" if factor is None: if stock_id is not None and start_time is not None and end_time is not None: @@ -537,7 +544,7 @@ class Exchange: the end time of trading range """ if not self.trade_w_adj_price and self.trade_unit is not None: - factor = self._get_factor_or_raise_erorr( + factor = self._get_factor_or_raise_error( factor=factor, stock_id=stock_id, start_time=start_time, end_time=end_time ) return self.trade_unit / factor @@ -556,7 +563,7 @@ class Exchange: """ if not self.trade_w_adj_price and self.trade_unit is not None: # the minimal amount is 1. Add 0.1 for solving precision problem. - factor = self._get_factor_or_raise_erorr( + factor = self._get_factor_or_raise_error( factor=factor, stock_id=stock_id, start_time=start_time, end_time=end_time ) return (deal_amount * factor + 0.1) // self.trade_unit * self.trade_unit / factor @@ -626,7 +633,7 @@ class Exchange: order.stock_id, order.start_time, order.end_time, order.deal_amount ) trade_val = order.deal_amount * trade_price - trade_cost = trade_val * self.open_cost + trade_cost = max(trade_val * self.open_cost, self.min_cost) else: raise NotImplementedError("order type {} error".format(order.type)) diff --git a/qlib/backtest/executor.py b/qlib/backtest/executor.py index b05b73801..89f5a2c4a 100644 --- a/qlib/backtest/executor.py +++ b/qlib/backtest/executor.py @@ -1,5 +1,6 @@ from abc import abstractclassmethod, abstractmethod import copy +from qlib.backtest.position import BasePosition from qlib.log import get_module_logger from types import GeneratorType from qlib.backtest.account import Account @@ -32,6 +33,7 @@ class BaseExecutor: track_data: bool = False, trade_exchange: Exchange = None, common_infra: CommonInfrastructure = None, + settle_type=BasePosition.ST_NO, **kwargs, ): """ @@ -95,6 +97,8 @@ class BaseExecutor: - trade_exchange : Exchange, optional exchange that provides market info + settle_type : str + Please refer to the docs of BasePosition.settle_start """ self.time_per_step = time_per_step self.indicator_config = indicator_config @@ -104,6 +108,7 @@ class BaseExecutor: self._trade_exchange = trade_exchange self.level_infra = LevelInfrastructure() self.level_infra.reset_infra(common_infra=common_infra) + self._settle_type = settle_type self.reset(start_time=start_time, end_time=end_time, common_infra=common_infra) if common_infra is None: get_module_logger("BaseExecutor").warning(f"`common_infra` is not set for {self}") @@ -235,6 +240,9 @@ class BaseExecutor: if atomic and trade_decision.get_range_limit(default_value=None) is not None: raise ValueError("atomic executor doesn't support specify `range_limit`") + if self._settle_type != BasePosition.ST_NO: + self.trade_account.current.settle_start(self._settle_type) + obj = self._collect_data(trade_decision=trade_decision, level=level) if isinstance(obj, GeneratorType): @@ -256,6 +264,10 @@ class BaseExecutor: ) self.trade_calendar.step() + + if self._settle_type != BasePosition.ST_NO: + self.trade_account.current.settle_commit() + if return_value is not None: return_value.update({"execute_result": res}) return res @@ -366,7 +378,7 @@ class NestedExecutor(BaseExecutor): trade_decision = self._update_trade_decision(trade_decision) if trade_decision.empty() and self._skip_empty_decision: - # give one chance for outer stategy to update the strategy + # give one chance for outer strategy to update the strategy # - For updating some information in the sub executor(the strategy have no knowledge of the inner # executor when generating the decision) break @@ -409,6 +421,9 @@ class NestedExecutor(BaseExecutor): class SimulatorExecutor(BaseExecutor): """Executor that simulate the true market""" + # TODO: TT_SERIAL & TT_PARAL will be replaced by feature fix_pos now. + # Please remove them in the future. + # available trade_types TT_SERIAL = "serial" ## The orders will be executed serially in a sequence @@ -486,34 +501,22 @@ class SimulatorExecutor(BaseExecutor): execute_result = [] for order in self._get_order_iterator(trade_decision): - if self.trade_exchange.check_order(order) is True: - # execute the order. - # NOTE: The trade_account will be changed in this function - trade_val, trade_cost, trade_price = self.trade_exchange.deal_order( - order, trade_account=self.trade_account - ) - execute_result.append((order, trade_val, trade_cost, trade_price)) - if self.verbose: - if order.direction == Order.SELL: # sell - action = "sell" - else: - action = "buy" - print( - "[I {:%Y-%m-%d %H:%M:%S}]: {} {}, price {:.2f}, amount {}, deal_amount {}, factor {}, value {:.2f}, cach {:.2f}.".format( - trade_start_time, - action, - order.stock_id, - trade_price, - order.amount, - order.deal_amount, - order.factor, - trade_val, - self.trade_account.get_cash(), - ) + # execute the order. + # NOTE: The trade_account will be changed in this function + trade_val, trade_cost, trade_price = self.trade_exchange.deal_order(order, trade_account=self.trade_account) + execute_result.append((order, trade_val, trade_cost, trade_price)) + if self.verbose: + print( + "[I {:%Y-%m-%d %H:%M:%S}]: {} {}, price {:.2f}, amount {}, deal_amount {}, factor {}, value {:.2f}, cash {:.2f}.".format( + trade_start_time, + "sell" if order.direction == Order.SELL else "buy", + order.stock_id, + trade_price, + order.amount, + order.deal_amount, + order.factor, + trade_val, + self.trade_account.get_cash(), ) - else: - if self.verbose: - print("[W {:%Y-%m-%d %H:%M:%S}]: {} wrong.".format(trade_start_time, order.stock_id)) - # do nothing - pass + ) return execute_result, {"trade_info": execute_result} diff --git a/qlib/backtest/order.py b/qlib/backtest/order.py index b99cdb8e3..bb615dc06 100644 --- a/qlib/backtest/order.py +++ b/qlib/backtest/order.py @@ -58,12 +58,19 @@ class Order: # 3) results # - users should not care about these values # - they are set by the backtest system after finishing the results. + # What the value should be about in all kinds of cases + # - not tradable: the deal_amount == 0 , factor is None + # - the stock is suspended and the entire order fails. No cost for this order + # - dealed or partially dealed: deal_amount >= 0 and factor is not None deal_amount: float = field(init=False) # `deal_amount` is a non-negative value factor: float = field(init=False) + # TODO: + # a status field to indicate the dealing result of the order + # FIXME: # for compatible now. - # Plese remove them in the future + # Please remove them in the future SELL: ClassVar[OrderDir] = OrderDir.SELL BUY: ClassVar[OrderDir] = OrderDir.BUY @@ -71,6 +78,7 @@ class Order: if self.direction not in {Order.SELL, Order.BUY}: raise NotImplementedError("direction not supported, `Order.SELL` for sell, `Order.BUY` for buy") self.deal_amount = 0 + self.factor = None @property def amount_delta(self) -> float: diff --git a/qlib/backtest/position.py b/qlib/backtest/position.py index 92b66a342..e4f1ab40c 100644 --- a/qlib/backtest/position.py +++ b/qlib/backtest/position.py @@ -20,8 +20,8 @@ class BasePosition: Please refer to the `Position` class for the position """ - def __init__(self, cash=0.0, *args, **kwargs) -> None: - pass + def __init__(self, cash=0.0, *args, **kwargs): + self._settle_type = self.ST_NO def skip_update(self) -> bool: """ @@ -124,13 +124,16 @@ class BasePosition: """ raise NotImplementedError(f"Please implement the `get_stock_amount` method") - def get_cash(self) -> float: + def get_cash(self, include_settle: bool = False) -> float: """ Returns ------- float: - the cash in position + the available(tradable) cash in position + include_settle: + will the unsettled(delayed) cash included + Default: not include those unavailable cash """ raise NotImplementedError(f"Please implement the `get_cash` method") @@ -188,6 +191,37 @@ class BasePosition: """ raise NotImplementedError(f"Please implement the `add_count_all` method") + ST_CASH = "cash" + ST_NO = None + + def settle_start(self, settle_type: str): + """ + settlement start + It will act like start and commit a transaction + + Parameters + ---------- + settle_type : str + Should we make delay the settlement in each execution (each execution will make the executor a step forward) + - "cash": make the cash settlement delayed. + - The cash you get can't be used in current step (e.g. you can't sell a stock to get cash to buy another + stock) + - None: not settlement mechanism + - TODO: other assets will be supported in the future. + """ + raise NotImplementedError(f"Please implement the `settle_conf` method") + + def settle_commit(self): + """ + settlement commit + + Parameters + ---------- + settle_type : str + please refer to the documents of Executor + """ + raise NotImplementedError(f"Please implement the `settle_commit` method") + class Position(BasePosition): """Position @@ -203,7 +237,7 @@ class Position(BasePosition): } """ - def __init__(self, start_time, freq, cash: float = 0, position_dict: Dict[str, Dict[str, float]] = {}): + def __init__(self, cash: float = 0, position_dict: Dict[str, Dict[str, float]] = {}): """Init position by cash and position_dict. Parameters @@ -217,11 +251,12 @@ class Position(BasePosition): if there is no price key in the dict of stocks, it will be filled by _fill_stock_value. by default {}. """ + super().__init__() # NOTE: The position dict must be copied!!! # Otherwise the initial value self.init_cash = cash - self.position = self._fill_stock_value(position_dict.copy(), start_time, freq) + self.position = position_dict.copy() self.position["cash"] = cash self.position["now_account_value"] = self.calculate_value() @@ -312,7 +347,13 @@ class Position(BasePosition): elif abs(self.position[stock_id]["amount"]) <= 1e-5: self._del_stock(stock_id) - self.position["cash"] += trade_val - cost + new_cash = trade_val - cost + if self._settle_type == self.ST_CASH: + self.position["cash_delay"] += new_cash + elif self._settle_type == self.ST_NO: + self.position["cash"] += new_cash + else: + raise NotImplementedError(f"This type of input is not supported") def _del_stock(self, stock_id): del self.position[stock_id] @@ -340,9 +381,6 @@ class Position(BasePosition): def update_stock_weight(self, stock_id, weight): self.position[stock_id]["weight"] = weight - def update_cash(self, cash): - self.position["cash"] = cash - def calculate_stock_value(self): stock_list = self.get_stock_list() value = 0 @@ -352,11 +390,11 @@ class Position(BasePosition): def calculate_value(self): value = self.calculate_stock_value() - value += self.position["cash"] + value += self.position["cash"] + self.position.get("cash_delay", 0.0) return value def get_stock_list(self): - stock_list = list(set(self.position.keys()) - {"cash", "now_account_value"}) + stock_list = list(set(self.position.keys()) - {"cash", "now_account_value", "cash_delay"}) return stock_list def get_stock_price(self, code): @@ -375,8 +413,11 @@ class Position(BasePosition): def get_stock_weight(self, code): return self.position[code]["weight"] - def get_cash(self): - return self.position["cash"] + def get_cash(self, include_settle=False): + cash = self.position["cash"] + if include_settle: + cash += self.position.get("cash_delay", 0.0) + return cash def get_stock_amount_dict(self): """generate stock amount dict {stock_id : amount of stock}""" @@ -388,7 +429,7 @@ class Position(BasePosition): def get_stock_weight_dict(self, only_stock=False): """get_stock_weight_dict - generate stock weight fict {stock_id : value weight of stock in the position} + generate stock weight dict {stock_id : value weight of stock in the position} it is meaningful in the beginning or the end of each trade date :param only_stock: If only_stock=True, the weight of each stock in total stock will be returned @@ -417,49 +458,20 @@ class Position(BasePosition): for stock_code, weight in weight_dict.items(): self.update_stock_weight(stock_code, weight) - def save_position(self, path): - path = pathlib.Path(path) - p = copy.deepcopy(self.position) - cash = pd.Series(dtype=float) - cash["init_cash"] = self.init_cash - cash["cash"] = p["cash"] - cash["now_account_value"] = p["now_account_value"] - del p["cash"] - del p["now_account_value"] - positions = pd.DataFrame.from_dict(p, orient="index") - with pd.ExcelWriter(path) as writer: - positions.to_excel(writer, sheet_name="position") - cash.to_excel(writer, sheet_name="info") + def settle_start(self, settle_type): + assert self._settle_type == self.ST_NO, "Currently, settlement can't be nested!!!!!" + self._settle_type = settle_type + if settle_type == self.ST_CASH: + self.position["cash_delay"] = 0.0 - def load_position(self, path): - """load position information from a file - should have format below - sheet "position" - columns: ['stock', f'count_{bar}', 'amount', 'price', 'weight'] - f'count_{bar}': , - 'amount': , - 'price': , - 'weight': , - - sheet "cash" - index: ['init_cash', 'cash', 'now_account_value'] - 'init_cash': , - 'cash': , - 'now_account_value': - """ - path = pathlib.Path(path) - positions = pd.read_excel(open(path, "rb"), sheet_name="position", index_col=0) - cash_record = pd.read_excel(open(path, "rb"), sheet_name="info", index_col=0) - positions = positions.to_dict(orient="index") - init_cash = cash_record.loc["init_cash"].values[0] - cash = cash_record.loc["cash"].values[0] - now_account_value = cash_record.loc["now_account_value"].values[0] - # assign values - self.position = {} - self.init_cash = init_cash - self.position = positions - self.position["cash"] = cash - self.position["now_account_value"] = now_account_value + def settle_commit(self): + if self._settle_type != self.ST_NO: + if self._settle_type == self.ST_CASH: + self.position["cash"] += self.position["cash_delay"] + del self.position["cash_delay"] + else: + raise NotImplementedError(f"This type of input is not supported") + self._settle_type = self.ST_NO class InfPosition(BasePosition): @@ -502,7 +514,7 @@ class InfPosition(BasePosition): def get_stock_amount(self, code) -> float: return np.inf - def get_cash(self) -> float: + def get_cash(self, include_settle=False) -> float: return np.inf def get_stock_amount_dict(self) -> Dict: @@ -516,3 +528,9 @@ class InfPosition(BasePosition): def update_weight_all(self): raise NotImplementedError(f"InfPosition doesn't support update_weight_all") + + def settle_start(self, settle_type: str): + pass + + def settle_commit(self): + pass diff --git a/qlib/contrib/strategy/rule_strategy.py b/qlib/contrib/strategy/rule_strategy.py index 36059f5a0..57ca005ff 100644 --- a/qlib/contrib/strategy/rule_strategy.py +++ b/qlib/contrib/strategy/rule_strategy.py @@ -18,7 +18,12 @@ from qlib.backtest.utils import get_start_end_idx class TWAPStrategy(BaseStrategy): - """TWAP Strategy for trading""" + """TWAP Strategy for trading + + NOTE: + - This TWAP strategy will celling round when trading. This will make the TWAP trading strategy produce the order + ealier when the total trade unit of amount is less than the trading step + """ def reset(self, outer_trade_decision: BaseTradeDecision = None, **kwargs): """ @@ -583,7 +588,11 @@ class FileOrderStrategy(BaseStrategy): """ def __init__( - self, file: Union[IO, str, Path], trade_range: Union[Tuple[int, int], TradeRange] = None, *args, **kwargs + self, + file: Union[IO, str, Path, pd.DataFrame], + trade_range: Union[Tuple[int, int], TradeRange] = None, + *args, + **kwargs, ): """ @@ -611,8 +620,11 @@ class FileOrderStrategy(BaseStrategy): """ super().__init__(*args, **kwargs) - with get_io_object(file) as f: - self.order_df = pd.read_csv(f, dtype={"datetime": np.str}) + if isinstance(file, pd.DataFrame): + self.order_df = file + else: + with get_io_object(file) as f: + self.order_df = pd.read_csv(f, dtype={"datetime": np.str}) self.order_df["datetime"] = self.order_df["datetime"].apply(pd.Timestamp) self.order_df = self.order_df.set_index(["datetime", "instrument"]) diff --git a/qlib/utils/exceptions.py b/qlib/utils/exceptions.py index dad12506b..dd9b3eaf6 100644 --- a/qlib/utils/exceptions.py +++ b/qlib/utils/exceptions.py @@ -1,17 +1,20 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. + # Base exception class class QlibException(Exception): def __init__(self, message): super(QlibException, self).__init__(message) -# Error type for reinitialization when starting an experiment class RecorderInitializationError(QlibException): + """Error type for re-initialization when starting an experiment""" + pass -# Error type for Recorder when can not load object class LoadObjectError(QlibException): + """Error type for Recorder when can not load object""" + pass