From 735153a50dc096be50144b214f307e6ab345c07e Mon Sep 17 00:00:00 2001 From: wangwenxi-handsome <77676340+wangwenxi-handsome@users.noreply.github.com> Date: Thu, 12 Aug 2021 23:44:22 +0800 Subject: [PATCH] Cash Update (#559) * fix negative cash * update order test * fix bug * update file_order_test --- qlib/backtest/account.py | 39 +++++---- qlib/backtest/exchange.py | 106 ++++++++++-------------- tests/backtest/test_file_strategy.py | 32 +++++-- tests/backtest/test_init_position.py | 119 --------------------------- 4 files changed, 87 insertions(+), 209 deletions(-) delete mode 100644 tests/backtest/test_init_position.py diff --git a/qlib/backtest/account.py b/qlib/backtest/account.py index a2e354f8c..163ee8c26 100644 --- a/qlib/backtest/account.py +++ b/qlib/backtest/account.py @@ -165,24 +165,25 @@ class Account: def get_cash(self): return self.current.get_cash() - def _update_state_from_order(self, order, trade_val, cost, trade_price): - # update turnover - self.accum_info.add_turnover(trade_val) - # update cost - self.accum_info.add_cost(cost) + def _update_accum_info_from_order(self, order, trade_val, cost, trade_price): + if self.is_port_metr_enabled(): + # update turnover + self.accum_info.add_turnover(trade_val) + # update cost + self.accum_info.add_cost(cost) - # update return from order - trade_amount = trade_val / trade_price - if order.direction == Order.SELL: # 0 for sell - # when sell stock, get profit from price change - profit = trade_val - self.current.get_stock_price(order.stock_id) * trade_amount - self.accum_info.add_return_value(profit) # note here do not consider cost + # update return from order + trade_amount = trade_val / trade_price + if order.direction == Order.SELL: # 0 for sell + # when sell stock, get profit from price change + profit = trade_val - self.current.get_stock_price(order.stock_id) * trade_amount + self.accum_info.add_return_value(profit) # note here do not consider cost - elif order.direction == Order.BUY: # 1 for buy - # when buy stock, we get return for the rtn computing method - # profit in buy order is to make rtn is consistent with earning at the end of bar - profit = self.current.get_stock_price(order.stock_id) * trade_amount - trade_val - self.accum_info.add_return_value(profit) # note here do not consider cost + elif order.direction == Order.BUY: # 1 for buy + # when buy stock, we get return for the rtn computing method + # profit in buy order is to make rtn is consistent with earning at the end of bar + profit = self.current.get_stock_price(order.stock_id) * trade_amount - trade_val + self.accum_info.add_return_value(profit) # note here do not consider cost def update_order(self, order, trade_val, cost, trade_price): if self.current.skip_update(): @@ -195,8 +196,7 @@ class Account: # The cost will be substracted from the cash at last. So the trading logic can ignore the cost calculation if order.direction == Order.SELL: # sell stock - if getattr(self, "accum_info") is not None: - self._update_state_from_order(order, trade_val, cost, trade_price) + self._update_accum_info_from_order(order, trade_val, cost, trade_price) # update current position # for may sell all of stock_id self.current.update_order(order, trade_val, cost, trade_price) @@ -204,8 +204,7 @@ class Account: # buy stock # deal order, then update state self.current.update_order(order, trade_val, cost, trade_price) - if getattr(self, "accum_info") is not None: - self._update_state_from_order(order, trade_val, cost, trade_price) + self._update_accum_info_from_order(order, trade_val, cost, trade_price) def update_bar_count(self): """at the end of the trading bar, update holding bar, count of stock""" diff --git a/qlib/backtest/exchange.py b/qlib/backtest/exchange.py index 91c8ca30d..d36675b01 100644 --- a/qlib/backtest/exchange.py +++ b/qlib/backtest/exchange.py @@ -407,9 +407,6 @@ class Exchange: trade_account.update_order(order=order, trade_val=trade_val, cost=trade_cost, trade_price=trade_price) elif position: position.update_order(order=order, trade_val=trade_val, cost=trade_cost, trade_price=trade_price) - else: - # if dealing is not successful, the trade_cost should be zero - trade_cost = 0 return trade_val, trade_cost, trade_price @@ -713,53 +710,31 @@ class Exchange: f"Order clipped due to volume limitation: {order}, {[(vol, rule) for vol, rule in zip(vol_limit_num, vol_limit)]}" ) - def _get_max_amount_by_cash_limit(self, trade_price, order, position): - """return the real order amount after cash limit. + def _get_buy_amount_by_cash_limit(self, trade_price, cash): + """return the real order amount after cash limit for buying. Parameters ---------- trade_price : float - order : Order - position : Position + position : cash Return ---------- float - the real order amount after cash limit. + the real order amount after cash limit for buying. """ - cash = position.get_cash() max_trade_amount = 0 if cash >= self.min_cost: - if order.direction == Order.SELL: - max_trade_amount = cash / self.close_cost / trade_price - elif order.direction == Order.BUY: - critical_amount = self.min_cost / (self.open_cost * trade_price) - critical_price = critical_amount * trade_price + self.min_cost - if cash >= critical_price: - max_trade_amount = cash / (1 + self.open_cost) / trade_price - else: - max_trade_amount = (cash - self.min_cost) / trade_price + # critical_amount means the stock transaction amount when the service fee is equal to min_cost. + critical_amount = self.min_cost / self.open_cost + self.min_cost + if cash >= critical_amount: + # the service fee is equal to open_cost * trade_amount + max_trade_amount = cash / (1 + self.open_cost) / trade_price + else: + # the service fee is equal to min_cost + max_trade_amount = (cash - self.min_cost) / trade_price return max_trade_amount - def _get_max_amount_by_stock_limit(self, order, position): - """return the real order amount after stock amount limit. - - Parameters - ---------- - order : Order - position : Position - - Return - ---------- - float - the real order amount after stock amount limit. - """ - if order.direction == Order.SELL: - current_amount = position.get_stock_amount(order.stock_id) if position.check_stock(order.stock_id) else 0 - return current_amount - elif order.direction == Order.BUY: - return np.inf - def _calc_trade_info_by_order(self, order, position: Position, dealt_order_amount): """ Calculation of trade info @@ -771,55 +746,60 @@ class Exchange: :param dealt_order_amount: the dealt order amount dict with the format of {stock_id: float} :return: trade_price, trade_val, trade_cost """ - trade_price = self.get_deal_price(order.stock_id, order.start_time, order.end_time, direction=order.direction) order.factor = self.get_factor(order.stock_id, order.start_time, order.end_time) - # get all limits amount - # cash limit - cash_max_amount = self._get_max_amount_by_cash_limit(trade_price, order, position) - # held stock limit - stock_max_amount = self._get_max_amount_by_stock_limit(order, position) - if order.direction == Order.SELL: + cost_ratio = self.close_cost # sell if position is not None: - if np.isclose(order.amount, stock_max_amount): + current_amount = ( + position.get_stock_amount(order.stock_id) if position.check_stock(order.stock_id) else 0 + ) + if np.isclose(order.amount, current_amount): # when selling last stock. The amount don't need rounding - if stock_max_amount <= cash_max_amount: - order.deal_amount = stock_max_amount - else: - order.deal_amount = self.round_amount_by_trade_unit(cash_max_amount, order.factor) + order.deal_amount = order.amount else: - now_trade_amount = min(order.amount, stock_max_amount) - if now_trade_amount > cash_max_amount: - self.logger.debug(f"Order clipped due to cash limitation: {order}") - order.deal_amount = self.round_amount_by_trade_unit( - min(now_trade_amount, cash_max_amount), order.factor - ) + order.deal_amount = self.round_amount_by_trade_unit(min(current_amount, order.amount), order.factor) + + # in case of negative value of cash + if position.get_cash() + order.deal_amount * trade_price < max( + order.deal_amount * trade_price * cost_ratio, + self.min_cost, + ): + order.deal_amount = 0 + self.logger.debug(f"Order clipped due to cash limitation: {order}") else: # TODO: We don't know current position. # We choose to sell all order.deal_amount = order.amount - self._clip_amount_by_volume(order, dealt_order_amount) - trade_val = order.deal_amount * trade_price - trade_cost = max(trade_val * self.close_cost, self.min_cost) elif order.direction == Order.BUY: + cost_ratio = self.open_cost # buy if position is not None: - if order.amount > cash_max_amount: + cash = position.get_cash() + trade_val = order.amount * trade_price + if cash < trade_val + max(trade_val * cost_ratio, self.min_cost): + # The money is not enough + max_buy_amount = self._get_buy_amount_by_cash_limit(trade_price, cash) + order.deal_amount = self.round_amount_by_trade_unit(max_buy_amount, order.factor) self.logger.debug(f"Order clipped due to cash limitation: {order}") - order.deal_amount = self.round_amount_by_trade_unit(min(order.amount, cash_max_amount), order.factor) + else: + # The money is enough + order.deal_amount = self.round_amount_by_trade_unit(order.amount, order.factor) else: # Unknown amount of money. Just round the amount order.deal_amount = self.round_amount_by_trade_unit(order.amount, order.factor) - self._clip_amount_by_volume(order, dealt_order_amount) - trade_val = order.deal_amount * trade_price - trade_cost = max(trade_val * self.open_cost, self.min_cost) else: raise NotImplementedError("order type {} error".format(order.type)) + self._clip_amount_by_volume(order, dealt_order_amount) + trade_val = order.deal_amount * trade_price + trade_cost = max(trade_val * cost_ratio, self.min_cost) + if trade_val <= 1e-5: + # if dealing is not successful, the trade_cost should be zero. + trade_cost = 0 return trade_price, trade_val, trade_cost def get_order_helper(self) -> OrderHelper: diff --git a/tests/backtest/test_file_strategy.py b/tests/backtest/test_file_strategy.py index cbc29bad7..9229581ac 100644 --- a/tests/backtest/test_file_strategy.py +++ b/tests/backtest/test_file_strategy.py @@ -16,6 +16,8 @@ class FileStrTest(TestAutoData): EXAMPLE_FILE = DIRNAME / "order_example.csv" + DEAL_NUM_FOR_1000 = 123.47105436976445 + def _gen_orders(self) -> pd.DataFrame: headers = [ "datetime", @@ -24,11 +26,18 @@ class FileStrTest(TestAutoData): "direction", ] orders = [ - ["20200102", self.TEST_INST, "1000", "sell"], + # test cash limit for buying ["20200103", self.TEST_INST, "1000", "buy"], + # test min_cost for buying + ["20200103", self.TEST_INST, "1", "buy"], + # test held stock limit for selling ["20200106", self.TEST_INST, "1000", "sell"], - ["20200106", self.TEST_INST, "1000", "buy"], - ["20200106", self.TEST_INST, "949.7773413058803", "sell"], + # test cash limit for buying + ["20200107", self.TEST_INST, "1000", "buy"], + # test min_cost for selling + ["20200108", self.TEST_INST, "1", "sell"], + # test selling all stocks + ["20200110", self.TEST_INST, str(self.DEAL_NUM_FOR_1000), "sell"], ] return pd.DataFrame(orders, columns=headers).set_index(["datetime", "instrument"]) @@ -54,7 +63,7 @@ class FileStrTest(TestAutoData): backtest_config = { "start_time": start_time, "end_time": end_time, - "account": 100000000, + "account": 30000, "benchmark": None, # benchmark is not required here for trading "exchange_kwargs": { "freq": freq, @@ -62,9 +71,9 @@ class FileStrTest(TestAutoData): "deal_price": "close", "open_cost": 0.0005, "close_cost": 0.0015, - "min_cost": 5, + "min_cost": 500, "codes": codes, - "trade_unit": 100, + "trade_unit": 2, }, # "pos_type": "InfPosition" # Position with infinitive position } @@ -80,7 +89,16 @@ class FileStrTest(TestAutoData): }, }, } - backtest(executor=executor_config, strategy=strategy_config, **backtest_config) + report_dict, indicator_dict = backtest(executor=executor_config, strategy=strategy_config, **backtest_config) + + # ffr valid + ffr_dict = indicator_dict["1day"]["ffr"].to_dict() + ffr_dict = {str(date).split()[0]: ffr_dict[date] for date in ffr_dict} + assert ffr_dict["2020-01-03"] == 0 + assert ffr_dict["2020-01-06"] == self.DEAL_NUM_FOR_1000 / 1000 + assert ffr_dict["2020-01-07"] == self.DEAL_NUM_FOR_1000 / 1000 + assert ffr_dict["2020-01-08"] == 0 + assert ffr_dict["2020-01-10"] == 1 self.EXAMPLE_FILE.unlink() diff --git a/tests/backtest/test_init_position.py b/tests/backtest/test_init_position.py deleted file mode 100644 index ec3dca714..000000000 --- a/tests/backtest/test_init_position.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -import unittest -import qlib -from qlib.backtest import backtest, order -from qlib.tests import TestAutoData -from qlib.backtest.order import TradeDecisionWO, TradeRangeByTime -import pandas as pd -from pathlib import Path - - -class FileStrTest(TestAutoData): - - TEST_INST = "SH600519" - - def init_qlib(self): - provider_uri_day = "/nfs_data1/stock_data/huaxia_1d_qlib" - provider_uri_1min = "/nfs_data1/stock_data/huaxia_1min_qlib" - provider_uri_map = {"1min": provider_uri_1min, "day": provider_uri_day} - - client_config = { - "calendar_provider": { - "class": "LocalCalendarProvider", - "module_path": "qlib.data.data", - "kwargs": { - "backend": { - "class": "FileCalendarStorage", - "module_path": "qlib.data.storage.file_storage", - "kwargs": {"provider_uri_map": provider_uri_map}, - } - }, - }, - "feature_provider": { - "class": "LocalFeatureProvider", - "module_path": "qlib.data.data", - "kwargs": { - "backend": { - "class": "FileFeatureStorage", - "module_path": "qlib.data.storage.file_storage", - "kwargs": {"provider_uri_map": provider_uri_map}, - } - }, - }, - } - qlib.init(provider_uri=provider_uri_day, **client_config, expression_cache=None, dataset_cache=None) - - def test_file_str(self): - freq = "1min" - inst = ["SH600000", "SH600011"] - start_time = "2020-01-01" - end_time = "2020-01-15 15:00" - - strategy_config = { - "class": "RandomOrderStrategy", - "module_path": "qlib.contrib.strategy.rule_strategy", - "kwargs": { - "trade_range": TradeRangeByTime("9:30", "15:00"), - "sample_ratio": 1.0, - "volume_ratio": 0.01, - "market": inst, - }, - } - position_dict = { - "cash": 100000000, - "SH600000": {"amount": 100}, - "SH600011": {"amount": 101}, - } - backtest_config = { - "start_time": start_time, - "end_time": end_time, - "account": position_dict, - "benchmark": None, # benchmark is not required here for trading - "exchange_kwargs": { - "freq": freq, - "limit_threshold": 0.095, - "deal_price": "close", - "open_cost": 0.0005, - "close_cost": 0.0015, - "min_cost": 5, - "codes": inst, - }, - "pos_type": "Position", # Position with infinitive position - } - executor_config = { - "class": "NestedExecutor", - "module_path": "qlib.backtest.executor", - "kwargs": { - "time_per_step": "day", - "inner_executor": { - "class": "SimulatorExecutor", - "module_path": "qlib.backtest.executor", - "kwargs": { - "time_per_step": freq, - "generate_report": False, - "verbose": False, - # "verbose": True, - "indicator_config": { - "show_indicator": False, - }, - }, - }, - "inner_strategy": { - "class": "TWAPStrategy", - "module_path": "qlib.contrib.strategy.rule_strategy", - }, - "track_data": True, - "generate_report": True, - "indicator_config": { - "show_indicator": True, - }, - }, - } - self.init_qlib() - backtest(executor=executor_config, strategy=strategy_config, **backtest_config) - - -if __name__ == "__main__": - unittest.main()