add cash settlement mechanism

qlib/backtest/__init__.py

@@ -184,7 +184,7 @@ def backtest(
     exchange_kwargs={},
     pos_type: str = "Position",
 ):
-    """initialize the strategy and executor, then backtest funciton for the interaction of the outermost strategy and executor in the nested decision execution
+    """initialize the strategy and executor, then backtest function for the interaction of the outermost strategy and executor in the nested decision execution
 
     Parameters
     ----------

qlib/backtest/account.py

@@ -1,9 +1,8 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
-
+from __future__ import annotations
-
 import copy
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, TYPE_CHECKING
 from qlib.utils import init_instance_by_config
 import warnings
 import pandas as pd
@@ -11,7 +10,9 @@ import pandas as pd
 from .position import BasePosition, InfPosition, Position
 from .report import Report, Indicator
 from .order import BaseTradeDecision, Order
-from .exchange import Exchange
+
+if TYPE_CHECKING:
+    from .exchange import Exchange
 
 """
 rtn & earning in the Account
@@ -105,8 +106,6 @@ class Account:
                 "kwargs": {
                     "cash": init_cash,
                     "position_dict": position_dict,
-                    "start_time": benchmark_config["start_time"],
-                    "freq": freq,
                 },
                 "module_path": "qlib.backtest.position",
             }
@@ -122,7 +121,7 @@ class Account:
             self.report = Report(freq, benchmark_config)
             self.positions = {}
 
-        # trading related matric(e.g. high-frequency trading)
+        # trading related metrics(e.g. high-frequency trading)
         self.indicator = Indicator()
 
     def reset(self, freq=None, benchmark_config=None, init_report=False, port_metr_enabled: bool = None):
@@ -302,7 +301,7 @@ class Account:
         if atomic is True and trade_info is None:
             raise ValueError("trade_info is necessary in atomic executor")
         elif atomic is False and inner_order_indicators is None:
-            raise ValueError("inner_order_indicators is necessary in unatomic executor")
+            raise ValueError("inner_order_indicators is necessary in un-atomic executor")
 
         # TODO:  `update_bar_count` and  `update_current` should placed in Position and be merged.
         self.update_bar_count()

qlib/backtest/exchange.py

@@ -1,8 +1,12 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
+from __future__ import annotations
+from typing import TYPE_CHECKING
 
+if TYPE_CHECKING:
+    from .account import Account
 
-from qlib.backtest.position import Position
+from qlib.backtest.position import BasePosition, Position
 import random
 import logging
 from typing import List, Tuple, Union, Callable, Iterable
@@ -278,7 +282,7 @@ class Exchange:
         else:
             return True
 
-    def deal_order(self, order, trade_account=None, position=None):
+    def deal_order(self, order, trade_account: Account = None, position: BasePosition = None):
         """
         Deal order when the actual transaction
 
@@ -289,13 +293,12 @@ class Exchange:
         :param position: position to be updated after dealing the order.
         :return: trade_val, trade_cost, trade_price
         """
-        # need to check order first
+        # check order first.
-        # TODO: check the order unit limit in the exchange!!!!
-        # The order limit is related to the adj factor and the cur_amount.
-        # factor = self.quote[(order.stock_id, order.trade_date)]['$factor']
-        # cur_amount = trade_account.current.get_stock_amount(order.stock_id)
         if self.check_order(order) is False:
-            raise AttributeError("need to check order first")
+            order.deal_amount = 0.0
+            # using np.nan instead of None to make it more convenient to should the value in format string
+            return 0.0, 0.0, np.nan
+
         if trade_account is not None and position is not None:
             raise ValueError("trade_account and position can only choose one")
 
@@ -304,14 +307,18 @@ class Exchange:
         trade_val, trade_cost = self._calc_trade_info_by_order(
             order, trade_account.current if trade_account else position
         )
-        # update account
         if order.deal_amount > 1e-5:
-            # If the order can only be deal 0 aomount. Nothing to be updated
+            # If the order can only be deal 0 amount. Nothing to be updated
-            # Otherwise, it will result some stock with 0 amount in the position
+            # Otherwise, it will result in
+            # 1) some stock with 0 amount in the position
+            # 2) `trade_unit` of trade_cost will be lost in user account
             if trade_account:
                 trade_account.update_order(order=order, trade_val=trade_val, cost=trade_cost, trade_price=trade_price)
             elif position:
                 position.update_order(order=order, trade_val=trade_val, cost=trade_cost, trade_price=trade_price)
+        else:
+            # if dealing is not successful, the trade_cost should be zero
+            trade_cost = 0
 
         return trade_val, trade_cost, trade_price
 
@@ -346,7 +353,7 @@ class Exchange:
             `None`: if the stock is suspended `None` may be returned
             `float`: return factor if the factor exists
         """
-        assert (start_time is not None and end_time is not None, "the time range must be given")
+        assert start_time is not None and end_time is not None, "the time range must be given"
         if stock_id not in self.quote.get_all_stock():
             return None
         return self.quote.get_data(stock_id, start_time, end_time, fields="$factor", method=ts_data_last)
@@ -509,7 +516,7 @@ class Exchange:
                 )
         return value
 
-    def _get_factor_or_raise_erorr(self, factor: float = None, stock_id: str = None, start_time=None, end_time=None):
+    def _get_factor_or_raise_error(self, factor: float = None, stock_id: str = None, start_time=None, end_time=None):
         """Please refer to the docs of get_amount_of_trade_unit"""
         if factor is None:
             if stock_id is not None and start_time is not None and end_time is not None:
@@ -537,7 +544,7 @@ class Exchange:
             the end time of trading range
         """
         if not self.trade_w_adj_price and self.trade_unit is not None:
-            factor = self._get_factor_or_raise_erorr(
+            factor = self._get_factor_or_raise_error(
                 factor=factor, stock_id=stock_id, start_time=start_time, end_time=end_time
             )
             return self.trade_unit / factor
@@ -556,7 +563,7 @@ class Exchange:
         """
         if not self.trade_w_adj_price and self.trade_unit is not None:
             # the minimal amount is 1. Add 0.1 for solving precision problem.
-            factor = self._get_factor_or_raise_erorr(
+            factor = self._get_factor_or_raise_error(
                 factor=factor, stock_id=stock_id, start_time=start_time, end_time=end_time
             )
             return (deal_amount * factor + 0.1) // self.trade_unit * self.trade_unit / factor
@@ -626,7 +633,7 @@ class Exchange:
                 order.stock_id, order.start_time, order.end_time, order.deal_amount
             )
             trade_val = order.deal_amount * trade_price
-            trade_cost = trade_val * self.open_cost
+            trade_cost = max(trade_val * self.open_cost, self.min_cost)
         else:
             raise NotImplementedError("order type {} error".format(order.type))
 

qlib/backtest/executor.py

@@ -1,5 +1,6 @@
 from abc import abstractclassmethod, abstractmethod
 import copy
+from qlib.backtest.position import BasePosition
 from qlib.log import get_module_logger
 from types import GeneratorType
 from qlib.backtest.account import Account
@@ -32,6 +33,7 @@ class BaseExecutor:
         track_data: bool = False,
         trade_exchange: Exchange = None,
         common_infra: CommonInfrastructure = None,
+        settle_type=BasePosition.ST_NO,
         **kwargs,
     ):
         """
@@ -95,6 +97,8 @@ class BaseExecutor:
             - trade_exchange : Exchange, optional
                 exchange that provides market info
 
+        settle_type : str
+            Please refer to the docs of BasePosition.settle_start
         """
         self.time_per_step = time_per_step
         self.indicator_config = indicator_config
@@ -104,6 +108,7 @@ class BaseExecutor:
         self._trade_exchange = trade_exchange
         self.level_infra = LevelInfrastructure()
         self.level_infra.reset_infra(common_infra=common_infra)
+        self._settle_type = settle_type
         self.reset(start_time=start_time, end_time=end_time, common_infra=common_infra)
         if common_infra is None:
             get_module_logger("BaseExecutor").warning(f"`common_infra` is not set for {self}")
@@ -235,6 +240,9 @@ class BaseExecutor:
         if atomic and trade_decision.get_range_limit(default_value=None) is not None:
             raise ValueError("atomic executor doesn't support specify `range_limit`")
 
+        if self._settle_type != BasePosition.ST_NO:
+            self.trade_account.current.settle_start(self._settle_type)
+
         obj = self._collect_data(trade_decision=trade_decision, level=level)
 
         if isinstance(obj, GeneratorType):
@@ -256,6 +264,10 @@ class BaseExecutor:
         )
 
         self.trade_calendar.step()
+
+        if self._settle_type != BasePosition.ST_NO:
+            self.trade_account.current.settle_commit()
+
         if return_value is not None:
             return_value.update({"execute_result": res})
         return res
@@ -366,7 +378,7 @@ class NestedExecutor(BaseExecutor):
             trade_decision = self._update_trade_decision(trade_decision)
 
             if trade_decision.empty() and self._skip_empty_decision:
-                # give one chance for outer stategy to update the strategy
+                # give one chance for outer strategy to update the strategy
                 # - For updating some information in the sub executor(the strategy have no knowledge of the inner
                 # executor when generating the decision)
                 break
@@ -409,6 +421,9 @@ class NestedExecutor(BaseExecutor):
 class SimulatorExecutor(BaseExecutor):
     """Executor that simulate the true market"""
 
+    # TODO: TT_SERIAL & TT_PARAL will be replaced by feature fix_pos now.
+    # Please remove them in the future.
+
     # available trade_types
     TT_SERIAL = "serial"
     ## The orders will be executed serially in a sequence
@@ -486,34 +501,22 @@ class SimulatorExecutor(BaseExecutor):
         execute_result = []
 
         for order in self._get_order_iterator(trade_decision):
-            if self.trade_exchange.check_order(order) is True:
+            # execute the order.
-                # execute the order.
+            # NOTE: The trade_account will be changed in this function
-                # NOTE: The trade_account will be changed in this function
+            trade_val, trade_cost, trade_price = self.trade_exchange.deal_order(order, trade_account=self.trade_account)
-                trade_val, trade_cost, trade_price = self.trade_exchange.deal_order(
+            execute_result.append((order, trade_val, trade_cost, trade_price))
-                    order, trade_account=self.trade_account
+            if self.verbose:
-                )
+                print(
-                execute_result.append((order, trade_val, trade_cost, trade_price))
+                    "[I {:%Y-%m-%d %H:%M:%S}]: {} {}, price {:.2f}, amount {}, deal_amount {}, factor {}, value {:.2f}, cash {:.2f}.".format(
-                if self.verbose:
+                        trade_start_time,
-                    if order.direction == Order.SELL:  # sell
+                        "sell" if order.direction == Order.SELL else "buy",
-                        action = "sell"
+                        order.stock_id,
-                    else:
+                        trade_price,
-                        action = "buy"
+                        order.amount,
-                    print(
+                        order.deal_amount,
-                        "[I {:%Y-%m-%d %H:%M:%S}]: {} {}, price {:.2f}, amount {}, deal_amount {}, factor {}, value {:.2f}, cach {:.2f}.".format(
+                        order.factor,
-                            trade_start_time,
+                        trade_val,
-                            action,
+                        self.trade_account.get_cash(),
-                            order.stock_id,
-                            trade_price,
-                            order.amount,
-                            order.deal_amount,
-                            order.factor,
-                            trade_val,
-                            self.trade_account.get_cash(),
-                        )
                     )
-            else:
+                )
-                if self.verbose:
-                    print("[W {:%Y-%m-%d %H:%M:%S}]: {} wrong.".format(trade_start_time, order.stock_id))
-                # do nothing
-                pass
         return execute_result, {"trade_info": execute_result}

qlib/backtest/order.py

@@ -58,12 +58,19 @@ class Order:
     # 3) results
     # - users should not care about these values
     # - they are set by the backtest system after finishing the results.
+    # What the value should be about in all kinds of cases
+    # - not tradable: the deal_amount == 0 , factor is None
+    #    - the stock is suspended and the entire order fails. No cost for this order
+    # - dealed or partially dealed: deal_amount >= 0 and factor is not None
     deal_amount: float = field(init=False)  # `deal_amount` is a non-negative value
     factor: float = field(init=False)
 
+    # TODO:
+    # a status field to indicate the dealing result of the order
+
     # FIXME:
     # for compatible now.
-    # Plese remove them in the future
+    # Please remove them in the future
     SELL: ClassVar[OrderDir] = OrderDir.SELL
     BUY: ClassVar[OrderDir] = OrderDir.BUY
 
@@ -71,6 +78,7 @@ class Order:
         if self.direction not in {Order.SELL, Order.BUY}:
             raise NotImplementedError("direction not supported, `Order.SELL` for sell, `Order.BUY` for buy")
         self.deal_amount = 0
+        self.factor = None
 
     @property
     def amount_delta(self) -> float:

qlib/backtest/position.py

@@ -20,8 +20,8 @@ class BasePosition:
     Please refer to the `Position` class for the position
     """
 
-    def __init__(self, cash=0.0, *args, **kwargs) -> None:
+    def __init__(self, cash=0.0, *args, **kwargs):
-        pass
+        self._settle_type = self.ST_NO
 
     def skip_update(self) -> bool:
         """
@@ -124,13 +124,16 @@ class BasePosition:
         """
         raise NotImplementedError(f"Please implement the `get_stock_amount` method")
 
-    def get_cash(self) -> float:
+    def get_cash(self, include_settle: bool = False) -> float:
         """
 
         Returns
         -------
         float:
-            the cash in position
+            the available(tradable) cash in position
+        include_settle:
+            will the unsettled(delayed) cash included
+            Default: not include those unavailable cash
         """
         raise NotImplementedError(f"Please implement the `get_cash` method")
 
@@ -188,6 +191,37 @@ class BasePosition:
         """
         raise NotImplementedError(f"Please implement the `add_count_all` method")
 
+    ST_CASH = "cash"
+    ST_NO = None
+
+    def settle_start(self, settle_type: str):
+        """
+        settlement start
+        It will act like start and commit a transaction
+
+        Parameters
+        ----------
+        settle_type : str
+            Should we make delay the settlement in each execution (each execution will make the executor a step forward)
+            - "cash": make the cash settlement delayed.
+                - The cash you get can't be used in current step (e.g. you can't sell a stock to get cash to buy another
+                        stock)
+            - None: not settlement mechanism
+            - TODO: other assets will be supported in the future.
+        """
+        raise NotImplementedError(f"Please implement the `settle_conf` method")
+
+    def settle_commit(self):
+        """
+        settlement commit
+
+        Parameters
+        ----------
+        settle_type : str
+            please refer to the documents of Executor
+        """
+        raise NotImplementedError(f"Please implement the `settle_commit` method")
+
 
 class Position(BasePosition):
     """Position
@@ -203,7 +237,7 @@ class Position(BasePosition):
     }
     """
 
-    def __init__(self, start_time, freq, cash: float = 0, position_dict: Dict[str, Dict[str, float]] = {}):
+    def __init__(self, cash: float = 0, position_dict: Dict[str, Dict[str, float]] = {}):
         """Init position by cash and position_dict.
 
         Parameters
@@ -217,11 +251,12 @@ class Position(BasePosition):
             if there is no price key in the dict of stocks, it will be filled by _fill_stock_value.
             by default {}.
         """
+        super().__init__()
 
         # NOTE: The position dict must be copied!!!
         # Otherwise the initial value
         self.init_cash = cash
-        self.position = self._fill_stock_value(position_dict.copy(), start_time, freq)
+        self.position = position_dict.copy()
         self.position["cash"] = cash
         self.position["now_account_value"] = self.calculate_value()
 
@@ -312,7 +347,13 @@ class Position(BasePosition):
             elif abs(self.position[stock_id]["amount"]) <= 1e-5:
                 self._del_stock(stock_id)
 
-        self.position["cash"] += trade_val - cost
+        new_cash = trade_val - cost
+        if self._settle_type == self.ST_CASH:
+            self.position["cash_delay"] += new_cash
+        elif self._settle_type == self.ST_NO:
+            self.position["cash"] += new_cash
+        else:
+            raise NotImplementedError(f"This type of input is not supported")
 
     def _del_stock(self, stock_id):
         del self.position[stock_id]
@@ -340,9 +381,6 @@ class Position(BasePosition):
     def update_stock_weight(self, stock_id, weight):
         self.position[stock_id]["weight"] = weight
 
-    def update_cash(self, cash):
-        self.position["cash"] = cash
-
     def calculate_stock_value(self):
         stock_list = self.get_stock_list()
         value = 0
@@ -352,11 +390,11 @@ class Position(BasePosition):
 
     def calculate_value(self):
         value = self.calculate_stock_value()
-        value += self.position["cash"]
+        value += self.position["cash"] + self.position.get("cash_delay", 0.0)
         return value
 
     def get_stock_list(self):
-        stock_list = list(set(self.position.keys()) - {"cash", "now_account_value"})
+        stock_list = list(set(self.position.keys()) - {"cash", "now_account_value", "cash_delay"})
         return stock_list
 
     def get_stock_price(self, code):
@@ -375,8 +413,11 @@ class Position(BasePosition):
     def get_stock_weight(self, code):
         return self.position[code]["weight"]
 
-    def get_cash(self):
+    def get_cash(self, include_settle=False):
-        return self.position["cash"]
+        cash = self.position["cash"]
+        if include_settle:
+            cash += self.position.get("cash_delay", 0.0)
+        return cash
 
     def get_stock_amount_dict(self):
         """generate stock amount dict {stock_id : amount of stock}"""
@@ -388,7 +429,7 @@ class Position(BasePosition):
 
     def get_stock_weight_dict(self, only_stock=False):
         """get_stock_weight_dict
-        generate stock weight fict {stock_id : value weight of stock in the position}
+        generate stock weight dict {stock_id : value weight of stock in the position}
         it is meaningful in the beginning or the end of each trade date
 
         :param only_stock: If only_stock=True, the weight of each stock in total stock will be returned
@@ -417,49 +458,20 @@ class Position(BasePosition):
         for stock_code, weight in weight_dict.items():
             self.update_stock_weight(stock_code, weight)
 
-    def save_position(self, path):
+    def settle_start(self, settle_type):
-        path = pathlib.Path(path)
+        assert self._settle_type == self.ST_NO, "Currently, settlement can't be nested!!!!!"
-        p = copy.deepcopy(self.position)
+        self._settle_type = settle_type
-        cash = pd.Series(dtype=float)
+        if settle_type == self.ST_CASH:
-        cash["init_cash"] = self.init_cash
+            self.position["cash_delay"] = 0.0
-        cash["cash"] = p["cash"]
-        cash["now_account_value"] = p["now_account_value"]
-        del p["cash"]
-        del p["now_account_value"]
-        positions = pd.DataFrame.from_dict(p, orient="index")
-        with pd.ExcelWriter(path) as writer:
-            positions.to_excel(writer, sheet_name="position")
-            cash.to_excel(writer, sheet_name="info")
 
-    def load_position(self, path):
+    def settle_commit(self):
-        """load position information from a file
+        if self._settle_type != self.ST_NO:
-        should have format below
+            if self._settle_type == self.ST_CASH:
-        sheet "position"
+                self.position["cash"] += self.position["cash_delay"]
-            columns: ['stock', f'count_{bar}', 'amount', 'price', 'weight']
+                del self.position["cash_delay"]
-                f'count_{bar}': <how many bars the security has been hold>,
+            else:
-                'amount': <the amount of the security>,
+                raise NotImplementedError(f"This type of input is not supported")
-                'price': <the close price of security in the last trading day>,
+            self._settle_type = self.ST_NO
-                'weight': <the security weight of total position value>,
-
-        sheet "cash"
-            index: ['init_cash', 'cash', 'now_account_value']
-            'init_cash': <inital cash when account was created>,
-            'cash': <current cash in account>,
-            'now_account_value': <current total account value, should equal to sum(price[stock]*amount[stock])>
-        """
-        path = pathlib.Path(path)
-        positions = pd.read_excel(open(path, "rb"), sheet_name="position", index_col=0)
-        cash_record = pd.read_excel(open(path, "rb"), sheet_name="info", index_col=0)
-        positions = positions.to_dict(orient="index")
-        init_cash = cash_record.loc["init_cash"].values[0]
-        cash = cash_record.loc["cash"].values[0]
-        now_account_value = cash_record.loc["now_account_value"].values[0]
-        # assign values
-        self.position = {}
-        self.init_cash = init_cash
-        self.position = positions
-        self.position["cash"] = cash
-        self.position["now_account_value"] = now_account_value
 
 
 class InfPosition(BasePosition):
@@ -502,7 +514,7 @@ class InfPosition(BasePosition):
     def get_stock_amount(self, code) -> float:
         return np.inf
 
-    def get_cash(self) -> float:
+    def get_cash(self, include_settle=False) -> float:
         return np.inf
 
     def get_stock_amount_dict(self) -> Dict:
@@ -516,3 +528,9 @@ class InfPosition(BasePosition):
 
     def update_weight_all(self):
         raise NotImplementedError(f"InfPosition doesn't support update_weight_all")
+
+    def settle_start(self, settle_type: str):
+        pass
+
+    def settle_commit(self):
+        pass

qlib/contrib/strategy/rule_strategy.py

@@ -18,7 +18,12 @@ from qlib.backtest.utils import get_start_end_idx
 
 
 class TWAPStrategy(BaseStrategy):
-    """TWAP Strategy for trading"""
+    """TWAP Strategy for trading
+
+    NOTE:
+        - This TWAP strategy will celling round when trading. This will make the TWAP trading strategy produce the order
+          ealier when the total trade unit of amount is less than the trading step
+    """
 
     def reset(self, outer_trade_decision: BaseTradeDecision = None, **kwargs):
         """
@@ -583,7 +588,11 @@ class FileOrderStrategy(BaseStrategy):
     """
 
     def __init__(
-        self, file: Union[IO, str, Path], trade_range: Union[Tuple[int, int], TradeRange] = None, *args, **kwargs
+        self,
+        file: Union[IO, str, Path, pd.DataFrame],
+        trade_range: Union[Tuple[int, int], TradeRange] = None,
+        *args,
+        **kwargs,
     ):
         """
 
@@ -611,8 +620,11 @@ class FileOrderStrategy(BaseStrategy):
 
         """
         super().__init__(*args, **kwargs)
-        with get_io_object(file) as f:
+        if isinstance(file, pd.DataFrame):
-            self.order_df = pd.read_csv(f, dtype={"datetime": np.str})
+            self.order_df = file
+        else:
+            with get_io_object(file) as f:
+                self.order_df = pd.read_csv(f, dtype={"datetime": np.str})
 
         self.order_df["datetime"] = self.order_df["datetime"].apply(pd.Timestamp)
         self.order_df = self.order_df.set_index(["datetime", "instrument"])

qlib/utils/exceptions.py

@@ -1,17 +1,20 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
+
 # Base exception class
 class QlibException(Exception):
     def __init__(self, message):
         super(QlibException, self).__init__(message)
 
 
-# Error type for reinitialization when starting an experiment
 class RecorderInitializationError(QlibException):
+    """Error type for re-initialization when starting an experiment"""
+
     pass
 
 
-# Error type for Recorder when can not load object
 class LoadObjectError(QlibException):
+    """Error type for Recorder when can not load object"""
+
     pass