diff --git a/qlib/backtest/exchange.py b/qlib/backtest/exchange.py index a759dbd86..8177d53ee 100644 --- a/qlib/backtest/exchange.py +++ b/qlib/backtest/exchange.py @@ -14,7 +14,7 @@ from ..data.dataset.utils import get_level_index from ..config import C, REG_CN from ..utils.resam import resam_ts_data from ..log import get_module_logger -from .order import Order +from .order import Order, OrderDir, OrderHelper class Exchange: @@ -526,3 +526,9 @@ class Exchange: raise NotImplementedError("order type {} error".format(order.type)) return trade_val, trade_cost + + def get_order_helper(self) -> OrderHelper: + if not hasattr(self, "_order_helper"): + # cache to avoid recreate the same instance + self._order_helper = OrderHelper(self) + return self._order_helper diff --git a/qlib/backtest/order.py b/qlib/backtest/order.py index 19ea807c1..9df162263 100644 --- a/qlib/backtest/order.py +++ b/qlib/backtest/order.py @@ -2,12 +2,14 @@ # Licensed under the MIT License. # TODO: rename it with decision.py from __future__ import annotations +from enum import IntEnum # try to fix circular imports when enabling type hints from typing import TYPE_CHECKING if TYPE_CHECKING: from qlib.strategy.base import BaseStrategy + from qlib.backtest.exchange import Exchange from qlib.backtest.utils import TradeCalendarManager import warnings import pandas as pd @@ -15,6 +17,12 @@ from dataclasses import dataclass, field from typing import ClassVar, Union, List, Set, Tuple +class OrderDir(IntEnum): + # Order direction + SELL = 0 + BUY = 1 + + @dataclass class Order: """ @@ -32,19 +40,97 @@ class Order: stock_id: str amount: float + + # The interval of the order which belongs to (NOTE: this is not the expected order dealing range time) start_time: pd.Timestamp end_time: pd.Timestamp + direction: int factor: float deal_amount: float = field(init=False) - SELL: ClassVar[int] = 0 - BUY: ClassVar[int] = 1 + + # FIXME: + # for compatible now. + # Plese remove them in the future + SELL: ClassVar[OrderDir] = OrderDir.SELL + BUY: ClassVar[OrderDir] = OrderDir.BUY def __post_init__(self): if self.direction not in {Order.SELL, Order.BUY}: raise NotImplementedError("direction not supported, `Order.SELL` for sell, `Order.BUY` for buy") self.deal_amount = 0 + @staticmethod + def parse_dir(direction: Union[str, int, OrderDir]) -> OrderDir: + if isinstance(direction, OrderDir): + return direction + elif isinstance(direction, int): + return OrderDir(direction) + elif isinstance(direction, str): + dl = direction.lower() + if dl.strip() == "sell": + return OrderDir.SELL + elif dl.strip() == "buy": + return OrderDir.BUY + else: + raise NotImplementedError(f"This type of input is not supported") + else: + raise NotImplementedError(f"This type of input is not supported") + + +class OrderHelper: + """ + Motivation + - Make generating order easier + - User may have no knowledge about the adjust-factor information about the system. + - It involves to much interaction with the exchange when generating orders. + """ + + def __init__(self, exchange: Exchange): + self.exchange = exchange + + def create( + self, + code: str, + amount: float, + direction: OrderDir, + start_time: Union[str, pd.Timestamp], + end_time: Union[str, pd.Timestamp], + ) -> Order: + """ + help to create a order + + # TODO: create order for unadjusted amount order + + Parameters + ---------- + code : str + the id of the instrument + amount : float + **adjusted trading amount** + direction : OrderDir + trading direction + start_time : Union[str, pd.Timestamp] + The interval of the order which belongs to + end_time : Union[str, pd.Timestamp] + The interval of the order which belongs to + + Returns + ------- + Order: + The created order + """ + start_time = pd.Timestamp(start_time) + end_time = pd.Timestamp(end_time) + return Order( + stock_id=code, + amount=amount, + start_time=start_time, + end_time=end_time, + direction=direction, + factor=self.exchange.get_factor(code, start_time, end_time), + ) + class BaseTradeDecision: """ diff --git a/qlib/contrib/strategy/rule_strategy.py b/qlib/contrib/strategy/rule_strategy.py index 20099d4d3..22483a79c 100644 --- a/qlib/contrib/strategy/rule_strategy.py +++ b/qlib/contrib/strategy/rule_strategy.py @@ -1,14 +1,19 @@ +from pathlib import Path import warnings import numpy as np import pandas as pd -from typing import List, Tuple, Union +from typing import IO, List, Tuple, Union +from qlib.data.dataset.utils import convert_index_format + +from qlib.utils import lazy_sort_index from ...utils.resam import resam_ts_data from ...data.data import D from ...strategy.base import BaseStrategy from ...backtest.order import BaseTradeDecision, Order, TradeDecisionWO -from ...backtest.exchange import Exchange +from ...backtest.exchange import Exchange, OrderHelper from ...backtest.utils import CommonInfrastructure, LevelInfrastructure +from qlib.utils.file import get_io_object def get_start_end_idx(strategy: BaseStrategy, outer_trade_decision: BaseTradeDecision) -> Union[int, int]: @@ -653,6 +658,9 @@ class RandomOrderStrategy(BaseStrategy): index_range : Tuple the intra day time index range of the orders the left and right is closed. + + If you want to get the index_range in intra-day + - `qlib/utils/time.py:def get_day_min_idx_range` can help you create the index range easier # TODO: this is a index_range level limitation. We'll implement a more detailed limitation later. sample_ratio : float the ratio of all orders are sampled @@ -684,7 +692,9 @@ class RandomOrderStrategy(BaseStrategy): if step_time_start in self.volume_df: for stock_id, volume in self.volume_df[step_time_start].dropna().sample(frac=self.sample_ratio).items(): order_list.append( - self.common_infra.get("trade_exchange").create_order( + self.common_infra.get("trade_exchange") + .get_order_helper() + .create( code=stock_id, amount=volume * self.volume_ratio, start_time=step_time_start, @@ -693,3 +703,53 @@ class RandomOrderStrategy(BaseStrategy): ) ) return TradeDecisionWO(order_list, self, self.index_range) + + +class FileOrderStrategy(BaseStrategy): + """ + Motivtaion: + - This class provides an interface for user to read orders from csv files. + - It is supposed to be used in + """ + + def __init__(self, file: Union[IO, str, Path], index_range: Tuple[int, int] = None, *args, **kwargs): + super().__init__(*args, **kwargs) + with get_io_object(file) as f: + self.order_df = pd.read_csv(f, dtype={"datetime": np.str}) + + self.order_df["datetime"] = self.order_df["datetime"].apply(pd.Timestamp) + self.order_df = self.order_df.set_index(["datetime", "instrument"]) + + # make sure the datetime is the first level for fast indexing + self.order_df = lazy_sort_index(convert_index_format(self.order_df, level="datetime")) + self.index_range = index_range + + def generate_trade_decision(self, execute_result=None) -> TradeDecisionWO: + """ + Parameters + ---------- + execute_result : + execute_result will be ignored in FileOrderStrategy + """ + oh: OrderHelper = self.common_infra.get("trade_exchange").get_order_helper() + tc = self.trade_calendar + step = tc.get_trade_step() + start, end = tc.get_step_time(step) + # CONVERSION: the bar is indexed by the time + try: + df = self.order_df.loc(axis=0)[start] + except KeyError: + return TradeDecisionWO([], self) + else: + order_list = [] + for idx, row in df.iterrows(): + order_list.append( + oh.create( + code=idx, + amount=row["amount"], + direction=Order.parse_dir(row["direction"]), + start_time=start, + end_time=end, + ) + ) + return TradeDecisionWO(order_list, self, self.index_range) diff --git a/qlib/data/dataset/utils.py b/qlib/data/dataset/utils.py index 3cb4dd3e2..c6b3d97b6 100644 --- a/qlib/data/dataset/utils.py +++ b/qlib/data/dataset/utils.py @@ -92,6 +92,9 @@ def convert_index_format(df: Union[pd.DataFrame, pd.Series], level: str = "datet - If `level` is the first level of df.MultiIndex, do nothing - If `level` is the second level of df.MultiIndex, swap the level of index. + NOTE: + the number of levels of df.MultiIndex should be 2 + Parameters ---------- df : Union[pd.DataFrame, pd.Series] diff --git a/qlib/utils/file.py b/qlib/utils/file.py new file mode 100644 index 000000000..611260c86 --- /dev/null +++ b/qlib/utils/file.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# TODO: move file related utils into this module +import contextlib +from typing import IO, Union +from pathlib import Path + + +@contextlib.contextmanager +def get_io_object(file: Union[IO, str, Path], *args, **kwargs) -> IO: + """ + providing a easy interface to get an IO object + + Parameters + ---------- + file : Union[IO, str, Path] + a object representing the file + + Returns + ------- + IO: + a IO-like object + + Raises + ------ + NotImplementedError: + """ + if isinstance(file, IO): + yield file + else: + if isinstance(file, str): + file = Path(file) + if not isinstance(file, Path): + raise NotImplementedError(f"This type[{type(file)}] of input is not supported") + with file.open(*args, **kwargs) as f: + yield f diff --git a/tests/backtest/test_file_strategy.py b/tests/backtest/test_file_strategy.py new file mode 100644 index 000000000..da52b0d53 --- /dev/null +++ b/tests/backtest/test_file_strategy.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import unittest +from qlib.backtest import backtest, order +from qlib.tests import TestAutoData +import pandas as pd +from pathlib import Path + +DIRNAME = Path(__file__).absolute().resolve().parent + + +class FileStrTest(TestAutoData): + + TEST_INST = "SH600519" + + EXAMPLE_FILE = DIRNAME / "order_example.csv" + + def _gen_orders(self) -> pd.DataFrame: + headers = [ + "datetime", + "instrument", + "amount", + "direction", + ] + orders = [ + ["20200102", self.TEST_INST, "1000", "sell"], + ["20200103", self.TEST_INST, "1000", "buy"], + ["20200106", self.TEST_INST, "1000", "sell"], + ] + return pd.DataFrame(orders, columns=headers).set_index(["datetime", "instrument"]) + + def test_file_str(self): + + orders = self._gen_orders() + print(orders) + orders.to_csv(self.EXAMPLE_FILE) + + orders = pd.read_csv(self.EXAMPLE_FILE, index_col=["datetime", "instrument"]) + + strategy_config = { + "class": "FileOrderStrategy", + "module_path": "qlib.contrib.strategy.rule_strategy", + "kwargs": {"file": self.EXAMPLE_FILE}, + } + + freq = "day" + start_time = "2020-01-01" + end_time = "2020-01-16" + codes = [self.TEST_INST] + + backtest_config = { + "start_time": start_time, + "end_time": end_time, + "account": 100000000, + "benchmark": None, # benchmark is not required here for trading + "exchange_kwargs": { + "freq": freq, + "limit_threshold": 0.095, + "deal_price": "close", + "open_cost": 0.0005, + "close_cost": 0.0015, + "min_cost": 5, + "codes": codes, + }, + # "pos_type": "InfPosition" # Position with infinitive position + } + executor_config = { + "class": "SimulatorExecutor", + "module_path": "qlib.backtest.executor", + "kwargs": { + "time_per_step": freq, + "generate_report": False, + "verbose": True, + "indicator_config": { + "show_indicator": False, + }, + }, + } + backtest(executor=executor_config, strategy=strategy_config, **backtest_config) + + self.EXAMPLE_FILE.unlink() + + +if __name__ == "__main__": + unittest.main()