Order execution open source (#1447)

* Waiting for bin data

* Complete readme

* CI

* Add inst filter by time

* Update qlib/data/dataset/processor.py

* typo

* Fix time filter bug

* Add Filter and set Universe

* Complete data pipeline

* Fix Provider Logger Info Args

* Add DQN; a minor bugfix in ppo reward.

* update readme. modify assertion logic in strategy check.

* Fix Doc issues and fix black

* Fix pylint Error

---------

Co-authored-by: Young <afe.young@gmail.com>
Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>

qlib/rl/contrib/backtest.py

@@ -357,7 +357,10 @@ def backtest(backtest_config: dict, with_simulator: bool = False) -> pd.DataFram
 
     if not output_path.exists():
         os.makedirs(output_path)
-    res.to_csv(output_path / "summary.csv")
+
+    if "pa" in res.columns:
+        res["pa"] = res["pa"] * 10000.0  # align with training metrics
+    res.to_csv(output_path / "backtest_result.csv")
     return res
 
 

qlib/rl/order_execution/policy.py

@@ -12,11 +12,11 @@ import torch
 import torch.nn as nn
 from gym.spaces import Discrete
 from tianshou.data import Batch, ReplayBuffer, to_torch
-from tianshou.policy import BasePolicy, PPOPolicy
+from tianshou.policy import BasePolicy, PPOPolicy, DQNPolicy
 
 from qlib.rl.trainer.trainer import Trainer
 
-__all__ = ["AllOne", "PPO"]
+__all__ = ["AllOne", "PPO", "DQN"]
 
 
 # baselines #
@@ -158,6 +158,56 @@ class PPO(PPOPolicy):
             set_weight(self, Trainer.get_policy_state_dict(weight_file))
 
 
+DQNModel = PPOActor  # Reuse PPOActor.
+
+
+class DQN(DQNPolicy):
+    """A wrapper of tianshou DQNPolicy.
+
+    Differences:
+
+    - Auto-create model network. Supports discrete action space only.
+    - Support a ``weight_file`` that supports loading checkpoint.
+    """
+
+    def __init__(
+        self,
+        network: nn.Module,
+        obs_space: gym.Space,
+        action_space: gym.Space,
+        lr: float,
+        weight_decay: float = 0.0,
+        discount_factor: float = 0.99,
+        estimation_step: int = 1,
+        target_update_freq: int = 0,
+        reward_normalization: bool = False,
+        is_double: bool = True,
+        clip_loss_grad: bool = False,
+        weight_file: Optional[Path] = None,
+    ) -> None:
+        assert isinstance(action_space, Discrete)
+
+        model = DQNModel(network, action_space.n)
+        optimizer = torch.optim.Adam(
+            model.parameters(),
+            lr=lr,
+            weight_decay=weight_decay,
+        )
+
+        super().__init__(
+            model,
+            optimizer,
+            discount_factor=discount_factor,
+            estimation_step=estimation_step,
+            target_update_freq=target_update_freq,
+            reward_normalization=reward_normalization,
+            is_double=is_double,
+            clip_loss_grad=clip_loss_grad,
+        )
+        if weight_file is not None:
+            set_weight(self, Trainer.get_policy_state_dict(weight_file))
+
+
 # utilities: these should be put in a separate (common) file. #
 
 

qlib/rl/order_execution/reward.py

@@ -70,7 +70,19 @@ class PPOReward(Reward[SAOEState]):
 
     def reward(self, simulator_state: SAOEState) -> float:
         if simulator_state.cur_step == self.max_step - 1 or simulator_state.position < 1e-6:
-            vwap_price = cast(dict, simulator_state.metrics)["trade_price"]
+            if simulator_state.history_exec["deal_amount"].sum() == 0.0:
+                vwap_price = cast(
+                    float,
+                    np.average(simulator_state.history_exec["market_price"]),
+                )
+            else:
+                vwap_price = cast(
+                    float,
+                    np.average(
+                        simulator_state.history_exec["market_price"],
+                        weights=simulator_state.history_exec["deal_amount"],
+                    ),
+                )
             twap_price = simulator_state.backtest_data.get_deal_price().mean()
 
             if simulator_state.order.direction == OrderDir.SELL:

qlib/rl/order_execution/strategy.py

@@ -7,6 +7,7 @@ import collections
 from types import GeneratorType
 from typing import Any, Callable, cast, Dict, Generator, List, Optional, Tuple, Union
 
+import warnings
 import numpy as np
 import pandas as pd
 import torch
@@ -137,7 +138,12 @@ class SAOEStateAdapter:
             exec_vol[idx - last_step_range[0]] = order.deal_amount
 
         if exec_vol.sum() > self.position and exec_vol.sum() > 0.0:
-            assert exec_vol.sum() < self.position + 1, f"{exec_vol} too large"
+            if exec_vol.sum() > self.position + 1.0:
+                warnings.warn(
+                    f"Sum of execution volume is {exec_vol.sum()} which is larger than "
+                    f"position + 1.0 = {self.position} + 1.0 = {self.position + 1.0}. "
+                    f"All execution volume is scaled down linearly to ensure that their sum does not position."
+                )
             exec_vol *= self.position / (exec_vol.sum())
 
         market_volume = cast(