Refine RL todos (#1332)

* Refine several todos

* CI issues

* Remove Dropna limitation of `quote_df` in Exchange  (#1334)

* Remove Dropna limitation of `quote_df` of Exchange

* Impreove docstring

* Fix type error when expression is specified (#1335)

* Refine fill_missing_data()

* Remove several TODO comments

* Add back env for interpreters

* Change Literal import

* Resolve PR comments

* Move  to SAOEState

* Add Trainer.get_policy_state_dict()

* Mypy issue

Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>

tests/backtest/test_file_strategy.py

@@ -107,7 +107,7 @@ class FileStrTest(TestAutoData):
         )
 
         # ffr valid
-        ffr_dict = indicator_dict["1day"]["ffr"].to_dict()
+        ffr_dict = indicator_dict["1day"][0]["ffr"].to_dict()
         ffr_dict = {str(date).split()[0]: ffr_dict[date] for date in ffr_dict}
         assert np.isclose(ffr_dict["2020-01-03"], dealt_num_for_1000 / 1000)
         assert np.isclose(ffr_dict["2020-01-06"], 0)

tests/backtest/test_high_freq_trading.py

@@ -125,7 +125,7 @@ class TestHFBacktest(TestAutoData):
         # NOTE: please refer to the docs of format_decisions
         # NOTE: `"track_data": True,`  is very NECESSARY for collecting the decision!!!!!
         f_dec = format_decisions(decisions)
-        print(indicator["1day"])
+        print(indicator["1day"][0])
 
 
 if __name__ == "__main__":

tests/rl/test_qlib_simulator.py

@@ -7,11 +7,11 @@ from typing import Tuple
 
 import pandas as pd
 import pytest
-from qlib.backtest.decision import Order, OrderDir, TradeRangeByTime
+
+from qlib.backtest.decision import Order, OrderDir
 from qlib.backtest.executor import SimulatorExecutor
 from qlib.rl.order_execution import CategoricalActionInterpreter
 from qlib.rl.order_execution.simulator_qlib import SingleAssetOrderExecution
-from qlib.rl.utils.env_wrapper import CollectDataEnvWrapper
 
 TOTAL_POSITION = 2100.0
 
@@ -183,8 +183,6 @@ def test_interpreter() -> None:
     order = get_order()
     simulator = get_simulator(order)
     interpreter_action = CategoricalActionInterpreter(values=NUM_EXECUTION)
-    interpreter_action.env = CollectDataEnvWrapper()
-    interpreter_action.env.reset()
 
     NUM_STEPS = 7
     state = simulator.get_state()

tests/rl/test_saoe_simple.py

@@ -20,7 +20,6 @@ from qlib.rl.data.pickle_styled import PickleProcessedDataProvider
 from qlib.rl.order_execution import *
 from qlib.rl.trainer import backtest, train
 from qlib.rl.utils import ConsoleWriter, CsvWriter, EnvWrapperStatus
-from qlib.rl.utils.env_wrapper import CollectDataEnvWrapper
 
 pytestmark = pytest.mark.skipif(sys.version_info < (3, 8), reason="Pickle styled data only supports Python >= 3.8")
 
@@ -186,10 +185,6 @@ def test_interpreter():
     assert np.sum(obs["data_processed"][60:]) == 0
 
     # second step: action
-    interpreter_action.env = CollectDataEnvWrapper()
-    interpreter_action_twap.env = CollectDataEnvWrapper()
-    interpreter_action.env.reset()
-    interpreter_action_twap.env.reset()
     action = interpreter_action(simulator.get_state(), 1)
     assert action == 15 / 20
 
@@ -260,8 +255,6 @@ def test_twap_strategy(finite_env_type):
 
     state_interp = FullHistoryStateInterpreter(13, 390, 5, PickleProcessedDataProvider(FEATURE_DATA_DIR))
     action_interp = TwapRelativeActionInterpreter()
-    action_interp.env = CollectDataEnvWrapper()
-    action_interp.env.reset()
     policy = AllOne(state_interp.observation_space, action_interp.action_space)
     csv_writer = CsvWriter(Path(__file__).parent / ".output")
 
@@ -291,8 +284,6 @@ def test_cn_ppo_strategy():
 
     state_interp = FullHistoryStateInterpreter(8, 240, 6, PickleProcessedDataProvider(CN_FEATURE_DATA_DIR))
     action_interp = CategoricalActionInterpreter(4)
-    action_interp.env = CollectDataEnvWrapper()
-    action_interp.env.reset()
     network = Recurrent(state_interp.observation_space)
     policy = PPO(network, state_interp.observation_space, action_interp.action_space, 1e-4)
     policy.load_state_dict(torch.load(CN_POLICY_WEIGHTS_DIR / "ppo_recurrent_30min.pth", map_location="cpu"))
@@ -324,8 +315,6 @@ def test_ppo_train():
 
     state_interp = FullHistoryStateInterpreter(8, 240, 6, PickleProcessedDataProvider(CN_FEATURE_DATA_DIR))
     action_interp = CategoricalActionInterpreter(4)
-    action_interp.env = CollectDataEnvWrapper()
-    action_interp.env.reset()
     network = Recurrent(state_interp.observation_space)
     policy = PPO(network, state_interp.observation_space, action_interp.action_space, 1e-4)