init commit

qlib/contrib/online/executor.py

@@ -0,0 +1,291 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+
+import re
+import json
+import copy
+import pathlib
+import pandas as pd
+from ...data import D
+from ...utils import get_date_in_file_name
+from ...utils import get_pre_trading_date
+from ..backtest.order import Order
+
+
+class BaseExecutor:
+    """
+    # Strategy framework document
+
+    class Executor(BaseExecutor):
+    """
+
+    def execute(self, trade_account, order_list, trade_date):
+        """
+        return the executed result (trade_info) after trading at trade_date.
+        NOTICE: trade_account will not be modified after executing.
+            Parameter
+            ---------
+                trade_account : Account()
+                order_list : list
+                    [Order()]
+                trade_date : pd.Timestamp
+            Return
+            ---------
+            trade_info : list
+                    [Order(), float, float, float]
+        """
+        raise NotImplementedError("get_execute_result for this model is not implemented.")
+
+    def save_executed_file_from_trade_info(self, trade_info, user_path, trade_date):
+        """
+        Save the trade_info to the .csv transaction file in disk
+        the columns of result file is
+        ['date', 'stock_id', 'direction', 'trade_val', 'trade_cost', 'trade_price', 'factor']
+            Parameter
+            ---------
+                trade_info : list of [Order(), float, float, float]
+                    (order, trade_val, trade_cost, trade_price), trade_info with out factor
+                user_path: str / pathlib.Path()
+                    the sub folder to save user data
+
+                transaction_path : string / pathlib.Path()
+        """
+        YYYY, MM, DD = str(trade_date.date()).split("-")
+        folder_path = pathlib.Path(user_path) / "trade" / YYYY / MM
+        if not folder_path.exists():
+            folder_path.mkdir(parents=True)
+        transaction_path = folder_path / "transaction_{}.csv".format(str(trade_date.date()))
+        columns = [
+            "date",
+            "stock_id",
+            "direction",
+            "amount",
+            "trade_val",
+            "trade_cost",
+            "trade_price",
+            "factor",
+        ]
+        data = []
+        for [order, trade_val, trade_cost, trade_price] in trade_info:
+            data.append(
+                [
+                    trade_date,
+                    order.stock_id,
+                    order.direction,
+                    order.amount,
+                    trade_val,
+                    trade_cost,
+                    trade_price,
+                    order.factor,
+                ]
+            )
+        df = pd.DataFrame(data, columns=columns)
+        df.to_csv(transaction_path, index=False)
+
+    def load_trade_info_from_executed_file(self, user_path, trade_date):
+        YYYY, MM, DD = str(trade_date.date()).split("-")
+        file_path = pathlib.Path(user_path) / "trade" / YYYY / MM / "transaction_{}.csv".format(str(trade_date.date()))
+        if not file_path.exists():
+            raise ValueError("File {} not exists!".format(file_path))
+
+        filedate = get_date_in_file_name(file_path)
+        transaction = pd.read_csv(file_path)
+        trade_info = []
+        for i in range(len(transaction)):
+            date = transaction.loc[i]["date"]
+            if not date == filedate:
+                continue
+                # raise ValueError("date in transaction file {} not equal to it's file date{}".format(date, filedate))
+            order = Order(
+                stock_id=transaction.loc[i]["stock_id"],
+                amount=transaction.loc[i]["amount"],
+                trade_date=transaction.loc[i]["date"],
+                direction=transaction.loc[i]["direction"],
+                factor=transaction.loc[i]["factor"],
+            )
+            trade_val = transaction.loc[i]["trade_val"]
+            trade_cost = transaction.loc[i]["trade_cost"]
+            trade_price = transaction.loc[i]["trade_price"]
+            trade_info.append([order, trade_val, trade_cost, trade_price])
+        return trade_info
+
+
+class SimulatorExecutor(BaseExecutor):
+    def __init__(self, trade_exchange, verbose=False):
+        self.trade_exchange = trade_exchange
+        self.verbose = verbose
+        self.order_list = []
+
+    def execute(self, trade_account, order_list, trade_date):
+        """
+        execute the order list, do the trading wil exchange at date.
+        Will not modify the trade_account.
+            Parameter
+                trade_account : Account()
+                order_list : list
+                    list or orders
+                trade_date : pd.Timestamp
+            :return:
+                trade_info : list of [Order(), float, float, float]
+                    (order, trade_val, trade_cost, trade_price), trade_info with out factor
+        """
+        account = copy.deepcopy(trade_account)
+        trade_info = []
+
+        for order in order_list:
+            # check holding thresh is done in strategy
+            # if order.direction==0: # sell order
+            #     # checking holding thresh limit for sell order
+            #     if trade_account.current.get_stock_count(order.stock_id) < thresh:
+            #         # can not sell this code
+            #         continue
+            # is order executable
+            # check order
+            if self.trade_exchange.check_order(order) is True:
+                # execute the order
+                trade_val, trade_cost, trade_price = self.trade_exchange.deal_order(order, trade_account=account)
+                trade_info.append([order, trade_val, trade_cost, trade_price])
+                if self.verbose:
+                    if order.direction == Order.SELL:  # sell
+                        print(
+                            "[I {:%Y-%m-%d}]: sell {}, price {:.2f}, amount {}, value {:.2f}.".format(
+                                trade_date,
+                                order.stock_id,
+                                trade_price,
+                                order.deal_amount,
+                                trade_val,
+                            )
+                        )
+                    else:
+                        print(
+                            "[I {:%Y-%m-%d}]: buy {}, price {:.2f}, amount {}, value {:.2f}.".format(
+                                trade_date,
+                                order.stock_id,
+                                trade_price,
+                                order.deal_amount,
+                                trade_val,
+                            )
+                        )
+
+            else:
+                if self.verbose:
+                    print("[W {:%Y-%m-%d}]: {} wrong.".format(trade_date, order.stock_id))
+                # do nothing
+                pass
+        return trade_info
+
+
+def save_score_series(score_series, user_path, trade_date):
+    """Save the score_series into a .csv file.
+    The columns of saved file is
+        [stock_id, score]
+
+    Parameter
+    ---------
+        order_list: [Order()]
+            list of Order()
+        date: pd.Timestamp
+            the date to save the order list
+        user_path: str / pathlib.Path()
+            the sub folder to save user data
+    """
+    user_path = pathlib.Path(user_path)
+    YYYY, MM, DD = str(trade_date.date()).split("-")
+    folder_path = user_path / "score" / YYYY / MM
+    if not folder_path.exists():
+        folder_path.mkdir(parents=True)
+    file_path = folder_path / "score_{}.csv".format(str(trade_date.date()))
+    score_series.to_csv(file_path)
+
+
+def load_score_series(user_path, trade_date):
+    """Save the score_series into a .csv file.
+    The columns of saved file is
+        [stock_id, score]
+
+    Parameter
+    ---------
+        order_list: [Order()]
+            list of Order()
+        date: pd.Timestamp
+            the date to save the order list
+        user_path: str / pathlib.Path()
+            the sub folder to save user data
+    """
+    user_path = pathlib.Path(user_path)
+    YYYY, MM, DD = str(trade_date.date()).split("-")
+    folder_path = user_path / "score" / YYYY / MM
+    if not folder_path.exists():
+        folder_path.mkdir(parents=True)
+    file_path = folder_path / "score_{}.csv".format(str(trade_date.date()))
+    score_series = pd.read_csv(file_path, index_col=0, header=None, names=["instrument", "score"])
+    return score_series
+
+
+def save_order_list(order_list, user_path, trade_date):
+    """
+    Save the order list into a json file.
+    Will calculate the real amount in order according to factors at date.
+
+    The format in json file like
+    {"sell": {"stock_id": amount, ...}
+    ,"buy": {"stock_id": amount, ...}}
+
+        :param
+            order_list: [Order()]
+                list of Order()
+            date: pd.Timestamp
+                the date to save the order list
+            user_path: str / pathlib.Path()
+                the sub folder to save user data
+    """
+    user_path = pathlib.Path(user_path)
+    YYYY, MM, DD = str(trade_date.date()).split("-")
+    folder_path = user_path / "trade" / YYYY / MM
+    if not folder_path.exists():
+        folder_path.mkdir(parents=True)
+    sell = {}
+    buy = {}
+    for order in order_list:
+        if order.direction == 0:  # sell
+            sell[order.stock_id] = [order.amount, order.factor]
+        else:
+            buy[order.stock_id] = [order.amount, order.factor]
+    order_dict = {"sell": sell, "buy": buy}
+    file_path = folder_path / "orderlist_{}.json".format(str(trade_date.date()))
+    with file_path.open("w") as fp:
+        json.dump(order_dict, fp)
+
+
+def load_order_list(user_path, trade_date):
+    user_path = pathlib.Path(user_path)
+    YYYY, MM, DD = str(trade_date.date()).split("-")
+    path = user_path / "trade" / YYYY / MM / "orderlist_{}.json".format(str(trade_date.date()))
+    if not path.exists():
+        raise ValueError("File {} not exists!".format(path))
+    # get orders
+    with path.open("r") as fp:
+        order_dict = json.load(fp)
+    order_list = []
+    for stock_id in order_dict["sell"]:
+        amount, factor = order_dict["sell"][stock_id]
+        order = Order(
+            stock_id=stock_id,
+            amount=amount,
+            trade_date=pd.Timestamp(trade_date),
+            direction=Order.SELL,
+            factor=factor,
+        )
+        order_list.append(order)
+    for stock_id in order_dict["buy"]:
+        amount, factor = order_dict["buy"][stock_id]
+        order = Order(
+            stock_id=stock_id,
+            amount=amount,
+            trade_date=pd.Timestamp(trade_date),
+            direction=Order.BUY,
+            factor=factor,
+        )
+        order_list.append(order)
+    return order_list

qlib/contrib/online/manager.py

@@ -0,0 +1,147 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import os
+import pickle
+import yaml
+import pathlib
+import pandas as pd
+import shutil
+from ..backtest.account import Account
+from ..backtest.exchange import Exchange
+from .user import User
+from .utils import load_instance
+from .utils import save_instance, init_instance_by_config
+
+
+class UserManager:
+    def __init__(self, user_data_path, save_report=True):
+        """
+        This module is designed to manager the users in online system
+        all users' data were assumed to be saved in user_data_path
+            Parameter
+                user_data_path : string
+                    data path that all users' data were saved in
+
+        variables:
+            data_path : string
+                data path that all users' data were saved in
+            users_file : string
+                A path of the file record the add_date of users
+            save_report : bool
+                whether to save report after each trading process
+            users : dict{}
+                [user_id]->User()
+                the python dict save instances of User() for each user_id
+            user_record : pd.Dataframe
+                user_id(string), add_date(string)
+                indicate the add_date for each users
+        """
+        self.data_path = pathlib.Path(user_data_path)
+        self.users_file = self.data_path / "users.csv"
+        self.save_report = save_report
+        self.users = {}
+        self.user_record = None
+
+    def load_users(self):
+        """
+        load all users' data into manager
+        """
+        self.users = {}
+        self.user_record = pd.read_csv(self.users_file, index_col=0)
+        for user_id in self.user_record.index:
+            self.users[user_id] = self.load_user(user_id)
+
+    def load_user(self, user_id):
+        """
+        return a instance of User() represents a user to be processed
+            Parameter
+                user_id : string
+            :return
+                user : User()
+        """
+        account_path = self.data_path / user_id
+        strategy_file = self.data_path / user_id / "strategy_{}.pickle".format(user_id)
+        model_file = self.data_path / user_id / "model_{}.pickle".format(user_id)
+        cur_user_list = [user_id for user_id in self.users]
+        if user_id in cur_user_list:
+            raise ValueError("User {} has been loaded".format(user_id))
+        else:
+            trade_account = Account(0)
+            trade_account.load_account(account_path)
+            strategy = load_instance(strategy_file)
+            model = load_instance(model_file)
+            user = User(account=trade_account, strategy=strategy, model=model)
+            return user
+
+    def save_user_data(self, user_id):
+        """
+        save a instance of User() to user data path
+            Parameter
+                user_id : string
+        """
+        if not user_id in self.users:
+            raise ValueError("Cannot find user {}".format(user_id))
+        self.users[user_id].account.save_account(self.data_path / user_id)
+        save_instance(
+            self.users[user_id].strategy,
+            self.data_path / user_id / "strategy_{}.pickle".format(user_id),
+        )
+        save_instance(
+            self.users[user_id].model,
+            self.data_path / user_id / "model_{}.pickle".format(user_id),
+        )
+
+    def add_user(self, user_id, config_file, add_date):
+        """
+        add the new user {user_id} into user data
+        will create a new folder named "{user_id}" in user data path
+            Parameter
+                user_id : string
+                init_cash : int
+                config_file : str/pathlib.Path()
+                   path of config file
+        """
+        config_file = pathlib.Path(config_file)
+        if not config_file.exists():
+            raise ValueError("Cannot find config file {}".format(config_file))
+        user_path = self.data_path / user_id
+        if user_path.exists():
+            raise ValueError("User data for {} already exists".format(user_id))
+
+        with config_file.open("r") as fp:
+            config = yaml.load(fp)
+        # load model
+        model = init_instance_by_config(config["model"])
+
+        # load strategy
+        strategy = init_instance_by_config(config["strategy"])
+        init_args = strategy.get_init_args_from_model(model, add_date)
+        strategy.init(**init_args)
+
+        # init Account
+        trade_account = Account(init_cash=config["init_cash"])
+
+        # save user
+        user_path.mkdir()
+        save_instance(model, self.data_path / user_id / "model_{}.pickle".format(user_id))
+        save_instance(strategy, self.data_path / user_id / "strategy_{}.pickle".format(user_id))
+        trade_account.save_account(self.data_path / user_id)
+        user_record = pd.read_csv(self.users_file, index_col=0)
+        user_record.loc[user_id] = [add_date]
+        user_record.to_csv(self.users_file)
+
+    def remove_user(self, user_id):
+        """
+        remove user {user_id} in current user dataset
+        will delete the folder "{user_id}" in user data path
+            :param
+                user_id : string
+        """
+        user_path = self.data_path / user_id
+        if not user_path.exists():
+            raise ValueError("Cannot find user data {}".format(user_id))
+        shutil.rmtree(user_path)
+        user_record = pd.read_csv(self.users_file, index_col=0)
+        user_record.drop([user_id], inplace=True)
+        user_record.to_csv(self.users_file)

qlib/contrib/online/online_model.py

@@ -0,0 +1,36 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import random
+import pandas as pd
+from ...data import D
+from ..model.base import Model
+
+
+class ScoreFileModel(Model):
+    """
+    This model will load a score file, and return score at date exists in score file.
+    """
+
+    def __init__(self, score_path):
+        pred_test = pd.read_csv(score_path, index_col=[0, 1], parse_dates=True, infer_datetime_format=True)
+        self.pred = pred_test
+
+    def get_data_with_date(self, date, **kwargs):
+        score = self.pred.loc(axis=0)[:, date]  # (stock_id, trade_date) multi_index, score in pdate
+        score_series = score.reset_index(level="datetime", drop=True)[
+            "score"
+        ]  # pd.Series ; index:stock_id, data: score
+        return score_series
+
+    def predict(self, x_test, **kwargs):
+        return x_test
+
+    def score(self, x_test, **kwargs):
+        return
+
+    def fit(self, x_train, y_train, x_valid, y_valid, w_train=None, w_valid=None, **kwargs):
+        return
+
+    def save(self, fname, **kwargs):
+        return

qlib/contrib/online/operator.py

@@ -0,0 +1,317 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import fire
+import pandas as pd
+import pathlib
+import qlib
+import logging
+
+from ...data import D
+from ...log import get_module_logger
+from ...utils import get_pre_trading_date, is_tradable_date
+from ..evaluate import risk_analysis
+from ..backtest.backtest import update_account
+
+from .manager import UserManager
+from .utils import prepare
+from .utils import create_user_folder
+from .executor import load_order_list, save_order_list
+from .executor import SimulatorExecutor
+from .executor import save_score_series, load_score_series
+
+
+class Operator(object):
+    def __init__(self, client: str):
+        """
+        Parameters
+        ----------
+            client: str
+                The qlib client config file(.yaml)
+        """
+        self.logger = get_module_logger("online operator", level=logging.INFO)
+        self.client = client
+
+    @staticmethod
+    def init(client, path, date=None):
+        """Initial UserManager(), get predict date and trade date
+        Parameters
+        ----------
+            client: str
+                The qlib client config file(.yaml)
+            path : str
+                Path to save user account.
+            date : str (YYYY-MM-DD)
+                Trade date, when the generated order list will be traded.
+        Return
+        ----------
+            um: UserManager()
+            pred_date: pd.Timestamp
+            trade_date: pd.Timestamp
+        """
+        qlib.init_from_yaml_conf(client)
+        um = UserManager(user_data_path=pathlib.Path(path))
+        um.load_users()
+        if not date:
+            trade_date, pred_date = None, None
+        else:
+            trade_date = pd.Timestamp(date)
+            if not is_tradable_date(trade_date):
+                raise ValueError("trade date is not tradable date".format(trade_date.date()))
+            pred_date = get_pre_trading_date(trade_date, future=True)
+        return um, pred_date, trade_date
+
+    def add_user(self, id, config, path, date):
+        """Add a new user into the a folder to run 'online' module.
+
+        Parameters
+        ----------
+        id : str
+            User id, should be unique.
+        config : str
+            The file path (yaml) of user config
+        path : str
+            Path to save user account.
+        date : str (YYYY-MM-DD)
+            The date that user account was added.
+        """
+        create_user_folder(path)
+        qlib.init_from_yaml_conf(self.client)
+        um = UserManager(user_data_path=path)
+        add_date = D.calendar(end_time=date)[-1]
+        if not is_tradable_date(add_date):
+            raise ValueError("add date is not tradable date".format(add_date.date()))
+        um.add_user(user_id=id, config_file=config, add_date=add_date)
+
+    def remove_user(self, id, path):
+        """Remove user from folder used in 'online' module.
+
+        Parameters
+        ----------
+        id : str
+            User id, should be unique.
+        path : str
+            Path to save user account.
+        """
+        um = UserManager(user_data_path=path)
+        um.remove_user(user_id=id)
+
+    def generate(self, date, path):
+        """Generate order list that will be traded at 'date'.
+
+        Parameters
+        ----------
+        date : str (YYYY-MM-DD)
+            Trade date, when the generated order list will be traded.
+        path : str
+            Path to save user account.
+        """
+        um, pred_date, trade_date = self.init(self.client, path, date)
+        for user_id, user in um.users.items():
+            dates, trade_exchange = prepare(um, pred_date, user_id)
+            # get and save the score at predict date
+            input_data = user.model.get_data_with_date(pred_date)
+            score_series = user.model.predict(input_data)
+            save_score_series(score_series, (pathlib.Path(path) / user_id), trade_date)
+
+            # update strategy (and model)
+            user.strategy.update(score_series, pred_date, trade_date)
+
+            # generate and save order list
+            order_list = user.strategy.generate_order_list(
+                score_series=score_series,
+                current=user.account.current,
+                trade_exchange=trade_exchange,
+                trade_date=trade_date,
+            )
+            save_order_list(
+                order_list=order_list,
+                user_path=(pathlib.Path(path) / user_id),
+                trade_date=trade_date,
+            )
+            self.logger.info("Generate order list at {} for {}".format(trade_date, user_id))
+            um.save_user_data(user_id)
+
+    def execute(self, date, exchange_config, path):
+        """Execute the orderlist at 'date'.
+
+        Parameters
+        ----------
+           date : str (YYYY-MM-DD)
+               Trade date, that the generated order list will be traded.
+           exchange_config: str
+               The file path (yaml) of exchange config
+           path : str
+               Path to save user account.
+        """
+        um, pred_date, trade_date = self.init(self.client, path, date)
+        for user_id, user in um.users.items():
+            dates, trade_exchange = prepare(um, trade_date, user_id, exchange_config)
+            executor = SimulatorExecutor(trade_exchange=trade_exchange)
+            if not str(dates[0].date()) == str(pred_date.date()):
+                raise ValueError(
+                    "The account data is not newest! last trading date {}, today {}".format(
+                        dates[0].date(), trade_date.date()
+                    )
+                )
+
+            # load and execute the order list
+            # will not modify the trade_account after executing
+            order_list = load_order_list(user_path=(pathlib.Path(path) / user_id), trade_date=trade_date)
+            trade_info = executor.execute(order_list=order_list, trade_account=user.account, trade_date=trade_date)
+            executor.save_executed_file_from_trade_info(
+                trade_info=trade_info,
+                user_path=(pathlib.Path(path) / user_id),
+                trade_date=trade_date,
+            )
+            self.logger.info("execute order list at {} for {}".format(trade_date.date(), user_id))
+
+    def update(self, date, path, type="SIM"):
+        """Update account at 'date'.
+
+        Parameters
+        ----------
+        date : str (YYYY-MM-DD)
+            Trade date, that the generated order list will be traded.
+        path : str
+            Path to save user account.
+        type : str
+            which executor was been used to execute the order list
+            'SIM': SimulatorExecutor()
+        """
+        if type not in ["SIM", "YC"]:
+            raise ValueError("type is invalid, {}".format(type))
+        um, pred_date, trade_date = self.init(self.client, path, date)
+        for user_id, user in um.users.items():
+            dates, trade_exchange = prepare(um, trade_date, user_id)
+            if type == "SIM":
+                executor = SimulatorExecutor(trade_exchange=trade_exchange)
+            else:
+                raise ValueError("not found executor")
+            # dates[0] is the last_trading_date
+            if str(dates[0].date()) > str(pred_date.date()):
+                raise ValueError(
+                    "The account data is not newest! last trading date {}, today {}".format(
+                        dates[0].date(), trade_date.date()
+                    )
+                )
+            # load trade info and update account
+            trade_info = executor.load_trade_info_from_executed_file(
+                user_path=(pathlib.Path(path) / user_id), trade_date=trade_date
+            )
+            score_series = load_score_series((pathlib.Path(path) / user_id), trade_date)
+            update_account(user.account, trade_info, trade_exchange, trade_date)
+
+            report = user.account.report.generate_report_dataframe()
+            self.logger.info(report)
+            um.save_user_data(user_id)
+            self.logger.info("Update account state {} for {}".format(trade_date, user_id))
+
+    def simulate(self, id, config, exchange_config, start, end, path, bench="SH000905"):
+        """Run the ( generate_order_list -> execute_order_list -> update_account) process everyday
+            from start date to end date.
+
+        Parameters
+        ----------
+        id : str
+            user id, need to be unique
+        config : str
+            The file path (yaml) of user config
+        exchange_config: str
+            The file path (yaml) of exchange config
+        start : str "YYYY-MM-DD"
+            The start date to run the online simulate
+        end : str "YYYY-MM-DD"
+            The end date to run the online simulate
+        path : str
+            Path to save user account.
+        bench : str
+            The benchmark that our result compared with.
+            'SH000905' for csi500, 'SH000300' for csi300
+        """
+        # Clear the current user if exists, then add a new user.
+        create_user_folder(path)
+        um = self.init(self.client, path, None)[0]
+        start_date, end_date = pd.Timestamp(start), pd.Timestamp(end)
+        try:
+            um.remove_user(user_id=id)
+        except BaseException:
+            pass
+        um.add_user(user_id=id, config_file=config, add_date=pd.Timestamp(start_date))
+
+        # Do the online simulate
+        um.load_users()
+        user = um.users[id]
+        dates, trade_exchange = prepare(um, end_date, id, exchange_config)
+        executor = SimulatorExecutor(trade_exchange=trade_exchange)
+        for pred_date, trade_date in zip(dates[:-2], dates[1:-1]):
+            user_path = pathlib.Path(path) / id
+
+            # 1. load and save score_series
+            input_data = user.model.get_data_with_date(pred_date)
+            score_series = user.model.predict(input_data)
+            save_score_series(score_series, (pathlib.Path(path) / id), trade_date)
+
+            # 2. update strategy (and model)
+            user.strategy.update(score_series, pred_date, trade_date)
+
+            # 3. generate and save order list
+            order_list = user.strategy.generate_order_list(
+                score_series=score_series,
+                current=user.account.current,
+                trade_exchange=trade_exchange,
+                trade_date=trade_date,
+            )
+            save_order_list(order_list=order_list, user_path=user_path, trade_date=trade_date)
+
+            # 4. auto execute order list
+            order_list = load_order_list(user_path=user_path, trade_date=trade_date)
+            trade_info = executor.execute(trade_account=user.account, order_list=order_list, trade_date=trade_date)
+            executor.save_executed_file_from_trade_info(
+                trade_info=trade_info, user_path=user_path, trade_date=trade_date
+            )
+            # 5. update account state
+            trade_info = executor.load_trade_info_from_executed_file(user_path=user_path, trade_date=trade_date)
+            update_account(user.account, trade_info, trade_exchange, trade_date)
+        report = user.account.report.generate_report_dataframe()
+        self.logger.info(report)
+        um.save_user_data(id)
+        self.show(id, path, bench)
+
+    def show(self, id, path, bench="SH000905"):
+        """show the newly report (mean, std, sharpe, annual)
+
+        Parameters
+        ----------
+        id : str
+            user id, need to be unique
+        path : str
+            Path to save user account.
+        bench : str
+            The benchmark that our result compared with.
+            'SH000905' for csi500, 'SH000300' for csi300
+        """
+        um = self.init(self.client, path, None)[0]
+        if id not in um.users:
+            raise ValueError("Cannot find user ".format(id))
+        bench = D.features([bench], ["$change"]).loc[bench, "$change"]
+        report = um.users[id].account.report.generate_report_dataframe()
+        report["bench"] = bench
+        analysis_result = {}
+        r = (report["return"] - report["bench"]).dropna()
+        analysis_result["sub_bench"] = risk_analysis(r)
+        r = (report["return"] - report["bench"] - report["cost"]).dropna()
+        analysis_result["sub_cost"] = risk_analysis(r)
+        print("Result:")
+        print("sub_bench:")
+        print(analysis_result["sub_bench"])
+        print("sub_cost:")
+        print(analysis_result["sub_cost"])
+
+
+def run():
+    fire.Fire(Operator)
+
+
+if __name__ == "__main__":
+    run()

qlib/contrib/online/user.py

@@ -0,0 +1,74 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import logging
+
+from ...log import get_module_logger
+from ..evaluate import risk_analysis
+from ...data import D
+
+
+class User:
+    def __init__(self, account, strategy, model, verbose=False):
+        """
+        A user in online system, which contains account, strategy and model three module.
+            Parameter
+                account : Account()
+                strategy :
+                    a strategy instance
+                model :
+                    a model instance
+                report_save_path : string
+                    the path to save report. Will not save report if None
+                verbose : bool
+                    Whether to print the info during the process
+        """
+        self.logger = get_module_logger("User", level=logging.INFO)
+        self.account = account
+        self.strategy = strategy
+        self.model = model
+        self.verbose = verbose
+
+    def init_state(self, date):
+        """
+        init state when each trading date begin
+            Parameter
+                date : pd.Timestamp
+        """
+        self.account.init_state(today=date)
+        self.strategy.init_state(trade_date=date, model=self.model, account=self.account)
+        return
+
+    def get_latest_trading_date(self):
+        """
+        return the latest trading date for user {user_id}
+            Parameter
+                user_id : string
+            :return
+                date : string (e.g '2018-10-08')
+        """
+        if not self.account.last_trade_date:
+            return None
+        return str(self.account.last_trade_date.date())
+
+    def showReport(self, benchmark="SH000905"):
+        """
+        show the newly report (mean, std, sharpe, annual)
+            Parameter
+                benchmark : string
+                    bench that to be compared, 'SH000905' for csi500
+        """
+        bench = D.features([benchmark], ["$change"], disk_cache=True).loc[benchmark, "$change"]
+        report = self.account.report.generate_report_dataframe()
+        report["bench"] = bench
+        analysis_result = {"pred": {}, "sub_bench": {}, "sub_cost": {}}
+        r = (report["return"] - report["bench"]).dropna()
+        analysis_result["sub_bench"][0] = risk_analysis(r)
+        r = (report["return"] - report["bench"] - report["cost"]).dropna()
+        analysis_result["sub_cost"][0] = risk_analysis(r)
+        self.logger.info("Result of porfolio:")
+        self.logger.info("sub_bench:")
+        self.logger.info(analysis_result["sub_bench"][0])
+        self.logger.info("sub_cost:")
+        self.logger.info(analysis_result["sub_cost"][0])
+        return report

qlib/contrib/online/utils.py

@@ -0,0 +1,110 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import pathlib
+import pickle
+import yaml
+import pandas as pd
+from ...data import D
+from ...log import get_module_logger
+from ...utils import get_module_by_module_path
+from ...utils import get_next_trading_date
+from ..backtest.exchange import Exchange
+
+log = get_module_logger("utils")
+
+
+def load_instance(file_path):
+    """
+    load a pickle file
+        Parameter
+           file_path : string / pathlib.Path()
+                path of file to be loaded
+        :return
+            An instance loaded from file
+    """
+    file_path = pathlib.Path(file_path)
+    if not file_path.exists():
+        raise ValueError("Cannot find file {}".format(file_path))
+    with file_path.open("rb") as fr:
+        instance = pickle.load(fr)
+    return instance
+
+
+def save_instance(instance, file_path):
+    """
+    save(dump) an instance to a pickle file
+        Parameter
+            instance :
+                data to te dumped
+            file_path : string / pathlib.Path()
+                path of file to be dumped
+    """
+    file_path = pathlib.Path(file_path)
+    with file_path.open("wb") as fr:
+        pickle.dump(instance, fr)
+
+
+def init_instance_by_config(config):
+    """
+    generate an instance with settings in config
+        Parameter
+            config : dict
+                python dict indicate a init parameters to create an item
+        :return
+            An instance
+    """
+    module = get_module_by_module_path(config["module_path"])
+    instance_class = getattr(module, config["class"])
+    instance = instance_class(**config["args"])
+    return instance
+
+
+def create_user_folder(path):
+    path = pathlib.Path(path)
+    if path.exists():
+        return
+    path.mkdir(parents=True)
+    head = pd.DataFrame(columns=("user_id", "add_date"))
+    head.to_csv(path / "users.csv", index=None)
+
+
+def prepare(um, today, user_id, exchange_config=None):
+    """
+    1. Get the dates that need to do trading till today for user {user_id}
+        dates[0] indicate the latest trading date of User{user_id},
+        if User{user_id} haven't do trading before, than dates[0] presents the init date of User{user_id}.
+    2. Set the exchange with exchange_config file
+
+        Parameter
+            um : UserManager()
+            today : pd.Timestamp()
+            user_id : str
+        :return
+            dates : list of pd.Timestamp
+            trade_exchange : Exchange()
+    """
+    # get latest trading date for {user_id}
+    # if is None, indicate it haven't traded, then last trading date is init date of {user_id}
+    latest_trading_date = um.users[user_id].get_latest_trading_date()
+    if not latest_trading_date:
+        latest_trading_date = um.user_record.loc[user_id][0]
+
+    if str(today.date()) < latest_trading_date:
+        log.warning("user_id:{}, last trading date {} after today {}".format(user_id, latest_trading_date, today))
+        return [pd.Timestamp(latest_trading_date)], None
+
+    dates = D.calendar(
+        start_time=pd.Timestamp(latest_trading_date),
+        end_time=pd.Timestamp(today),
+        future=True,
+    )
+    dates = list(dates)
+    dates.append(get_next_trading_date(dates[-1], future=True))
+    if exchange_config:
+        with pathlib.Path(exchange_config).open("r") as fp:
+            exchange_paras = yaml.load(fp)
+    else:
+        exchange_paras = {}
+    trade_exchange = Exchange(trade_dates=dates, **exchange_paras)
+    return dates, trade_exchange