init commit

qlib/contrib/tuner/config.py

@@ -0,0 +1,88 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import yaml
+import copy
+import os
+
+
+class TunerConfigManager(object):
+    def __init__(self, config_path):
+
+        if not config_path:
+            raise ValueError("Config path is invalid.")
+        self.config_path = config_path
+
+        with open(config_path) as fp:
+            config = yaml.load(fp)
+        self.config = copy.deepcopy(config)
+
+        self.pipeline_ex_config = PipelineExperimentConfig(config.get("experiment", dict()), self)
+        self.pipeline_config = config.get("tuner_pipeline", list())
+        self.optim_config = OptimizationConfig(config.get("optimization_criteria", dict()), self)
+
+        self.time_config = config.get("time_period", dict())
+        self.data_config = config.get("data", dict())
+        self.backtest_config = config.get("backtest", dict())
+        self.qlib_client_config = config.get("qlib_client", dict())
+
+
+class PipelineExperimentConfig(object):
+    def __init__(self, config, TUNER_CONFIG_MANAGER):
+        """
+        :param config:  The config dict for tuner experiment
+        :param TUNER_CONFIG_MANAGER:   The tuner config manager
+        """
+        self.name = config.get("name", "tuner_experiment")
+        # The dir of the config
+        self.global_dir = config.get("dir", os.path.dirname(TUNER_CONFIG_MANAGER.config_path))
+        # The dir of the result of tuner experiment
+        self.tuner_ex_dir = config.get("tuner_ex_dir", os.path.join(self.global_dir, self.name))
+        if not os.path.exists(self.tuner_ex_dir):
+            os.makedirs(self.tuner_ex_dir)
+        # The dir of the results of all estimator experiments
+        self.estimator_ex_dir = config.get("estimator_ex_dir", os.path.join(self.tuner_ex_dir, "estimator_experiment"))
+        if not os.path.exists(self.estimator_ex_dir):
+            os.makedirs(self.estimator_ex_dir)
+        # Get the tuner type
+        self.tuner_module_path = config.get("tuner_module_path", "qlib.contrib.tuner.tuner")
+        self.tuner_class = config.get("tuner_class", "QLibTuner")
+        # Save the tuner experiment for further view
+        tuner_ex_config_path = os.path.join(self.tuner_ex_dir, "tuner_config.yaml")
+        with open(tuner_ex_config_path, "w") as fp:
+            yaml.dump(TUNER_CONFIG_MANAGER.config, fp)
+
+
+class OptimizationConfig(object):
+    def __init__(self, config, TUNER_CONFIG_MANAGER):
+
+        self.report_type = config.get("report_type", "pred_long")
+        if self.report_type not in [
+            "pred_long",
+            "pred_long_short",
+            "pred_short",
+            "sub_bench",
+            "sub_cost",
+            "model",
+        ]:
+            raise ValueError(
+                "report_type should be one of pred_long, pred_long_short, pred_short, sub_bench, sub_cost and model"
+            )
+
+        self.report_factor = config.get("report_factor", "sharpe")
+        if self.report_factor not in [
+            "annual",
+            "sharpe",
+            "mdd",
+            "mean",
+            "std",
+            "model_score",
+            "model_pearsonr",
+        ]:
+            raise ValueError(
+                "report_factor should be one of annual, sharpe, mdd, mean, std, model_pearsonr and model_score"
+            )
+
+        self.optim_type = config.get("optim_type", "max")
+        if self.optim_type not in ["min", "max", "correlation"]:
+            raise ValueError("optim_type should be min, max or correlation")

qlib/contrib/tuner/launcher.py

@@ -0,0 +1,34 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# coding=utf-8
+
+import argparse
+import importlib
+import os
+import yaml
+
+from .config import TunerConfigManager
+
+
+args_parser = argparse.ArgumentParser(prog="tuner")
+args_parser.add_argument(
+    "-c",
+    "--config_path",
+    required=True,
+    type=str,
+    help="config path indicates where to load yaml config.",
+)
+
+args = args_parser.parse_args()
+
+TUNER_CONFIG_MANAGER = TunerConfigManager(args.config_path)
+
+
+def run():
+    # 1. Get pipeline class.
+    tuner_pipeline_class = getattr(importlib.import_module(".pipeline", package="qlib.contrib.tuner"), "Pipeline")
+    # 2. Init tuner pipeline.
+    tuner_pipeline = tuner_pipeline_class(TUNER_CONFIG_MANAGER)
+    # 3. Begin to tune
+    tuner_pipeline.run()

qlib/contrib/tuner/pipeline.py

@@ -0,0 +1,86 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import os
+import json
+import logging
+import importlib
+from abc import abstractmethod
+
+from ...log import get_module_logger, TimeInspector
+from ...utils import get_module_by_module_path
+
+
+class Pipeline(object):
+
+    GLOBAL_BEST_PARAMS_NAME = "global_best_params.json"
+
+    def __init__(self, tuner_config_manager):
+
+        self.logger = get_module_logger("Pipeline", sh_level=logging.INFO)
+
+        self.tuner_config_manager = tuner_config_manager
+
+        self.pipeline_ex_config = tuner_config_manager.pipeline_ex_config
+        self.optim_config = tuner_config_manager.optim_config
+        self.time_config = tuner_config_manager.time_config
+        self.pipeline_config = tuner_config_manager.pipeline_config
+        self.data_config = tuner_config_manager.data_config
+        self.backtest_config = tuner_config_manager.backtest_config
+        self.qlib_client_config = tuner_config_manager.qlib_client_config
+
+        self.global_best_res = None
+        self.global_best_params = None
+        self.best_tuner_index = None
+
+    def run(self):
+
+        TimeInspector.set_time_mark()
+        for tuner_index, tuner_config in enumerate(self.pipeline_config):
+            tuner = self.init_tuner(tuner_index, tuner_config)
+            tuner.tune()
+            if self.global_best_res is None or self.global_best_res > tuner.best_res:
+                self.global_best_res = tuner.best_res
+                self.global_best_params = tuner.best_params
+                self.best_tuner_index = tuner_index
+        TimeInspector.log_cost_time("Finished tuner pipeline.")
+
+        self.save_tuner_exp_info()
+
+    def init_tuner(self, tuner_index, tuner_config):
+        """
+        Implement this method to build the tuner by config
+        return: tuner
+        """
+        # 1. Add experiment config in tuner_config
+        tuner_config["experiment"] = {
+            "name": "estimator_experiment_{}".format(tuner_index),
+            "id": tuner_index,
+            "dir": self.pipeline_ex_config.estimator_ex_dir,
+            "observer_type": "file_storage",
+        }
+        tuner_config["qlib_client"] = self.qlib_client_config
+        # 2. Add data config in tuner_config
+        tuner_config["data"] = self.data_config
+        # 3. Add backtest config in tuner_config
+        tuner_config["backtest"] = self.backtest_config
+        # 4. Update trainer in tuner_config
+        tuner_config["trainer"].update({"args": self.time_config})
+
+        # 5. Import Tuner class
+        tuner_module = get_module_by_module_path(self.pipeline_ex_config.tuner_module_path)
+        tuner_class = getattr(tuner_module, self.pipeline_ex_config.tuner_class)
+        # 6. Return the specific tuner
+        return tuner_class(tuner_config, self.optim_config)
+
+    def save_tuner_exp_info(self):
+
+        TimeInspector.set_time_mark()
+        save_path = os.path.join(self.pipeline_ex_config.tuner_ex_dir, Pipeline.GLOBAL_BEST_PARAMS_NAME)
+        with open(save_path, "w") as fp:
+            json.dump(self.global_best_params, fp)
+        TimeInspector.log_cost_time("Finished save global best tuner parameters.")
+
+        self.logger.info("Best Tuner id: {}.".format(self.best_tuner_index))
+        self.logger.info("Global best parameters: {}.".format(self.global_best_params))
+        self.logger.info("You can check the best parameters at {}.".format(save_path))

qlib/contrib/tuner/space.py

@@ -0,0 +1,17 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+from hyperopt import hp
+
+
+TopkAmountStrategySpace = {
+    "topk": hp.choice("topk", [30, 35, 40]),
+    "buffer_margin": hp.choice("buffer_margin", [200, 250, 300]),
+}
+
+QLibDataLabelSpace = {
+    "labels": hp.choice(
+        "labels",
+        [["Ref($vwap, -2)/Ref($vwap, -1) - 1"], ["Ref($close, -5)/$close - 1"]],
+    )
+}

qlib/contrib/tuner/tuner.py

@@ -0,0 +1,218 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import os
+import yaml
+import json
+import copy
+import pickle
+import logging
+import importlib
+import subprocess
+import pandas as pd
+import numpy as np
+
+from abc import abstractmethod
+
+from ...log import get_module_logger, TimeInspector
+from hyperopt import fmin, tpe
+from hyperopt import STATUS_OK, STATUS_FAIL
+
+
+class Tuner(object):
+    def __init__(self, tuner_config, optim_config):
+
+        self.logger = get_module_logger("Tuner", sh_level=logging.INFO)
+
+        self.tuner_config = tuner_config
+        self.optim_config = optim_config
+
+        self.max_evals = self.tuner_config.get("max_evals", 10)
+        self.ex_dir = os.path.join(
+            self.tuner_config["experiment"]["dir"],
+            self.tuner_config["experiment"]["name"],
+        )
+
+        self.best_params = None
+        self.best_res = None
+
+        self.space = self.setup_space()
+
+    def tune(self):
+
+        TimeInspector.set_time_mark()
+        fmin(
+            fn=self.objective,
+            space=self.space,
+            algo=tpe.suggest,
+            max_evals=self.max_evals,
+        )
+        self.logger.info("Local best params: {} ".format(self.best_params))
+        TimeInspector.log_cost_time(
+            "Finished searching best parameters in Tuner {}.".format(self.tuner_config["experiment"]["id"])
+        )
+
+        self.save_local_best_params()
+
+    @abstractmethod
+    def objective(self, params):
+        """
+        Implement this method to give an optimization factor using parameters in space.
+        :return: {'loss': a factor for optimization, float type,
+                  'status': the status of this evaluation step, STATUS_OK or STATUS_FAIL}.
+        """
+        pass
+
+    @abstractmethod
+    def setup_space(self):
+        """
+        Implement this method to setup the searching space of tuner.
+        :return: searching space, dict type.
+        """
+        pass
+
+    @abstractmethod
+    def save_local_best_params(self):
+        """
+        Implement this method to save the best parameters of this tuner.
+        """
+        pass
+
+
+class QLibTuner(Tuner):
+
+    ESTIMATOR_CONFIG_NAME = "estimator_config.yaml"
+    EXP_INFO_NAME = "exp_info.json"
+    EXP_RESULT_DIR = "sacred/{}"
+    EXP_RESULT_NAME = "analysis.pkl"
+    LOCAL_BEST_PARAMS_NAME = "local_best_params.json"
+
+    def objective(self, params):
+
+        # 1. Setup an config for a spcific estimator process
+        estimator_path = self.setup_estimator_config(params)
+        self.logger.info("Searching params: {} ".format(params))
+
+        # 2. Use subprocess to do the estimator program, this process will wait until subprocess finish
+        sub_fails = subprocess.call("estimator -c {}".format(estimator_path), shell=True)
+        if sub_fails:
+            # If this subprocess failed, ignore this evaluation step
+            self.logger.info("Estimator experiment failed when using this searching parameters")
+            return {"loss": np.nan, "status": STATUS_FAIL}
+
+        # 3. Fetch the result of subprocess, and check whether the result is Nan
+        res = self.fetch_result()
+        if np.isnan(res):
+            status = STATUS_FAIL
+        else:
+            status = STATUS_OK
+
+        # 4. Save the best score and params
+        if self.best_res is None or self.best_res > res:
+            self.best_res = res
+            self.best_params = params
+
+        # 5. Return the result as optim objective
+        return {"loss": res, "status": status}
+
+    def fetch_result(self):
+
+        # 1. Get experiment information
+        exp_info_path = os.path.join(self.ex_dir, QLibTuner.EXP_INFO_NAME)
+        with open(exp_info_path) as fp:
+            exp_info = json.load(fp)
+        estimator_ex_id = exp_info["id"]
+
+        # 2. Return model result if needed
+        if self.optim_config.report_type == "model":
+            if self.optim_config.report_factor == "model_score":
+                # if estimator experiment is multi-label training, user need to process the scores by himself
+                # Default method is return the average score
+                return np.mean(exp_info["performance"]["model_score"])
+            elif self.optim_config.report_factor == "model_pearsonr":
+                # pearsonr is a correlation coefficient, 1 is the best
+                return np.abs(exp_info["performance"]["model_pearsonr"] - 1)
+
+        # 3. Get backtest results
+        exp_result_dir = os.path.join(self.ex_dir, QLibTuner.EXP_RESULT_DIR.format(estimator_ex_id))
+        exp_result_path = os.path.join(exp_result_dir, QLibTuner.EXP_RESULT_NAME)
+        with open(exp_result_path, "rb") as fp:
+            analysis_df = pickle.load(fp)
+
+        # 4. Get the backtest factor which user want to optimize, if user want to maximize the factor, then reverse the result
+        res = analysis_df.loc[self.optim_config.report_type].loc[self.optim_config.report_factor]
+        # res = res.values[0] if self.optim_config.optim_type == 'min' else -res.values[0]
+        if self.optim_config == "min":
+            return res.values[0]
+        elif self.optim_config == "max":
+            return -res.values[0]
+        else:
+            # self.optim_config == 'correlation'
+            return np.abs(res.values[0] - 1)
+
+    def setup_estimator_config(self, params):
+
+        estimator_config = copy.deepcopy(self.tuner_config)
+        estimator_config["model"].update({"args": params["model_space"]})
+        estimator_config["strategy"].update({"args": params["strategy_space"]})
+        if params.get("data_label_space", None) is not None:
+            estimator_config["data"]["args"].update(params["data_label_space"])
+
+        estimator_path = os.path.join(
+            self.tuner_config["experiment"].get("dir", "../"),
+            QLibTuner.ESTIMATOR_CONFIG_NAME,
+        )
+
+        with open(estimator_path, "w") as fp:
+            yaml.dump(estimator_config, fp)
+
+        return estimator_path
+
+    def setup_space(self):
+        # 1. Setup model space
+        model_space_name = self.tuner_config["model"].get("space", None)
+        if model_space_name is None:
+            raise ValueError("Please give the search space of model.")
+        model_space = getattr(
+            importlib.import_module(".space", package="qlib.contrib.tuner"),
+            model_space_name,
+        )
+
+        # 2. Setup strategy space
+        strategy_space_name = self.tuner_config["strategy"].get("space", None)
+        if strategy_space_name is None:
+            raise ValueError("Please give the search space of strategy.")
+        strategy_space = getattr(
+            importlib.import_module(".space", package="qlib.contrib.tuner"),
+            strategy_space_name,
+        )
+
+        # 3. Setup data label space if given
+        if self.tuner_config.get("data_label", None) is not None:
+            data_label_space_name = self.tuner_config["data_label"].get("space", None)
+            if data_label_space_name is not None:
+                data_label_space = getattr(
+                    importlib.import_module(".space", package="qlib.contrib.tuner"),
+                    data_label_space_name,
+                )
+        else:
+            data_label_space_name = None
+
+        # 4. Combine the searching space
+        space = dict()
+        space.update({"model_space": model_space})
+        space.update({"strategy_space": strategy_space})
+        if data_label_space_name is not None:
+            space.update({"data_label_space": data_label_space})
+
+        return space
+
+    def save_local_best_params(self):
+
+        TimeInspector.set_time_mark()
+        local_best_params_path = os.path.join(self.ex_dir, QLibTuner.LOCAL_BEST_PARAMS_NAME)
+        with open(local_best_params_path, "w") as fp:
+            json.dump(self.best_params, fp)
+        TimeInspector.log_cost_time(
+            "Finished saving local best tuner parameters to: {} .".format(local_best_params_path)
+        )