liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-09 14:00:55 +08:00
Code Activity

Online Serving V4

Browse Source
This commit is contained in:
lzh222333
2021-03-26 04:20:25 +00:00
parent 8abdd63869
commit 46cd57688e
12 changed files with 366 additions and 323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
examples/taskmanager/task_manager_rolling.py → examples/model_rolling/task_manager_rolling.py
View File

@@ -1,13 +1,13 @@
from pprint import pprint
import fire
import qlib
from qlib.config import REG_CN
from qlib.workflow.task.gen import RollingGen, task_generator
from qlib.workflow.task.manage import TaskManager
from qlib.config import C
from qlib.workflow.task.manage import run_task
from qlib.workflow.task.collect import RollingCollector
from qlib.model.trainer import task_train
from qlib.workflow import R
from pprint import pprint
from qlib.workflow.task.collect import RollingCollector
from qlib.workflow.task.gen import RollingGen, task_generator
from qlib.workflow.task.manage import TaskManager, run_task
data_handler_config = {
"start_time": "2008-01-01",
@@ -66,14 +66,14 @@ task_xgboost_config = {
}
# Reset all things to the first status, be careful to save important data
def reset():
def reset(task_pool, exp_name):
print("========== reset ==========")
TaskManager(task_pool=task_pool).remove()
# exp = R.get_exp(experiment_name=exp_name)
exp, _ = R.exp_manager._get_or_create_exp(experiment_name=exp_name)
# for rid in R.list_recorders():
# exp.delete_recorder(rid)
for rid in exp.list_recorders():
exp.delete_recorder(rid)
# This part corresponds to "Task Generating" in the document
@@ -92,51 +92,58 @@ def task_generating():
# This part corresponds to "Task Storing" in the document
def task_storing(tasks):
def task_storing(tasks, task_pool, exp_name):
print("========== task_storing ==========")
tm = TaskManager(task_pool=task_pool)
tm.create_task(tasks) # all tasks will be saved to MongoDB
# This part corresponds to "Task Running" in the document
def task_running():
def task_running(task_pool, exp_name):
print("========== task_running ==========")
run_task(task_train, task_pool, experiment_name=exp_name) # all tasks will be trained using "task_train" method
# This part corresponds to "Task Collecting" in the document
def task_collecting():
def task_collecting(task_pool, exp_name):
print("========== task_collecting ==========")
def get_task_key(task_config):
def get_group_key_func(recorder):
task_config = recorder.load_object("task")
return task_config["model"]["class"]
def my_filter(recorder):
# only choose the results of "LGBModel"
task_key = get_task_key(rolling_collector.get_task(recorder))
task_key = get_group_key_func(recorder)
if task_key == "LGBModel":
return True
return False
rolling_collector = RollingCollector(exp_name)
# group tasks by "get_task_key" and filter tasks by "my_filter"
pred_rolling = rolling_collector.collect_rolling_predictions(get_task_key, my_filter)
pred_rolling = rolling_collector.collect(get_group_key_func, my_filter)
print(pred_rolling)
if __name__ == "__main__":
provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
def main(
provider_uri="~/.qlib/qlib_data/cn_data",
task_url="mongodb://10.0.0.4:27017/",
task_db_name="rolling_db",
exp_name="rolling_exp",
task_pool="rolling_task",
):
mongo_conf = {
"task_url": "mongodb://10.0.0.4:27017/", # maybe you need to change it to your url
"task_db_name": "rolling_db",
"task_url": task_url,
"task_db_name": task_db_name,
}
exp_name = "rolling_exp" # experiment name, will be used as the experiment in MLflow
task_pool = "rolling_task" # task pool name, will be used as the document in MongoDB
qlib.init(provider_uri=provider_uri, region=REG_CN, mongo=mongo_conf)
reset()
reset(task_pool, exp_name)
tasks = task_generating()
task_storing(tasks)
task_running()
task_collecting()
task_storing(tasks, task_pool, exp_name)
task_running(task_pool, exp_name)
task_collecting(task_pool, exp_name)
if __name__ == "__main__":
fire.Fire()

109
examples/taskmanager/task_manager_rolling_with_updating.py → examples/online_svr/task_manager_rolling_with_updating.py
View File

@@ -1,16 +1,15 @@
import qlib
import fire
import mlflow
from qlib.config import C
from qlib.workflow import R
from pprint import pprint
import fire
import qlib
from qlib.config import REG_CN
from qlib.model.trainer import task_train
from qlib.workflow.task.manage import run_task
from qlib.workflow.task.manage import TaskManager
from qlib.workflow import R
from qlib.workflow.task.collect import RollingCollector
from qlib.workflow.task.gen import RollingGen, task_generator
from qlib.workflow.task.manage import TaskManager, run_task
from qlib.workflow.task.online import RollingOnlineManager
from qlib.workflow.task.utils import list_recorders
data_handler_config = {
"start_time": "2013-01-01",
@@ -70,12 +69,15 @@ task_xgboost_config = {
def print_online_model():
print("========== print_online_model ==========")
print("Current 'online' model:")
for online in rolling_online_manager.list_online_model().values():
print(online.info["id"])
for rid, rec in list_recorders(exp_name).items():
if rolling_online_manager.get_online_tag(rec) == rolling_online_manager.ONLINE_TAG:
print(rid)
print("Current 'next online' model:")
for online in rolling_online_manager.list_next_online_model().values():
print(online.info["id"])
for rid, rec in list_recorders(exp_name).items():
if rolling_online_manager.get_online_tag(rec) == rolling_online_manager.NEXT_ONLINE_TAG:
print(rid)
# This part corresponds to "Task Generating" in the document
@@ -110,119 +112,76 @@ def task_running():
def task_collecting():
print("========== task_collecting ==========")
def get_task_key(task_config):
def get_group_key_func(recorder):
task_config = recorder.load_object("task")
return task_config["model"]["class"]
def my_filter(recorder):
# only choose the results of "LGBModel"
task_key = get_task_key(rolling_collector.get_task(recorder))
task_key = get_group_key_func(recorder)
if task_key == "LGBModel":
return True
return False
rolling_collector = RollingCollector(exp_name)
# group tasks by "get_task_key" and filter tasks by "my_filter"
pred_rolling = rolling_collector.collect_rolling_predictions(get_task_key, my_filter)
pred_rolling = rolling_collector.collect(get_group_key_func, my_filter)
print(pred_rolling)
# Reset all things to the first status, be careful to save important data
def reset(force_end=False):
def reset():
print("========== reset ==========")
task_manager.remove()
for error in task_manager.query():
assert False
exp = R.get_exp(experiment_name=exp_name)
recs = exp.list_recorders()
for rid in recs:
exp, _ = R.exp_manager._get_or_create_exp(experiment_name=exp_name)
for rid in exp.list_recorders():
exp.delete_recorder(rid)
try:
if force_end:
mlflow.end_run()
except Exception:
pass
# Run this firstly to see the workflow in Task Management
def first_run():
print("========== first_run ==========")
reset(force_end=True)
reset()
tasks = task_generating()
task_storing(tasks)
task_running()
task_collecting()
rolling_online_manager.set_latest_model_to_next_online()
rolling_online_manager.reset_online_model()
# Update the predictions of online model
def update_predictions():
print("========== update_predictions ==========")
rolling_online_manager.update_online_pred()
task_collecting()
# if there are some next_online_model, then online them. if no, still use current online_model.
print_online_model()
rolling_online_manager.reset_online_model()
print_online_model()
# Update the models using the latest date and set them to online model
def update_model():
print("========== update_model ==========")
rolling_online_manager.prepare_new_models()
print_online_model()
rolling_online_manager.set_latest_model_to_next_online()
print_online_model()
latest_rec, _ = rolling_online_manager.list_latest_recorders()
rolling_online_manager.reset_online_tag(latest_rec.values())
def after_day():
rolling_online_manager.prepare_signals()
update_model()
update_predictions()
# Run whole workflow completely
def whole_workflow():
print("========== whole_workflow ==========")
# run this at the first time
first_run()
# run this every day after trading
after_day()
print("========== after_day ==========")
print_online_model()
rolling_online_manager.after_day()
print_online_model()
task_collecting()
if __name__ == "__main__":
####### to train the first version's models, use the command below
# python task_manager_rolling_with_updating.py first_run
####### to update the models using the latest date, use the command below
# python task_manager_rolling_with_updating.py update_model
####### to update the predictions to the latest date, use the command below
# python task_manager_rolling_with_updating.py update_predictions
####### to run whole workflow completely, use the command below
# python task_manager_rolling_with_updating.py whole_workflow
####### to update the models and predictions after the trading time, use the command below
# python task_manager_rolling_with_updating.py after_day
#################### you need to finish the configurations below #########################
provider_uri = "~/.qlib/qlib_data/cn_data" # data_dir
qlib.init(provider_uri=provider_uri, region=REG_CN)
C["mongo"] = {
mongo_conf = {
"task_url": "mongodb://10.0.0.4:27017/", # your MongoDB url
"task_db_name": "online", # database name
"task_db_name": "rolling_db", # database name
}
qlib.init(provider_uri=provider_uri, region=REG_CN, mongo=mongo_conf)
exp_name = "rolling_exp" # experiment name, will be used as the experiment in MLflow
task_pool = "rolling_task" # task pool name, will be used as the document in MongoDB
rolling_step = 550
##########################################################################################
rolling_gen = RollingGen(step=550, rtype=RollingGen.ROLL_SD)
rolling_gen = RollingGen(step=rolling_step, rtype=RollingGen.ROLL_SD)
rolling_online_manager = RollingOnlineManager(
experiment_name=exp_name, rolling_gen=rolling_gen, task_pool=task_pool
)

23
examples/taskmanager/update_online_pred.py → examples/online_svr/update_online_pred.py
View File

@@ -1,9 +1,9 @@
import qlib
from qlib.model.trainer import task_train
from qlib.workflow.task.online import OnlineManager
from qlib.config import REG_CN
import fire
from qlib.workflow import R
import qlib
from qlib.config import REG_CN
from qlib.model.trainer import task_train
from qlib.workflow.task.online import OnlineManagerR
from qlib.workflow.task.utils import list_recorders
data_handler_config = {
"start_time": "2008-01-01",
@@ -56,19 +56,20 @@ def first_train(experiment_name="online_svr"):
rid = task_train(task_config=task, experiment_name=experiment_name)
rom = OnlineManager(experiment_name)
rom.reset_online_model(rid)
online_manager = OnlineManagerR(experiment_name)
online_manager.reset_online_tag(rid)
def update_online_pred(experiment_name="online_svr"):
rom = OnlineManager(experiment_name)
online_manager = OnlineManagerR(experiment_name)
print("Here are the online models waiting for update:")
for rid, rec in rom.list_online_model().items():
print(rid)
for rid, rec in list_recorders(experiment_name).items():
if online_manager.get_online_tag(rec) == OnlineManagerR.ONLINE_TAG:
print(rid)
rom.update_online_pred()
online_manager.update_online_pred()
if __name__ == "__main__":