liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-16 17:12:20 +08:00
Code Activity

Init workspace and CMDTask (#1537)

Browse Source 
* Update setup.py and config

* WIP

* init_workspace and CMDTask

* Delete test_sumarize.py
This commit is contained in:
you-n-g
2023-06-01 23:32:35 +08:00
committed by GitHub
parent 3b56b8e6c0
commit 73d51f05b4
9 changed files with 209 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
qlib/contrib/data/handler.py
View File

@@ -59,7 +59,7 @@ class Alpha360(DataHandlerLP):
fit_end_time=None, fit_end_time=None,
filter_pipe=None, filter_pipe=None,
inst_processors=None, inst_processors=None,
data_loader: Optional[dict]=None, data_loader: Optional[dict] = None,
**kwargs **kwargs
): ):
infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time) infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time)
@@ -158,7 +158,7 @@ class Alpha158(DataHandlerLP):
process_type=DataHandlerLP.PTYPE_A, process_type=DataHandlerLP.PTYPE_A,
filter_pipe=None, filter_pipe=None,
inst_processors=None, inst_processors=None,
data_loader: Optional[dict]=None, data_loader: Optional[dict] = None,
**kwargs **kwargs
): ):
infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time) infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time)

17
qlib/finco/conf.py
View File

@@ -1,15 +1,21 @@
# TODO: use pydantic for other modules in Qlib # TODO: use pydantic for other modules in Qlib
from pydantic import (BaseSettings) from pydantic import BaseSettings
from qlib.finco.utils import Singleton from qlib.finco.utils import Singleton
import os import os
class Config(Singleton): class Config(Singleton):
"""
This config is for fast demo purpose.
Please use BaseSettings insetead in the future
"""
def __init__(self): def __init__(self):
self.use_azure = os.getenv("USE_AZURE") == "True" self.use_azure = os.getenv("USE_AZURE") == "True"
self.temperature = 0.5 if os.getenv("TEMPERATURE") is None else float(os.getenv("TEMPERATURE")) self.temperature = 0.5 if os.getenv("TEMPERATURE") is None else float(os.getenv("TEMPERATURE"))
self.max_tokens = 800 if os.getenv("MAX_TOKENS") is None else int(os.getenv("MAX_TOKENS")) self.max_tokens = 800 if os.getenv("MAX_TOKENS") is None else int(os.getenv("MAX_TOKENS"))
self.openai_api_key = os.getenv("OPENAI_API_KEY") self.openai_api_key = os.getenv("OPENAI_API_KEY")
self.use_azure = os.getenv("USE_AZURE") == "True" self.use_azure = os.getenv("USE_AZURE") == "True"
self.azure_api_base = os.getenv("AZURE_API_BASE") self.azure_api_base = os.getenv("AZURE_API_BASE")
@@ -18,5 +24,8 @@ class Config(Singleton):
self.max_retry = int(os.getenv("MAX_RETRY")) if os.getenv("MAX_RETRY") is not None else None self.max_retry = int(os.getenv("MAX_RETRY")) if os.getenv("MAX_RETRY") is not None else None
self.continous_mode = os.getenv("CONTINOUS_MODE") == "True" if os.getenv("CONTINOUS_MODE") is not None else False self.continous_mode = (
self.debug_mode = os.getenv("DEBUG_MODE") == "True" if os.getenv("DEBUG_MODE") is not None else False os.getenv("CONTINOUS_MODE") == "True" if os.getenv("CONTINOUS_MODE") is not None else False
)
self.debug_mode = os.getenv("DEBUG_MODE") == "True" if os.getenv("DEBUG_MODE") is not None else False
self.workspace = os.getenv("WORKSPACE") if os.getenv("WORKSPACE") is not None else "./finco_workspace"

12
qlib/finco/llm.py
View File

@@ -23,8 +23,10 @@ class APIBackend(Singleton):
self.debug_mode = True self.debug_mode = True
cwd = os.getcwd() cwd = os.getcwd()
self.cache_file_location = os.path.join(cwd, "prompt_cache.json") self.cache_file_location = os.path.join(cwd, "prompt_cache.json")
self.cache = json.load(open(self.cache_file_location, "r")) if os.path.exists(self.cache_file_location) else {} self.cache = (
json.load(open(self.cache_file_location, "r")) if os.path.exists(self.cache_file_location) else {}
)
def build_messages_and_create_chat_completion(self, user_prompt, system_prompt=None): def build_messages_and_create_chat_completion(self, user_prompt, system_prompt=None):
"""build the messages to avoid implementing several redundant lines of code""" """build the messages to avoid implementing several redundant lines of code"""
cfg = Config() cfg = Config()
@@ -64,11 +66,11 @@ class APIBackend(Singleton):
def create_chat_completion( def create_chat_completion(
self, self,
messages, messages,
model = None, model=None,
temperature: float = None, temperature: float = None,
max_tokens: Optional[int] = None, max_tokens: Optional[int] = None,
) -> str: ) -> str:
if self.debug_mode: if self.debug_mode:
if messages[1]["content"] in self.cache: if messages[1]["content"] in self.cache:
return self.cache[messages[1]["content"]] return self.cache[messages[1]["content"]]
@@ -77,7 +79,7 @@ class APIBackend(Singleton):
temperature = self.cfg.temperature temperature = self.cfg.temperature
if max_tokens is None: if max_tokens is None:
max_tokens = self.cfg.max_tokens max_tokens = self.cfg.max_tokens
if self.cfg.use_azure: if self.cfg.use_azure:
response = openai.ChatCompletion.create( response = openai.ChatCompletion.create(
engine=self.cfg.model, engine=self.cfg.model,

202
qlib/finco/task.py
View File

@@ -6,11 +6,15 @@ from jinja2 import Template
import abc import abc
import re import re
import logging import logging
import subprocess
import platform
from qlib.log import get_module_logger from qlib.log import get_module_logger
from qlib.finco.llm import APIBackend from qlib.finco.llm import APIBackend
from qlib.finco.tpl import get_tpl_path
class Task():
class Task:
""" """
The user's intention, which was initially represented by a prompt, is achieved through a sequence of tasks. The user's intention, which was initially represented by a prompt, is achieved through a sequence of tasks.
This class doesn't have to be abstract, but it is abstract in the sense that it is not supposed to be instantiated directly because it doesn't have any implementation. This class doesn't have to be abstract, but it is abstract in the sense that it is not supposed to be instantiated directly because it doesn't have any implementation.
@@ -30,8 +34,8 @@ class Task():
def __init__(self) -> None: def __init__(self) -> None:
self._context_manager = None self._context_manager = None
self.executed = False self.executed = False
self.logger : logging.Logger = get_module_logger(f"finco.{self.__class__.__name__}") self.logger: logging.Logger = get_module_logger(f"finco.{self.__class__.__name__}")
def summarize(self) -> str: def summarize(self) -> str:
"""After the execution of the task, it is supposed to generated some context about the execution""" """After the execution of the task, it is supposed to generated some context about the execution"""
"""This function might be converted to abstract method in the future""" """This function might be converted to abstract method in the future"""
@@ -41,7 +45,7 @@ class Task():
"""assign the workflow context manager to the task""" """assign the workflow context manager to the task"""
"""then all tasks can use this context manager to share the same context""" """then all tasks can use this context manager to share the same context"""
self._context_manager = context_manager self._context_manager = context_manager
def save_chat_history_to_context_manager(self, user_input, response, system_prompt): def save_chat_history_to_context_manager(self, user_input, response, system_prompt):
chat_history = self._context_manager.get_context("chat_history") chat_history = self._context_manager.get_context("chat_history")
if chat_history is None: if chat_history is None:
@@ -63,12 +67,14 @@ class Task():
"""All sub classes should implement the interact method to determine the next task""" """All sub classes should implement the interact method to determine the next task"""
"""In continous mode, this method will not be called and the next task will be determined by the execution method only""" """In continous mode, this method will not be called and the next task will be determined by the execution method only"""
raise NotImplementedError("The interact method is not implemented, but workflow not in continous mode") raise NotImplementedError("The interact method is not implemented, but workflow not in continous mode")
class WorkflowTask(Task): class WorkflowTask(Task):
"""This task is supposed to be the first task of the workflow""" """This task is supposed to be the first task of the workflow"""
def __init__(self,) -> None:
def __init__(
self,
) -> None:
super().__init__() super().__init__()
self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT = """ self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT = """
Your task is to determine the workflow in Qlib (supervised learning or reinforcement learning) ensuring the workflow can meet the user's requirements. Your task is to determine the workflow in Qlib (supervised learning or reinforcement learning) ensuring the workflow can meet the user's requirements.
@@ -94,14 +100,18 @@ workflow: reinforcement learning
"Response only with the output in the exact format specified in the system prompt, with no explanation or conversation.\n" "Response only with the output in the exact format specified in the system prompt, with no explanation or conversation.\n"
) )
def execute(self,) -> List[Task]: def execute(
self,
) -> List[Task]:
"""make the choice which main workflow (RL, SL) will be used""" """make the choice which main workflow (RL, SL) will be used"""
user_prompt = self._context_manager.get_context("user_prompt") user_prompt = self._context_manager.get_context("user_prompt")
prompt_workflow_selection = Template( prompt_workflow_selection = Template(self.__DEFAULT_WORKFLOW_USER_PROMPT).render(user_prompt=user_prompt)
self.__DEFAULT_WORKFLOW_USER_PROMPT response = APIBackend().build_messages_and_create_chat_completion(
).render(user_prompt=user_prompt) prompt_workflow_selection, self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT
response = APIBackend().build_messages_and_create_chat_completion(prompt_workflow_selection, self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT) )
self.save_chat_history_to_context_manager(prompt_workflow_selection, response, self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT) self.save_chat_history_to_context_manager(
prompt_workflow_selection, response, self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT
)
workflow = response.split(":")[1].strip().lower() workflow = response.split(":")[1].strip().lower()
self.executed = True self.executed = True
self._context_manager.set_context("workflow", workflow) self._context_manager.set_context("workflow", workflow)
@@ -111,7 +121,7 @@ workflow: reinforcement learning
return [RLPlanTask()] return [RLPlanTask()]
else: else:
raise ValueError(f"The workflow: {workflow} is not supported") raise ValueError(f"The workflow: {workflow} is not supported")
def interact(self) -> Any: def interact(self) -> Any:
assert self.executed == True, "The workflow task has not been executed yet" assert self.executed == True, "The workflow task has not been executed yet"
## TODO use logger ## TODO use logger
@@ -132,13 +142,16 @@ workflow: reinforcement learning
else: else:
# TODO add self feedback # TODO add self feedback
raise ValueError("The input cannot be interpreted as a valid input") raise ValueError("The input cannot be interpreted as a valid input")
class PlanTask(Task): class PlanTask(Task):
pass pass
class SLPlanTask(PlanTask): class SLPlanTask(PlanTask):
def __init__(self,) -> None: def __init__(
self,
) -> None:
super().__init__() super().__init__()
self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT = """ self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT = """
Your task is to determine the 5 crucial components in Qlib (Dataset, Model, Record, Strategy, Backtest) ensuring the workflow can meet the user's requirements. Your task is to determine the 5 crucial components in Qlib (Dataset, Model, Record, Strategy, Backtest) ensuring the workflow can meet the user's requirements.
@@ -174,22 +187,29 @@ components:
user_prompt = self._context_manager.get_context("user_prompt") user_prompt = self._context_manager.get_context("user_prompt")
assert user_prompt is not None, "The user prompt is not provided" assert user_prompt is not None, "The user prompt is not provided"
prompt_plan_all = Template( prompt_plan_all = Template(self.__DEFAULT_WORKFLOW_USER_PROMPT).render(user_prompt=user_prompt)
self.__DEFAULT_WORKFLOW_USER_PROMPT response = APIBackend().build_messages_and_create_chat_completion(
).render(user_prompt=user_prompt) prompt_plan_all, self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT
response = APIBackend().build_messages_and_create_chat_completion(prompt_plan_all, self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT) )
self.save_chat_history_to_context_manager(prompt_plan_all, response, self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT) self.save_chat_history_to_context_manager(prompt_plan_all, response, self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT)
if "components" not in response: if "components" not in response:
self.logger.warning("The response is not in the correct format, which probably means the answer is not correct") self.logger.warning(
"The response is not in the correct format, which probably means the answer is not correct"
)
regex_dict = { regex_dict = {
"Dataset":re.compile("Dataset: \((.*?)\) (.*?)\n"), "Dataset": re.compile("Dataset: \((.*?)\) (.*?)\n"),
"Model":re.compile("Model: \((.*?)\) (.*?)\n"), "Model": re.compile("Model: \((.*?)\) (.*?)\n"),
"Record":re.compile("Record: \((.*?)\) (.*?)\n"), "Record": re.compile("Record: \((.*?)\) (.*?)\n"),
"Strategy":re.compile("Strategy: \((.*?)\) (.*?)\n"), "Strategy": re.compile("Strategy: \((.*?)\) (.*?)\n"),
"Backtest":re.compile("Backtest: \((.*?)\) (.*?)$"), "Backtest": re.compile("Backtest: \((.*?)\) (.*?)$"),
} }
new_task = [] new_task = []
# 1) create a workspace
# TODO: we have to make choice between `sl` and `sl-cfg`
new_task.append(CMDTask(cmd_intention=f"Copy folder from {get_tpl_path() / 'sl'} to {self._context_manager.get_context('workspace')}"))
# 2) CURD on the workspace
for name, regex in regex_dict.items(): for name, regex in regex_dict.items():
res = re.search(regex, response) res = re.search(regex, response)
if not res: if not res:
@@ -203,9 +223,12 @@ components:
elif res.group(1) == "Personized": elif res.group(1) == "Personized":
new_task.extend([ConfigActionTask(name), ImplementActionTask(name)]) new_task.extend([ConfigActionTask(name), ImplementActionTask(name)])
return new_task return new_task
class RLPlanTask(PlanTask): class RLPlanTask(PlanTask):
def __init__(self,) -> None: def __init__(
self,
) -> None:
super().__init__() super().__init__()
self.logger.error("The RL task is not implemented yet") self.logger.error("The RL task is not implemented yet")
exit() exit()
@@ -221,6 +244,45 @@ class RLPlanTask(PlanTask):
class ActionTask(Task): class ActionTask(Task):
pass pass
class CMDTask(ActionTask):
"""
This CMD task is responsible for ensuring compatibility across different operating systems.
"""
__DEFAULT_WORKFLOW_SYSTEM_PROMPT = """
You are an expert system administrator.
Your task is to convert the user's intention into a specific runnable command for a particular system.
Example input:
- User intention: Copy the folder from a/b/c to d/e/f
- User OS: Linux
Example output:
cp -r a/b/c d/e/f
"""
__DEFAULT_WORKFLOW_USER_PROMPT = """
Example input:
- User intention: "{{cmd_intention}}"
- User OS: "{{user_os}}"
Example output:
"""
def __init__(self, cmd_intention: str, cwd=None):
self.cwd = cwd
self.cmd_intention = cmd_intention
self._output = None
def execute(self):
prompt = Template(self.__DEFAULT_WORKFLOW_USER_PROMPT).render(cmd_intention=self.cmd_intention,
user_os=platform.system())
response = APIBackend().build_messages_and_create_chat_completion(prompt, self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT)
self._output = subprocess.check_output(response, shell=True, cwd=self.cwd)
return []
def summarize(self):
if self._output is not None:
# TODO: it will be overrides by later commands
self._context_manager.set_context(self.__class__.__name__, self._output.decode("utf8"))
class ConfigActionTask(ActionTask): class ConfigActionTask(ActionTask):
def __init__(self, component) -> None: def __init__(self, component) -> None:
super().__init__() super().__init__()
@@ -268,8 +330,7 @@ Reason: I choose the hyperparameters above because they are the default hyperpar
Improve suggestion: You can try to tune the num_leaves in range [100, 300], max_depth in [5, 10], learning_rate in [0.01, 1] and other hyperparameters in the config. Since you're trying to get a long tern return, if you have enough computation resource, you can try to use a larger num_leaves and max_depth and a smaller learning_rate. Improve suggestion: You can try to tune the num_leaves in range [100, 300], max_depth in [5, 10], learning_rate in [0.01, 1] and other hyperparameters in the config. Since you're trying to get a long tern return, if you have enough computation resource, you can try to use a larger num_leaves and max_depth and a smaller learning_rate.
""" """
self.__CONFIG_ACTION_SYSTEM_PROMPT_TEMPLATE = ( self.__CONFIG_ACTION_SYSTEM_PROMPT_TEMPLATE = """
"""
user requirement: {{user_requirement}} user requirement: {{user_requirement}}
user plan: user plan:
- Dataset: ({{dataset_decision}}) {{dataset_plan}} - Dataset: ({{dataset_decision}}) {{dataset_plan}}
@@ -279,7 +340,6 @@ user plan:
- Backtest: ({{backtest_decision}}) {{backtest_plan}} - Backtest: ({{backtest_decision}}) {{backtest_plan}}
target component: {{target_component}} target component: {{target_component}}
""" """
)
def execute(self): def execute(self):
user_prompt = self._context_manager.get_context("user_prompt") user_prompt = self._context_manager.get_context("user_prompt")
@@ -288,7 +348,7 @@ target component: {{target_component}}
for component in component_list: for component in component_list:
prompt_element_dict[f"{component}_decision"] = self._context_manager.get_context(f"{component}_decision") prompt_element_dict[f"{component}_decision"] = self._context_manager.get_context(f"{component}_decision")
prompt_element_dict[f"{component}_plan"] = self._context_manager.get_context(f"{component}_plan") prompt_element_dict[f"{component}_plan"] = self._context_manager.get_context(f"{component}_plan")
assert None not in prompt_element_dict.values(), "Some decision or plan is not set by plan maker" assert None not in prompt_element_dict.values(), "Some decision or plan is not set by plan maker"
config_prompt = Template(self.__CONFIG_ACTION_SYSTEM_PROMPT_TEMPLATE).render( config_prompt = Template(self.__CONFIG_ACTION_SYSTEM_PROMPT_TEMPLATE).render(
@@ -303,12 +363,16 @@ target component: {{target_component}}
strategy_plan=prompt_element_dict["Strategy_plan"], strategy_plan=prompt_element_dict["Strategy_plan"],
backtest_decision=prompt_element_dict["Backtest_decision"], backtest_decision=prompt_element_dict["Backtest_decision"],
backtest_plan=prompt_element_dict["Backtest_plan"], backtest_plan=prompt_element_dict["Backtest_plan"],
target_component=self.target_componet target_component=self.target_componet,
)
response = APIBackend().build_messages_and_create_chat_completion(
config_prompt, self.__DEFAULT_CONFIG_ACTION_SYSTEM_PROMPT
) )
response = APIBackend().build_messages_and_create_chat_completion(config_prompt, self.__DEFAULT_CONFIG_ACTION_SYSTEM_PROMPT)
self.save_chat_history_to_context_manager(config_prompt, response, self.__DEFAULT_CONFIG_ACTION_SYSTEM_PROMPT) self.save_chat_history_to_context_manager(config_prompt, response, self.__DEFAULT_CONFIG_ACTION_SYSTEM_PROMPT)
res = re.search(r"Config:(.*)Reason:(.*)Improve suggestion:(.*)", response, re.S) res = re.search(r"Config:(.*)Reason:(.*)Improve suggestion:(.*)", response, re.S)
assert res is not None and len(res.groups()) == 3, "The response of config action task is not in the correct format" assert (
res is not None and len(res.groups()) == 3
), "The response of config action task is not in the correct format"
config = re.search(r"```yaml(.*)```", res.group(1), re.S) config = re.search(r"```yaml(.*)```", res.group(1), re.S)
assert config is not None, "The config part of config action task response is not in the correct format" assert config is not None, "The config part of config action task response is not in the correct format"
@@ -396,7 +460,7 @@ dataset:
csv_path: path/to/your/csv/data csv_path: path/to/your/csv/data
``` ```
""" """
self.__DEFAULT_IMPLEMENT_ACTION_USER_PROMPT = (""" self.__DEFAULT_IMPLEMENT_ACTION_USER_PROMPT = """
user requirement: {{user_requirement}} user requirement: {{user_requirement}}
user plan: user plan:
- Dataset: ({{dataset_decision}}) {{dataset_plan}} - Dataset: ({{dataset_decision}}) {{dataset_plan}}
@@ -409,8 +473,8 @@ User config:
{{user_config}} {{user_config}}
``` ```
target component: {{target_component}} target component: {{target_component}}
""") """
def execute(self): def execute(self):
""" """
return a list of interested tasks return a list of interested tasks
@@ -423,7 +487,7 @@ target component: {{target_component}}
for component in component_list: for component in component_list:
prompt_element_dict[f"{component}_decision"] = self._context_manager.get_context(f"{component}_decision") prompt_element_dict[f"{component}_decision"] = self._context_manager.get_context(f"{component}_decision")
prompt_element_dict[f"{component}_plan"] = self._context_manager.get_context(f"{component}_plan") prompt_element_dict[f"{component}_plan"] = self._context_manager.get_context(f"{component}_plan")
assert None not in prompt_element_dict.values(), "Some decision or plan is not set by plan maker" assert None not in prompt_element_dict.values(), "Some decision or plan is not set by plan maker"
config = self._context_manager.get_context(f"{self.target_component}_config") config = self._context_manager.get_context(f"{self.target_component}_config")
@@ -440,20 +504,28 @@ target component: {{target_component}}
backtest_decision=prompt_element_dict["Backtest_decision"], backtest_decision=prompt_element_dict["Backtest_decision"],
backtest_plan=prompt_element_dict["Backtest_plan"], backtest_plan=prompt_element_dict["Backtest_plan"],
target_component=self.target_component, target_component=self.target_component,
user_config=config user_config=config,
)
response = APIBackend().build_messages_and_create_chat_completion(
implement_prompt, self.__DEFAULT_IMPLEMENT_ACTION_SYSTEM_PROMPT
)
self.save_chat_history_to_context_manager(
implement_prompt, response, self.__DEFAULT_IMPLEMENT_ACTION_SYSTEM_PROMPT
) )
response = APIBackend().build_messages_and_create_chat_completion(implement_prompt, self.__DEFAULT_IMPLEMENT_ACTION_SYSTEM_PROMPT)
self.save_chat_history_to_context_manager(implement_prompt, response, self.__DEFAULT_IMPLEMENT_ACTION_SYSTEM_PROMPT)
res = re.search(r"Code:(.*)Explanation:(.*)Modified config:(.*)", response, re.S) res = re.search(r"Code:(.*)Explanation:(.*)Modified config:(.*)", response, re.S)
assert res is not None and len(res.groups()) == 3, f"The response of implement action task of component {self.target_component} is not in the correct format" assert (
res is not None and len(res.groups()) == 3
), f"The response of implement action task of component {self.target_component} is not in the correct format"
code = re.search(r"```python(.*)```", res.group(1), re.S) code = re.search(r"```python(.*)```", res.group(1), re.S)
assert code is not None, "The code part of implementation action task response is not in the correct format" assert code is not None, "The code part of implementation action task response is not in the correct format"
code = code.group(1) code = code.group(1)
explanation = res.group(2) explanation = res.group(2)
modified_config = re.search(r"```yaml(.*)```", res.group(3), re.S) modified_config = re.search(r"```yaml(.*)```", res.group(3), re.S)
assert modified_config is not None, "The modified config part of implementation action task response is not in the correct format" assert (
modified_config is not None
), "The modified config part of implementation action task response is not in the correct format"
modified_config = modified_config.group(1) modified_config = modified_config.group(1)
self._context_manager.set_context(f"{self.target_component}_code", code) self._context_manager.set_context(f"{self.target_component}_code", code)
@@ -462,8 +534,9 @@ target component: {{target_component}}
return [] return []
class SummarizeTask(Task): class SummarizeTask(Task):
__DEFAULT_OUTPUT_PATH = "./" __DEFAULT_WORKSPACE = "./"
__DEFAULT_WORKFLOW_SYSTEM_PROMPT = """ __DEFAULT_WORKFLOW_SYSTEM_PROMPT = """
You are an expert in quant domain. You are an expert in quant domain.
@@ -510,7 +583,7 @@ class SummarizeTask(Task):
# TODO: 2048 is close to exceed GPT token limit # TODO: 2048 is close to exceed GPT token limit
__MAX_LENGTH_OF_FILE = 2048 __MAX_LENGTH_OF_FILE = 2048
__DEFAULT_REPORT_NAME = 'finCoReport.md' __DEFAULT_REPORT_NAME = "finCoReport.md"
def __init__(self): def __init__(self):
super().__init__() super().__init__()
@@ -519,22 +592,24 @@ class SummarizeTask(Task):
user_prompt = self._context_manager.get_context("user_prompt") user_prompt = self._context_manager.get_context("user_prompt")
user_prompt = user_prompt if user_prompt is not None else self.__DEFAULT_USER_PROMPT user_prompt = user_prompt if user_prompt is not None else self.__DEFAULT_USER_PROMPT
system_prompt = self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT system_prompt = self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT
output_path = self._context_manager.get_context("output_path") workspace = self._context_manager.get_context("workspace")
output_path = output_path if output_path is not None else self.__DEFAULT_OUTPUT_PATH workspace = workspace if workspace is not None else self.__DEFAULT_WORKSPACE
file_info = self.get_info_from_file(output_path) file_info = self.get_info_from_file(workspace)
context_info = self.get_info_from_context() context_info = self.get_info_from_context()
information = context_info + file_info information = context_info + file_info
prompt_workflow_selection = Template(self.__DEFAULT_WORKFLOW_USER_PROMPT).render(information=information, prompt_workflow_selection = Template(self.__DEFAULT_WORKFLOW_USER_PROMPT).render(
user_prompt=user_prompt) information=information, user_prompt=user_prompt
)
response = APIBackend().build_messages_and_create_chat_completion(user_prompt=prompt_workflow_selection, response = APIBackend().build_messages_and_create_chat_completion(
system_prompt=system_prompt) user_prompt=prompt_workflow_selection, system_prompt=system_prompt
)
self.save_markdown(content=response) self.save_markdown(content=response)
return [] return []
def summarize(self) -> str: def summarize(self) -> str:
return '' return ""
def interact(self) -> Any: def interact(self) -> Any:
return return
@@ -552,26 +627,33 @@ class SummarizeTask(Task):
result = [] result = []
for file in file_list: for file in file_list:
postfix = file.split('.')[-1] postfix = file.split(".")[-1]
if postfix in ['py', 'log', 'yaml']: if postfix in ["py", "log", "yaml"]:
with open(file) as f: with open(file) as f:
content = f.read() content = f.read()
self.logger.info(f"file to summarize: {file}") self.logger.info(f"file to summarize: {file}")
# in case of too large file # in case of too large file
# TODO: Perhaps summarization method instead of truncation would be a better approach # TODO: Perhaps summarization method instead of truncation would be a better approach
result.append({'file': file, 'content': content[:self.__MAX_LENGTH_OF_FILE]}) result.append({"file": file, "content": content[: self.__MAX_LENGTH_OF_FILE]})
return result return result
def get_info_from_context(self): def get_info_from_context(self):
context = [] context = []
# TODO: get all keys from context? # TODO: get all keys from context?
for key in ["user_prompt", "chat_history", "Dataset_plan", "Model_plan", "Record_plan", for key in [
"Strategy_plan", "Backtest_plan"]: "user_prompt",
"chat_history",
"Dataset_plan",
"Model_plan",
"Record_plan",
"Strategy_plan",
"Backtest_plan",
]:
c = self._context_manager.get_context(key=key) c = self._context_manager.get_context(key=key)
if c is not None: if c is not None:
c = str(c) c = str(c)
context.append({key: c[:self.__MAX_LENGTH_OF_FILE]}) context.append({key: c[: self.__MAX_LENGTH_OF_FILE]})
return context return context
def save_markdown(self, content: str): def save_markdown(self, content: str):

1
qlib/finco/tpl/__init__.py
View File

@@ -1,6 +1,7 @@
# Copyright (c) Microsoft Corporation. # Copyright (c) Microsoft Corporation.
# Licensed under the MIT License. # Licensed under the MIT License.
from pathlib import Path from pathlib import Path
DIRNAME = Path(__file__).absolute().resolve().parent DIRNAME = Path(__file__).absolute().resolve().parent

13
qlib/finco/utils.py
View File

@@ -1,12 +1,13 @@
import json import json
class Singleton(): class Singleton:
_instance = None _instance = None
def __new__(cls, *args, **kwargs):
if cls._instance is None: def __new__(cls, *args, **kwargs):
cls._instance = super().__new__(cls, *args, **kwargs) if cls._instance is None:
return cls._instance cls._instance = super().__new__(cls, *args, **kwargs)
return cls._instance
def parse_json(response): def parse_json(response):
@@ -15,4 +16,4 @@ def parse_json(response):
except json.decoder.JSONDecodeError: except json.decoder.JSONDecodeError:
pass pass
raise Exception(f"Failed to parse response: {response}, please report it or help us to fix it.") raise Exception(f"Failed to parse response: {response}, please report it or help us to fix it.")

39
qlib/finco/workflow.py
View File

@@ -1,5 +1,7 @@
import sys
import copy import copy
from pathlib import Path from pathlib import Path
import shutil
from qlib.log import get_module_logger from qlib.log import get_module_logger
from qlib.finco.conf import Config from qlib.finco.conf import Config
@@ -18,9 +20,7 @@ class WorkflowContextManager:
def set_context(self, key, value): def set_context(self, key, value):
if key in self.context: if key in self.context:
self.logger.warning( self.logger.warning("The key already exists in the context, the value will be overwritten")
"The key already exists in the context, the value will be overwritten"
)
self.context[key] = value self.context[key] = value
def get_context(self, key): def get_context(self, key):
@@ -45,14 +45,32 @@ class WorkflowContextManager:
class WorkflowManager: class WorkflowManager:
"""This manange the whole task automation workflow including tasks and actions""" """This manange the whole task automation workflow including tasks and actions"""
def __init__(self, name="project", output_path=None) -> None: def __init__(self, workspace=None) -> None:
if output_path is None: if workspace is None:
self._output_path = Path.cwd() / name self._workspace = Path.cwd() / "finco_workspace"
else: else:
self._output_path = Path(output_path) self._workspace = Path(workspace)
self._confirm_and_rm()
self._context = WorkflowContextManager() self._context = WorkflowContextManager()
self._context.set_context("workspace", self._workspace)
self.default_user_prompt = "Please help me build a low turnover strategy that focus more on longterm return in China a stock market. I want to construct a new dataset covers longer history" self.default_user_prompt = "Please help me build a low turnover strategy that focus more on longterm return in China a stock market. I want to construct a new dataset covers longer history"
def _confirm_and_rm(self):
# if workspace exists, please confirm and remove it. Otherwise exit.
if self._workspace.exists():
flag = input(
f"Will be deleted: "
f"\n\t{self._workspace}"
f"\nIf you do not need to delete {self._workspace}, please change the workspace dir or rename existing files "
f"\nAre you sure you want to delete, yes(Y/y), no (N/n):"
)
if str(flag) not in ["Y", "y"]:
sys.exit()
else:
# remove self._workspace
shutil.rmtree(self._workspace)
def set_context(self, key, value): def set_context(self, key, value):
"""Direct call set_context method of the context manager""" """Direct call set_context method of the context manager"""
self._context.set_context(key, value) self._context.set_context(key, value)
@@ -104,9 +122,8 @@ class WorkflowManager:
if not cfg.continous_mode: if not cfg.continous_mode:
res = t.interact() res = t.interact()
t.summarize() t.summarize()
if isinstance(t, WorkflowTask) or isinstance(t, PlanTask) or isinstance(t, ActionTask) \ if isinstance(t, (WorkflowTask, PlanTask, ActionTask, SummarizeTask)):
or isinstance(t, SummarizeTask):
task_list = res + task_list task_list = res + task_list
else: else:
raise NotImplementedError("Unsupported action type") raise NotImplementedError(f"Unsupported Task type {t}")
return self._output_path return self._workspace

5
scripts/finco/README.md
View File

@@ -2,9 +2,10 @@
# Requirements # Requirements
Use following install command to complete the project.
``` ```
pydantic pip install -e '.[finco]'
openai
``` ```

6
setup.py
View File

@@ -173,6 +173,12 @@ setup(
"tianshou<=0.4.10", "tianshou<=0.4.10",
"torch", "torch",
], ],
"finco": [
# finco is not necessary for all Qlib users; So a single require section is used for it.
"openapi",
"pydantic", # Please add it to basic requirements after the design of pydantic is state.
"python-dotenv", # I don't think this is necessary if we use pydantic.
],
}, },
include_package_data=True, include_package_data=True,
classifiers=[ classifiers=[