liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-02 02:21:18 +08:00
Code Activity

Optimize summarize task prompt and others (#1533)

Browse Source 
* 1.update prompt;
2.update fetch information method.

* 1.update prompt;
2.save result to markdown;

* 1.get context info from context_manager;
2.run the entire process successfully.
This commit is contained in:
Fivele-Li
2023-06-01 21:22:24 +08:00
committed by GitHub
parent 40e0c329ba
commit 3b56b8e6c0
4 changed files with 120 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
qlib/finco/task.py
View File

@@ -466,17 +466,52 @@ class SummarizeTask(Task):
__DEFAULT_OUTPUT_PATH = "./"
__DEFAULT_WORKFLOW_SYSTEM_PROMPT = """
Your task is to help user to analysis the output of qlib, your information including the strategy's backtest index
and runtime log. You may receive some scripts of the code as well, you can use them to analysis the output.
You are an expert in quant domain.
Your task is to help user to analysis the output of qlib, your main focus is on the backtesting metrics of
user strategies. Warnings reported during runtime can be ignored if deemed appropriate.
your information including the strategy's backtest log and runtime log.
You may receive some scripts of the codes as well, you can use them to analysis the output.
At the same time, you can also use your knowledge of the Microsoft/Qlib project and finance to complete your tasks.
If there are any abnormal areas in the log or scripts, please also point them out.
Example output 1:
The backtest indexes show that your strategy's max draw down is a bit large,
The matrix in log shows that your strategy's max draw down is a bit large, based on your annualized return,
your strategy has a relatively low Sharpe ratio. Here are a few suggestions:
You can try diversifying your positions across different assets.
Example output 2:
The output log shows the result of running `qlib` with `LinearModel` strategy on the Chinese stock market CSI 300
from 2008-01-01 to 2020-08-01, based on the Alpha158 data handler from 2015-01-01. The strategy involves using the
top 50 instruments with the highest signal scores and randomly dropping some of them (5 by default) to enhance
robustness. The backtesting result is shown in the table below:
| Metrics | Value |
| ------- | ----- |
| IC | 0.040 |
| ICIR | 0.312 |
| Long-Avg Ann Return | 0.093 |
| Long-Avg Ann Sharpe | 0.462 |
| Long-Short Ann Return | 0.245 |
| Long-Short Ann Sharpe | 4.098 |
| Rank IC | 0.048 |
| Rank ICIR | 0.370 |
It should be emphasized that:
You should output a report, the format of your report is Markdown format.
Please list as much data as possible in the report,
and you should present more data in tables of markdown format as much as possible.
The numbers in the report do not need to have too many significant figures.
You can add subheadings and paragraphs in Markdown for readability.
You can bold or use other formatting options to highlight keywords in the main text.
"""
__DEFAULT_WORKFLOW_USER_PROMPT = "Here is my information: '{{information}}'\n{{user_prompt}}"
__DEFAULT_USER_PROMPT = "Please summarize them and give me some advice."
# TODO: 2048 is close to exceed GPT token limit
__MAX_LENGTH_OF_FILE = 2048
__DEFAULT_REPORT_NAME = 'finCoReport.md'
def __init__(self):
super().__init__()
@@ -486,22 +521,17 @@ class SummarizeTask(Task):
system_prompt = self.__DEFAULT_WORKFLOW_SYSTEM_PROMPT
output_path = self._context_manager.get_context("output_path")
output_path = output_path if output_path is not None else self.__DEFAULT_OUTPUT_PATH
information = self.parse2txt(output_path)
file_info = self.get_info_from_file(output_path)
context_info = self.get_info_from_context()
information = context_info + file_info
prompt_workflow_selection = Template(self.__DEFAULT_WORKFLOW_USER_PROMPT).render(information=information,
user_prompt=user_prompt)
messages = [
{
"role": "system",
"content": system_prompt,
},
{
"role": "user",
"content": prompt_workflow_selection,
},
]
response = try_create_chat_completion(messages=messages)
return response
response = APIBackend().build_messages_and_create_chat_completion(user_prompt=prompt_workflow_selection,
system_prompt=system_prompt)
self.save_markdown(content=response)
return []
def summarize(self) -> str:
return ''
@@ -509,22 +539,42 @@ class SummarizeTask(Task):
def interact(self) -> Any:
return
@staticmethod
def parse2txt(path) -> List:
def get_info_from_file(self, path) -> List:
"""
read specific type of files under path
"""
file_list = []
path = Path.cwd().joinpath(path)
path = Path.cwd().joinpath(path).resolve()
for root, dirs, files in os.walk(path):
for filename in files:
file_path = os.path.join(root, filename)
print(file_path)
file_list.append(file_path)
result = []
for file in file_list:
postfix = file.split('.')[-1]
if postfix in ['txt', 'py', 'log']:
if postfix in ['py', 'log', 'yaml']:
with open(file) as f:
content = f.read()
print(content)
result.append({'postfix': postfix, 'content': content})
self.logger.info(f"file to summarize: {file}")
# in case of too large file
# TODO: Perhaps summarization method instead of truncation would be a better approach
result.append({'file': file, 'content': content[:self.__MAX_LENGTH_OF_FILE]})
return result
def get_info_from_context(self):
context = []
# TODO: get all keys from context?
for key in ["user_prompt", "chat_history", "Dataset_plan", "Model_plan", "Record_plan",
"Strategy_plan", "Backtest_plan"]:
c = self._context_manager.get_context(key=key)
if c is not None:
c = str(c)
context.append({key: c[:self.__MAX_LENGTH_OF_FILE]})
return context
def save_markdown(self, content: str):
with open(self.__DEFAULT_REPORT_NAME, "w") as f:
f.write(content)
self.logger.info(f"report has saved to {self.__DEFAULT_REPORT_NAME}")

25
qlib/finco/test_sumarize.py
View File

@@ -1,25 +0,0 @@
import unittest
from dotenv import load_dotenv
from qlib.finco.task import SummarizeTask
from qlib.finco.workflow import WorkflowContextManager
load_dotenv(verbose=True, override=True)
class TestSummarize(unittest.TestCase):
def test_execution(self):
task = SummarizeTask()
task.assign_context_manager(WorkflowContextManager())
resp = task.execution()
print(resp)
def test_parse2txt(self):
task = SummarizeTask()
resp = task.parse2txt('')
print(resp)
if __name__ == '__main__':
unittest.main()

3
qlib/finco/workflow.py
View File

@@ -104,7 +104,8 @@ class WorkflowManager:
if not cfg.continous_mode:
res = t.interact()
t.summarize()
if isinstance(t, WorkflowTask) or isinstance(t, PlanTask) or isinstance(t, ActionTask):
if isinstance(t, WorkflowTask) or isinstance(t, PlanTask) or isinstance(t, ActionTask) \
or isinstance(t, SummarizeTask):
task_list = res + task_list
else:
raise NotImplementedError("Unsupported action type")

45
tests/finco/test_sumarize.py Normal file
View File

@@ -0,0 +1,45 @@
import unittest
from dotenv import load_dotenv
# pydantic support load_dotenv, so load_dotenv will be deprecated in the future.
from qlib.finco.task import SummarizeTask
from qlib.finco.workflow import WorkflowContextManager
from qlib.finco.llm import try_create_chat_completion
load_dotenv(verbose=True, override=True)
class TestSummarize(unittest.TestCase):
def test_chat(self):
messages = [
{
"role": "system",
"content": "Your are a professional financial assistant.",
},
{
"role": "user",
"content": "How to write a perfect quant strategy.",
},
]
response = try_create_chat_completion(messages=messages)
print(response)
def test_execution(self):
task = SummarizeTask()
context = WorkflowContextManager()
context.set_context("output_path", "../../examples/benchmarks/Linear")
context.set_context("user_prompt", "My main focus is on the performance of the strategy's return."
"Please summarize the information and give me some advice.")
task.assign_context_manager(context)
resp = task.execution()
print(resp)
def test_parse2txt(self):
task = SummarizeTask()
resp = task.get_info_from_file('')
print(resp)
if __name__ == '__main__':
unittest.main()