liuyu/qlib
1
0
Fork 0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-16 17:12:20 +08:00
Code Activity

Connect TrainTask & Rolling & DDG-DA (#1599)

Browse Source 
* Connect train task to ddg-da & rolling

* Pylint & black formatting

* Formatting
This commit is contained in:
you-n-g
2023-07-17 09:58:58 +08:00
committed by GitHub
parent 5e0873ca81
commit 1c9841b15e
14 changed files with 584 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
qlib/finco/.env.example
View File

@@ -15,4 +15,6 @@ MAX_TOKENS=1600
MAX_RETRY=120 MAX_RETRY=120
CONTINOUS_MODE=True CONTINOUS_MODE=True
DEBUG_MODE=True DEBUG_MODE=True
# TEMPERATURE=

2
qlib/finco/__init__.py
View File

@@ -10,4 +10,4 @@ def get_finco_path() -> Path:
return the template path return the template path
Because the template path is located in the folder. We don't know where it is located. So __file__ for this module will be used. Because the template path is located in the folder. We don't know where it is located. So __file__ for this module will be used.
""" """
return DIRNAME return DIRNAME

2
qlib/finco/conf.py
View File

@@ -1,5 +1,5 @@
# TODO: use pydantic for other modules in Qlib # TODO: use pydantic for other modules in Qlib
from pydantic import BaseSettings # from pydantic_settings import BaseSettings
from qlib.finco.utils import SingletonBaseClass from qlib.finco.utils import SingletonBaseClass
import os import os

97
qlib/finco/context.py Normal file
View File

@@ -0,0 +1,97 @@
from dataclasses import dataclass, field
import copy
from pathlib import Path
from typing import Optional, List
from qlib.finco.log import FinCoLog
from qlib.typehint import Literal
from qlib.finco.utils import similarity
@dataclass
class Design:
plan: str
classes: str
decision: str
@dataclass
class Exp:
"""Experiment"""
# compoments
dataset: Optional[Design] = None
datahandler: Optional[Design] = None
model: Optional[Design] = None
record: Optional[Design] = None
strategy: Optional[Design] = None
backtest: Optional[Design] = None
# basic
template: Optional[Path] = None
# rolling strategy. None indicates no rolling
rolling: Optional[Literal["base", "ddgda"]] = None
@dataclass
class StructContext:
"""Part of the context have clear meaning and structure, so they will be saved here and can be easily retrieved and understood"""
# TODO: move more content in WorkflowContextManager.context to here
workspace: Path
exp_list: List[Exp] = field(default_factory=list) # the planned experiments
class WorkflowContextManager:
"""Context Manager stores the context of the workflow"""
"""All context are key value pairs which saves the input, output and status of the whole workflow"""
def __init__(self, workspace: Path) -> None:
self.context = {}
self.logger = FinCoLog()
# this context is public
self.struct_context = StructContext(workspace) # TODO: move more content in context to here
self.set_context("workspace", workspace) # TODO: remove me
def set_context(self, key, value):
if key in self.context:
self.logger.warning("The key already exists in the context, the value will be overwritten")
self.context[key] = value
def get_context(self, key):
# NOTE: if the key doesn't exist, return None. In the future, we may raise an error to detect abnormal behavior
if key not in self.context:
self.logger.warning("The key doesn't exist in the context")
return None
return self.context[key]
def update_context(self, key, new_value):
# NOTE: if the key doesn't exist, return None. In the future, we may raise an error to detect abnormal behavior
if key not in self.context:
self.logger.warning("The key doesn't exist in the context")
self.context.update({key: new_value})
def get_all_context(self):
"""return a deep copy of the context"""
"""TODO: do we need to return a deep copy?"""
return copy.deepcopy(self.context)
def retrieve(self, query: str) -> dict:
if query in self.context.keys():
return {query: self.context.get(query)}
# Note: retrieve information from context by string similarity maybe abandon in future
scores = {}
for k, v in self.context.items():
scores.update({k: max(similarity(query, k), similarity(query, v))})
max_score_key = max(scores, key=scores.get)
return {max_score_key: self.context.get(max_score_key)}
def clear(self, reserve: list = None):
if reserve is None:
reserve = []
_context = {k: self.get_context(k) for k in reserve}
self.context = _context

48
qlib/finco/knowledge.py
View File

@@ -65,6 +65,7 @@ class YamlStorage(Storage):
This class is responsible for storage and loading of Knowledge related data in yaml format. This class is responsible for storage and loading of Knowledge related data in yaml format.
""" """
DEFAULT_NAME = "storage.yml" DEFAULT_NAME = "storage.yml"
def __init__(self, path: Union[str, Path]): def __init__(self, path: Union[str, Path]):
@@ -82,7 +83,7 @@ class YamlStorage(Storage):
def save(self, **kwargs): def save(self, **kwargs):
"""use pickle as the default save method""" """use pickle as the default save method"""
Path.mkdir(self.path.parent, exist_ok=True) Path.mkdir(self.path.parent, exist_ok=True)
with open(self.path, 'w') as f: with open(self.path, "w") as f:
yaml.dump(self.documents, f) yaml.dump(self.documents, f)
@@ -190,9 +191,14 @@ class ExperimentKnowledge(Knowledge):
def brief(self): def brief(self):
docs = [] docs = []
for recorder in self.storage.recs: for recorder in self.storage.recs:
docs.append({"exp_name": self.storage.exp.name, "record_info": recorder.info, docs.append(
"config": recorder.load_object("config"), {
"context_summary": recorder.load_object("context_summary")}) "exp_name": self.storage.exp.name,
"record_info": recorder.info,
"config": recorder.load_object("config"),
"context_summary": recorder.load_object("context_summary"),
}
)
return docs return docs
@@ -295,7 +301,7 @@ class InfrastructureKnowledge(Knowledge):
""" """
functions = [] functions = []
for py_file_path in Path(directory).rglob('*.py'): for py_file_path in Path(directory).rglob("*.py"):
for _functions in self.get_functions_with_docstrings(py_file_path): for _functions in self.get_functions_with_docstrings(py_file_path):
functions.append(_functions) functions.append(_functions)
@@ -314,7 +320,7 @@ class InfrastructureKnowledge(Knowledge):
docstring = None docstring = None
for line in lines: for line in lines:
if line.strip().startswith("def ") or line.strip().startswith("class "): if line.strip().startswith("def ") or line.strip().startswith("class "):
func = line.strip().split(' ')[1].split('(')[0] func = line.strip().split(" ")[1].split("(")[0]
if func.startswith("__"): if func.startswith("__"):
continue continue
if current_func is not None: if current_func is not None:
@@ -336,7 +342,6 @@ class InfrastructureKnowledge(Knowledge):
class Topic: class Topic:
def __init__(self, name: str, describe: Template): def __init__(self, name: str, describe: Template):
self.name = name self.name = name
self.describe = describe self.describe = describe
@@ -347,9 +352,7 @@ class Topic:
def summarize(self, docs: list): def summarize(self, docs: list):
self.logger.info(f"Summarize topic: \nname: {self.name}\ndescribe: {self.describe.module}") self.logger.info(f"Summarize topic: \nname: {self.name}\ndescribe: {self.describe.module}")
prompt_workflow_selection = self.describe.render(docs=docs) prompt_workflow_selection = self.describe.render(docs=docs)
response = APIBackend().build_messages_and_create_chat_completion( response = APIBackend().build_messages_and_create_chat_completion(user_prompt=prompt_workflow_selection)
user_prompt=prompt_workflow_selection
)
self.knowledge = response self.knowledge = response
self.docs = docs self.docs = docs
@@ -395,22 +398,26 @@ class KnowledgeBase:
def load_practice_knowledge(self, path: Path) -> PracticeKnowledge: def load_practice_knowledge(self, path: Path) -> PracticeKnowledge:
self.practice_knowledge = PracticeKnowledge( self.practice_knowledge = PracticeKnowledge(
YamlStorage(path.joinpath(f"{self.KT_PRACTICE}/{YamlStorage.DEFAULT_NAME}"))) YamlStorage(path.joinpath(f"{self.KT_PRACTICE}/{YamlStorage.DEFAULT_NAME}"))
)
return self.practice_knowledge return self.practice_knowledge
def load_execute_knowledge(self, path: Path) -> ExecuteKnowledge: def load_execute_knowledge(self, path: Path) -> ExecuteKnowledge:
self.execute_knowledge = ExecuteKnowledge( self.execute_knowledge = ExecuteKnowledge(
YamlStorage(path.joinpath(f"{self.KT_EXECUTE}/{YamlStorage.DEFAULT_NAME}"))) YamlStorage(path.joinpath(f"{self.KT_EXECUTE}/{YamlStorage.DEFAULT_NAME}"))
)
return self.execute_knowledge return self.execute_knowledge
def load_finance_knowledge(self, path: Path) -> FinanceKnowledge: def load_finance_knowledge(self, path: Path) -> FinanceKnowledge:
self.finance_knowledge = FinanceKnowledge( self.finance_knowledge = FinanceKnowledge(
YamlStorage(path.joinpath(f"{self.KT_FINANCE}/{YamlStorage.DEFAULT_NAME}"))) YamlStorage(path.joinpath(f"{self.KT_FINANCE}/{YamlStorage.DEFAULT_NAME}"))
)
return self.finance_knowledge return self.finance_knowledge
def load_infrastructure_knowledge(self, path: Path) -> InfrastructureKnowledge: def load_infrastructure_knowledge(self, path: Path) -> InfrastructureKnowledge:
self.infrastructure_knowledge = InfrastructureKnowledge( self.infrastructure_knowledge = InfrastructureKnowledge(
YamlStorage(path.joinpath(f"{self.KT_INFRASTRUCTURE}/{YamlStorage.DEFAULT_NAME}"))) YamlStorage(path.joinpath(f"{self.KT_INFRASTRUCTURE}/{YamlStorage.DEFAULT_NAME}"))
)
return self.infrastructure_knowledge return self.infrastructure_knowledge
def get_knowledge(self, knowledge_type: str = None): def get_knowledge(self, knowledge_type: str = None):
@@ -423,8 +430,12 @@ class KnowledgeBase:
elif knowledge_type == self.KT_INFRASTRUCTURE: elif knowledge_type == self.KT_INFRASTRUCTURE:
knowledge = self.infrastructure_knowledge.knowledge knowledge = self.infrastructure_knowledge.knowledge
else: else:
knowledge = self.execute_knowledge.knowledge + self.practice_knowledge.knowledge \ knowledge = (
+ self.finance_knowledge.knowledge + self.infrastructure_knowledge.knowledge self.execute_knowledge.knowledge
+ self.practice_knowledge.knowledge
+ self.finance_knowledge.knowledge
+ self.infrastructure_knowledge.knowledge
)
return knowledge return knowledge
def query(self, knowledge_type: str = None, content: str = None, n: int = 5): def query(self, knowledge_type: str = None, content: str = None, n: int = 5):
@@ -450,8 +461,9 @@ class KnowledgeBase:
similar_n_docs = [knowledge[i] for i in similar_n_indexes] similar_n_docs = [knowledge[i] for i in similar_n_indexes]
prompt = Template( prompt = Template(
"""find the most relevant doc with this query: '{{content}}' from docs='{{docs}}'. """find the most relevant doc with this query: '{{content}}' from docs='{{docs}}'.
Just return the most relevant item I provided, no more explain. For example: {'function': 'config.resolve_path', 'docstring': None}""") Just return the most relevant item I provided, no more explain. For example: {'function': 'config.resolve_path', 'docstring': None}"""
)
prompt_workflow_selection = prompt.render(content=content, docs=similar_n_docs) prompt_workflow_selection = prompt.render(content=content, docs=similar_n_docs)
response = APIBackend().build_messages_and_create_chat_completion( response = APIBackend().build_messages_and_create_chat_completion(
user_prompt=prompt_workflow_selection, system_prompt="You are an excellent assistant." user_prompt=prompt_workflow_selection, system_prompt="You are an excellent assistant."

3
qlib/finco/llm.py
View File

@@ -43,7 +43,7 @@ class APIBackend(SingletonBaseClass):
"content": system_prompt, "content": system_prompt,
} }
] ]
messages.extend(former_messages[-1*cfg.max_past_message_include:]) messages.extend(former_messages[-1 * cfg.max_past_message_include :])
messages.append( messages.append(
{ {
"role": "user", "role": "user",
@@ -82,7 +82,6 @@ class APIBackend(SingletonBaseClass):
temperature: float = None, temperature: float = None,
max_tokens: Optional[int] = None, max_tokens: Optional[int] = None,
) -> str: ) -> str:
if self.debug_mode: if self.debug_mode:
key = json.dumps(messages) key = json.dumps(messages)
if key in self.cache: if key in self.cache:

23
qlib/finco/log.py
View File

@@ -12,6 +12,7 @@ class LogColors:
""" """
ANSI color codes for use in console output. ANSI color codes for use in console output.
""" """
RED = "\033[91m" RED = "\033[91m"
GREEN = "\033[92m" GREEN = "\033[92m"
YELLOW = "\033[93m" YELLOW = "\033[93m"
@@ -58,8 +59,11 @@ def formatting_log(logger, title="Info"):
a context manager, print liens before and after a function a context manager, print liens before and after a function
""" """
length = {"Start": 120, "Task": 120, "Info": 60, "Interact": 60, "End": 120}.get(title, 60) length = {"Start": 120, "Task": 120, "Info": 60, "Interact": 60, "End": 120}.get(title, 60)
color, bold = (LogColors.YELLOW, LogColors.BOLD) \ color, bold = (
if title in ["Start", "Task", "Info", "Interact", "End"] else (LogColors.CYAN, "") (LogColors.YELLOW, LogColors.BOLD)
if title in ["Start", "Task", "Info", "Interact", "End"]
else (LogColors.CYAN, "")
)
logger.info("") logger.info("")
logger.info(f"{color}{bold}{'-'} {title} {'-' * (length - len(title))}{LogColors.END}") logger.info(f"{color}{bold}{'-'} {title} {'-' * (length - len(title))}{LogColors.END}")
yield yield
@@ -99,12 +103,12 @@ class FinCoLog(SingletonBaseClass):
f"{LogColors.MAGENTA}{LogColors.BOLD}Role:{LogColors.END} " f"{LogColors.MAGENTA}{LogColors.BOLD}Role:{LogColors.END} "
f"{LogColors.CYAN}{m['role']}{LogColors.END}\n" f"{LogColors.CYAN}{m['role']}{LogColors.END}\n"
+ f"{LogColors.MAGENTA}{LogColors.BOLD}Content:{LogColors.END} " + f"{LogColors.MAGENTA}{LogColors.BOLD}Content:{LogColors.END} "
f"{LogColors.CYAN}{m['content']}{LogColors.END}\n") f"{LogColors.CYAN}{m['content']}{LogColors.END}\n"
)
def log_response(self, response: str): def log_response(self, response: str):
with formatting_log(self.logger, "GPT Response"): with formatting_log(self.logger, "GPT Response"):
self.logger.info( self.logger.info(f"{LogColors.CYAN}{response}{LogColors.END}\n")
f"{LogColors.CYAN}{response}{LogColors.END}\n")
# TODO: # TODO:
# It looks wierd if we only have logger # It looks wierd if we only have logger
@@ -118,14 +122,13 @@ class FinCoLog(SingletonBaseClass):
def plain_info(self, *args): def plain_info(self, *args):
for arg in args: for arg in args:
self.logger.info( self.logger.info(
f"{LogColors.YELLOW}{LogColors.BOLD}Info:{LogColors.END}{LogColors.WHITE}{arg}{LogColors.END}") f"{LogColors.YELLOW}{LogColors.BOLD}Info:{LogColors.END}{LogColors.WHITE}{arg}{LogColors.END}"
)
def warning(self, *args): def warning(self, *args):
for arg in args: for arg in args:
self.logger.warning( self.logger.warning(f"{LogColors.BLUE}{LogColors.BOLD}Warning:{LogColors.END}{arg}")
f"{LogColors.BLUE}{LogColors.BOLD}Warning:{LogColors.END}{arg}")
def error(self, *args): def error(self, *args):
for arg in args: for arg in args:
self.logger.error( self.logger.error(f"{LogColors.RED}{LogColors.BOLD}Error:{LogColors.END}{arg}")
f"{LogColors.RED}{LogColors.BOLD}Error:{LogColors.END}{arg}")

7
qlib/finco/prompt_template.py
View File

@@ -10,8 +10,9 @@ from qlib.finco import get_finco_path
class PromptTemplate(SingletonBaseClass): class PromptTemplate(SingletonBaseClass):
def __init__(self) -> None: def __init__(self) -> None:
super().__init__() super().__init__()
_template = yaml.load(open(Path.joinpath(get_finco_path(), "prompt_template.yaml"), "r"), _template = yaml.load(
Loader=yaml.FullLoader) open(Path.joinpath(get_finco_path(), "prompt_template.yaml"), "r"), Loader=yaml.FullLoader
)
for k, v in _template.items(): for k, v in _template.items():
if k == "mods": if k == "mods":
continue continue
@@ -28,5 +29,5 @@ class PromptTemplate(SingletonBaseClass):
file_path = Path(file_path) file_path = Path(file_path)
Path.mkdir(file_path.parent, exist_ok=True) Path.mkdir(file_path.parent, exist_ok=True)
with open(file_path, 'w') as f: with open(file_path, "w") as f:
yaml.dump(self.__dict__, f) yaml.dump(self.__dict__, f)

598
qlib/finco/task.py
View File

File diff suppressed because it is too large Load Diff

33
qlib/finco/tpl/sl/workflow_config.yaml
View File

@@ -10,13 +10,25 @@ data_handler_config: &data_handler_config
fit_start_time: 2008-01-01 fit_start_time: 2008-01-01
fit_end_time: 2014-12-31 fit_end_time: 2014-12-31
instruments: *market instruments: *market
infer_processors:
- class: RobustZScoreNorm
kwargs:
fields_group: feature
clip_outlier: true
- class: Fillna
kwargs:
fields_group: feature
learn_processors:
- class: DropnaLabel
- class: CSRankNorm
kwargs:
fields_group: label
port_analysis_config: &port_analysis_config port_analysis_config: &port_analysis_config
strategy: strategy:
class: TopkDropoutStrategy class: TopkDropoutStrategy
module_path: qlib.contrib.strategy module_path: qlib.contrib.strategy
kwargs: kwargs:
model: <MODEL> signal: <PRED>
dataset: <DATASET>
topk: 50 topk: 50
n_drop: 5 n_drop: 5
backtest: backtest:
@@ -32,18 +44,11 @@ port_analysis_config: &port_analysis_config
min_cost: 5 min_cost: 5
task: task:
model: model:
class: LGBModel class: LinearModel
module_path: qlib.contrib.model.gbdt module_path: qlib.contrib.model.linear
kwargs: kwargs:
loss: mse estimator: ridge
colsample_bytree: 0.8879 alpha: 0.05
learning_rate: 0.2
subsample: 0.8789
lambda_l1: 205.6999
lambda_l2: 580.9768
max_depth: 8
num_leaves: 210
num_threads: 20
dataset: dataset:
class: DatasetH class: DatasetH
module_path: qlib.data.dataset module_path: qlib.data.dataset
@@ -65,7 +70,7 @@ task:
- class: SigAnaRecord - class: SigAnaRecord
module_path: qlib.workflow.record_temp module_path: qlib.workflow.record_temp
kwargs: kwargs:
ana_long_short: False ana_long_short: True
ann_scaler: 252 ann_scaler: 252
- class: PortAnaRecord - class: PortAnaRecord
module_path: qlib.workflow.record_temp module_path: qlib.workflow.record_temp

3
qlib/finco/utils.py
View File

@@ -20,6 +20,7 @@ class SingletonBaseClass(metaclass=SingletonMeta):
This class becomes necessary This class becomes necessary
""" """
# TODO: Add move this class to Qlib's general utils. # TODO: Add move this class to Qlib's general utils.
@@ -42,4 +43,4 @@ def similarity(text1, text2):
def random_string(length=10): def random_string(length=10):
letters = string.ascii_letters + string.digits letters = string.ascii_letters + string.digits
return ''.join(random.choice(letters) for i in range(length)) return "".join(random.choice(letters) for i in range(length))

113
qlib/finco/workflow.py
View File

@@ -1,67 +1,40 @@
import sys import sys
import copy
import shutil import shutil
from pathlib import Path
from typing import List from typing import List
from pathlib import Path
from qlib.finco.task import HighLevelPlanTask, SummarizeTask, TrainTask from qlib.finco.task import HighLevelPlanTask, SummarizeTask
from qlib.finco.prompt_template import PromptTemplate, Template from qlib.finco.prompt_template import PromptTemplate, Template
from qlib.finco.log import FinCoLog, LogColors from qlib.finco.log import FinCoLog, LogColors
from qlib.finco.utils import similarity
from qlib.finco.llm import APIBackend from qlib.finco.llm import APIBackend
from qlib.finco.conf import Config from qlib.finco.conf import Config
from qlib.finco.knowledge import KnowledgeBase, Topic from qlib.finco.knowledge import KnowledgeBase, Topic
from qlib.finco.context import WorkflowContextManager
class WorkflowContextManager: # TODO: it is not necessary in current phase
"""Context Manager stores the context of the workflow""" # class TaskDAG:
# """
"""All context are key value pairs which saves the input, output and status of the whole workflow""" # This is a Task manager. it maintains a graph and a stack stucture to manager the task
# The reason why the DGA relationship is maintained outside instead of inside the task is that
def __init__(self) -> None: # - To make the creating of task simpler(user don't have to care about the relation-ship)
self.context = {} # - To manage the relation ship when poping and executing the tasks is relatively easier instead of scattering them everywhere
self.logger = FinCoLog() # """
# def __init__(self) -> None:
def set_context(self, key, value): # self._finished = []
if key in self.context: # self._stack = []
self.logger.warning("The key already exists in the context, the value will be overwritten") # self._dag = defaultdict(list) # from id(object) -> list of id(object)
self.context[key] = value #
# def pop(self):
def get_context(self, key): # return self._stack.pop(0)
# NOTE: if the key doesn't exist, return None. In the future, we may raise an error to detect abnormal behavior #
if key not in self.context: # def push(self, task: Union[Task, List[Task]], parent: Optional[Task] = None):
self.logger.warning("The key doesn't exist in the context") # if isinstance(task, Task):
return None # task = [task]
return self.context[key] # if parent is not None:
# self._dag
def update_context(self, key, new_value): #
# NOTE: if the key doesn't exist, return None. In the future, we may raise an error to detect abnormal behavior # def done(self) -> bool:
if key not in self.context: # return len(self._stack) == 0
self.logger.warning("The key doesn't exist in the context")
self.context.update({key: new_value})
def get_all_context(self):
"""return a deep copy of the context"""
"""TODO: do we need to return a deep copy?"""
return copy.deepcopy(self.context)
def retrieve(self, query: str) -> dict:
if query in self.context.keys():
return {query: self.context.get(query)}
# Note: retrieve information from context by string similarity maybe abandon in future
scores = {}
for k, v in self.context.items():
scores.update({k: max(similarity(query, k), similarity(query, v))})
max_score_key = max(scores, key=scores.get)
return {max_score_key: self.context.get(max_score_key)}
def clear(self, reserve: list = None):
if reserve is None:
reserve = []
_context = {k: self.get_context(k) for k in reserve}
self.context = _context
class WorkflowManager: class WorkflowManager:
@@ -78,8 +51,7 @@ class WorkflowManager:
self._confirm_and_rm() self._confirm_and_rm()
self.prompt_template = PromptTemplate() self.prompt_template = PromptTemplate()
self.context = WorkflowContextManager() self.context = WorkflowContextManager(workspace=self._workspace)
self.context.set_context("workspace", self._workspace)
self.default_user_prompt = "Please help me build a low turnover strategy that focus more on longterm return in China A csi300. Please help to use lightgbm model." self.default_user_prompt = "Please help me build a low turnover strategy that focus more on longterm return in China A csi300. Please help to use lightgbm model."
def _confirm_and_rm(self): def _confirm_and_rm(self):
@@ -92,7 +64,8 @@ class WorkflowManager:
f"If you do not need to delete {self._workspace}," f"If you do not need to delete {self._workspace},"
f" please change the workspace dir or rename existing files\n" f" please change the workspace dir or rename existing files\n"
f"Are you sure you want to delete, yes(Y/y), no (N/n):", f"Are you sure you want to delete, yes(Y/y), no (N/n):",
color=LogColors.WHITE) color=LogColors.WHITE,
)
) )
if str(flag) not in ["Y", "y"]: if str(flag) not in ["Y", "y"]:
sys.exit() sys.exit()
@@ -150,10 +123,12 @@ class WorkflowManager:
# TODO: sort the task list based on the priority of the task # TODO: sort the task list based on the priority of the task
# task_list = sorted(task_list, key=lambda x: x.task_type) # task_list = sorted(task_list, key=lambda x: x.task_type)
t = task_list.pop(0) t = task_list.pop(0)
self.logger.info(f"Task finished: {[str(task) for task in task_finished]}", self.logger.info(
f"Task in queue: {task_list_info}", f"Task finished: {[str(task) for task in task_finished]}",
f"Executing task: {str(t)}", f"Task in queue: {task_list_info}",
title="Task") f"Executing task: {str(t)}",
title="Task",
)
t.assign_context_manager(self.context) t.assign_context_manager(self.context)
res = t.execute() res = t.execute()
@@ -174,9 +149,10 @@ class LearnManager:
self.epoch = 0 self.epoch = 0
self.wm = WorkflowManager() self.wm = WorkflowManager()
self.topics = [Topic(name=topic, describe=self.wm.prompt_template.get(f"Topic_{topic}")) for topic in self.topics = [
self.__DEFAULT_TOPICS] Topic(name=topic, describe=self.wm.prompt_template.get(f"Topic_{topic}")) for topic in self.__DEFAULT_TOPICS
self.knowledge_base = KnowledgeBase(workdir=Path.cwd().joinpath('knowledge')) ]
self.knowledge_base = KnowledgeBase(workdir=Path.cwd().joinpath("knowledge"))
self.knowledge_base.execute_knowledge.add([]) self.knowledge_base.execute_knowledge.add([])
self.knowledge_base.query(knowledge_type="infrastructure", content="resolve_path") self.knowledge_base.query(knowledge_type="infrastructure", content="resolve_path")
@@ -209,14 +185,17 @@ class LearnManager:
for task in task_finished: for task in task_finished:
prompt_workflow_selection = self.wm.prompt_template.get(f"{self.__class__.__name__}_user").render( prompt_workflow_selection = self.wm.prompt_template.get(f"{self.__class__.__name__}_user").render(
summary=summary, brief=knowledge_of_topics, summary=summary,
brief=knowledge_of_topics,
task_finished=[str(t) for t in task_finished], task_finished=[str(t) for t in task_finished],
task=task.__class__.__name__, system=task.system.render(), user_prompt=user_prompt task=task.__class__.__name__,
system=task.system.render(),
user_prompt=user_prompt,
) )
response = APIBackend().build_messages_and_create_chat_completion( response = APIBackend().build_messages_and_create_chat_completion(
user_prompt=prompt_workflow_selection, user_prompt=prompt_workflow_selection,
system_prompt=self.wm.prompt_template.get(f"{self.__class__.__name__}_system").render() system_prompt=self.wm.prompt_template.get(f"{self.__class__.__name__}_system").render(),
) )
# todo: response assertion # todo: response assertion

2
scripts/finco/cmd.sh
View File

@@ -12,4 +12,4 @@ if [ -e $DIR/cridential.sh ]; then
fi fi
# run the command # run the command
python -m qlib.finco.cli "please help me build a low turnover strategy that focus more on longterm return" python -m qlib.finco.cli "build an A-share stock market daily portfolio in quantitative investment and minimize the maximum drawdown."

5
setup.py
View File

@@ -176,11 +176,12 @@ setup(
], ],
"finco": [ "finco": [
# finco is not necessary for all Qlib users; So a single require section is used for it. # finco is not necessary for all Qlib users; So a single require section is used for it.
"openapi", "openai",
"pydantic", # Please add it to basic requirements after the design of pydantic is state. "pydantic", # Please add it to basic requirements after the design of pydantic is state.
"pydantic-settings",
"python-dotenv", # I don't think this is necessary if we use pydantic. "python-dotenv", # I don't think this is necessary if we use pydantic.
"fuzzywuzzy", "fuzzywuzzy",
"python-Levenshtein" # not necessary but accelerate fuzzywuzzy calculation "python-Levenshtein", # not necessary but accelerate fuzzywuzzy calculation
], ],
}, },
include_package_data=True, include_package_data=True,