optimize log

qlib/finco/knowledge.py

@@ -215,6 +215,8 @@ class PracticeKnowledge(Knowledge):
         self.summarize()
 
     def add(self, docs: List, storage_name: str = YamlStorage.DEFAULT_NAME):
+        s = "\n".join(docs)
+        logger.info(f'Add to Practice Knowledge:\n {s}')
         storage = self.get_storage(storage_name)
         if storage is None:
             storage = YamlStorage(path=self.workdir.joinpath(self.name).joinpath(storage_name))

qlib/finco/log.py

@@ -2,6 +2,7 @@
 This module will base on Qlib's logger module and provides some interactive functions.
 """
 import logging
+import time
 
 from typing import Dict, List
 from qlib.finco.utils import SingletonBaseClass
@@ -58,15 +59,18 @@ def formatting_log(logger, title="Info"):
     """
     a context manager, print liens before and after a function
     """
-    length = {"Start": 120, "Task": 120, "Info": 60, "Interact": 60, "End": 120}.get(title, 60)
+    length = {"Start": 90, "Round": 90, "Task": 90, "Info": 60, "Interact": 60, "End": 90}.get(title, 60)
     color, bold = (
         (LogColors.YELLOW, LogColors.BOLD)
-        if title in ["Start", "Task", "Info", "Interact", "End"]
+        if title in ["Start", "Round", "Task", "Info", "Interact", "End"]
         else (LogColors.CYAN, "")
     )
     logger.info("")
     logger.info(f"{color}{bold}{'-'} {title} {'-' * (length - len(title))}{LogColors.END}")
+
     yield
+    if color == LogColors.YELLOW:
+        time.sleep(2)
     logger.info("")
 
 
@@ -109,6 +113,7 @@ class FinCoLog(SingletonBaseClass):
     def log_response(self, response: str):
         with formatting_log(self.logger, "GPT Response"):
             self.logger.info(f"{LogColors.CYAN}{response}{LogColors.END}\n")
+            time.sleep(1)
 
     # TODO:
     # It looks wierd if we only have logger

qlib/finco/task.py

@@ -1,4 +1,5 @@
 import os
+import time
 
 from pathlib import Path
 import io
@@ -19,6 +20,7 @@ from qlib.workflow import R
 from qlib.finco.log import FinCoLog, LogColors
 from qlib.finco.conf import Config
 from qlib.finco.knowledge import KnowledgeBase
+from qlib.finco.utils import directory_tree
 
 from qlib.finco.context import Design, Exp, WorkflowContextManager
 
@@ -192,6 +194,8 @@ class IdeaTask(PlanTask):
         )
         self.save_chat_history_to_context_manager(user_prompt, response, system_prompt)
         
+        time.sleep(3)
+
         re_search_pattern = f"Target: (.*)Deliverables:(.*)Thinking directions:(.*)Business level:(.*)Algorithm level:(.*)Details:(.*)"
         re_search_res = re.search(re_search_pattern, response, re.S)
 
@@ -279,7 +283,7 @@ class SLPlanTask(PlanTask):
 
     def execute(self):
         workflow = self._context_manager.get_context("high_level_workflow")
-        assert workflow.lower() == "supervised learning", "The workflow is not supervised learning"
+        assert "supervised learning" in workflow.lower(), "The workflow is not supervised learning"
 
         target = self._context_manager.get_context("target")
         deliverable = self._context_manager.get_context("deliverable")
@@ -414,7 +418,7 @@ class TrainTask(Task):
 
     def execute(self):
         workflow_config = f"experiment_{self._experiment_index}.yaml"
-
+        time.sleep(2)
         workspace = self._context_manager.get_context("workspace")
         workflow_path = workspace.joinpath(workflow_config)
         with workflow_path.open() as f:
@@ -434,6 +438,8 @@ class TrainTask(Task):
         R.set_uri(Path(workspace).joinpath("mlruns").as_uri())
         if not self._rolling:
             command = f"qrun {str(workflow_path)}"
+            self.logger.plain_info(f"Run the command: {command}")
+            
             try:
                 # Run the command and capture the output
                 workspace = self._context_manager.get_context("workspace")
@@ -474,6 +480,7 @@ class TrainTask(Task):
                     return ret_list
         elif not self._ddgda:
             command = f"python -m qlib.contrib.rolling base --conf_path {workflow_path} run"
+            self.logger.plain_info(f"Run the command: {command}")
             # Run the command and capture the output
             workspace = self._context_manager.struct_context.workspace
             subprocess.run(
@@ -484,6 +491,7 @@ class TrainTask(Task):
 
         else:
             command = f"python -m qlib.contrib.rolling ddgda --conf_path {workflow_path} run"
+            self.logger.plain_info(f"Run the command: {command}")
             # Run the command and capture the output
             workspace = self._context_manager.struct_context.workspace
             subprocess.run(
@@ -500,6 +508,8 @@ class TrainTask(Task):
         with open(rf"{workspace}/README.md", "w") as fw:
             fw.write(f"\n")
             fw.flush()
+
+        self.logger.plain_info(f"Workspace output:\n{directory_tree(workspace, max_depth=1)}")
         # first recorder is the latest
         recorder = exp.list_recorders(rtype=exp.RT_L)[0]
         self._context_manager.set_context(f"experiment_{self._experiment_index}_recorder", recorder)

qlib/finco/utils.py

@@ -2,6 +2,8 @@ import json
 import string
 import random
 
+from typing import List
+from pathlib import Path
 from fuzzywuzzy import fuzz
 
 
@@ -44,3 +46,26 @@ def similarity(text1, text2):
 def random_string(length=10):
     letters = string.ascii_letters + string.digits
     return "".join(random.choice(letters) for i in range(length))
+
+
+def directory_tree(root_dif, max_depth=None):
+
+    def _directory_tree(root_dir, padding="", deep=1, max_d=None) -> List:
+        _output = []
+        if max_d and deep > max_d:
+            return _output
+
+        files = sorted(root_dir.iterdir())
+        for i, file in enumerate(files):
+            if i == len(files) - 1:
+                _output.append(padding + '└── ' + file.name)
+                if file.is_dir():
+                    _output.extend(_directory_tree(file, padding + "    ", deep=deep + 1, max_d=max_d))
+            else:
+                _output.append(padding + '├── ' + file.name)
+                if file.is_dir():
+                    _output.extend(_directory_tree(file, padding + "│   ", deep=deep + 1, max_d=max_d))
+        return _output
+
+    output = _directory_tree(root_dif, max_d=max_depth)
+    return '\n'.join(output)

qlib/finco/workflow.py

@@ -1,4 +1,5 @@
 import sys
+import time
 import shutil
 from typing import List
 
@@ -146,7 +147,7 @@ class WorkflowManager:
 
 
 class LearnManager:
-    __DEFAULT_TOPICS = ["IC", "MaxDropDown", "RollingModel"]
+    __DEFAULT_TOPICS = ["RollingModel"]
 
     def __init__(self):
         self.epoch = 0
@@ -160,6 +161,7 @@ class LearnManager:
     def run(self, prompt):
         # todo: add early stop condition
         for i in range(10):
+            self.wm.logger.info(f"Round: {self.epoch+1}", title="Round")
             self.wm.run(prompt)
             self.learn()
             self.epoch += 1