Adding ability to remember instruction after using too many tools

Author: joaomdmoura

pyproject.toml

@@ -1,7 +1,7 @@
 
 [tool.poetry]
 name = "crewai"
-version = "0.8.0"
+version = "0.10.0"
 description = "Cutting-edge framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks."
 authors = ["Joao Moura <joao@crewai.com>"]
 readme = "README.md"

src/crewai/agent.py

@@ -150,12 +150,14 @@ def execute_task(
         tools = tools or self.tools
         self.agent_executor.tools = tools
         self.agent_executor.task = task
+        self.agent_executor.tools_description = (render_text_description(tools),)
+        self.agent_executor.tools_names = self.__tools_names(tools)
 
         result = self.agent_executor.invoke(
             {
                 "input": task_prompt,
-                "tool_names": self.__tools_names(tools),
-                "tools": render_text_description(tools),
+                "tool_names": self.agent_executor.tools_names,
+                "tools": self.agent_executor.tools_description,
             }
         )["output"]
 

src/crewai/agents/executor.py

@@ -21,6 +21,9 @@ class CrewAgentExecutor(AgentExecutor):
     i18n: I18N = I18N()
     llm: Any = None
     iterations: int = 0
+    task: Any = None
+    tools_description: str = ""
+    tools_names: str = ""
     request_within_rpm_limit: Any = None
     tools_handler: InstanceOf[ToolsHandler] = None
     max_iterations: Optional[int] = 15
@@ -187,7 +190,12 @@ def _iter_next_step(
                 if return_direct:
                     tool_run_kwargs["llm_prefix"] = ""
                 observation = ToolUsage(
-                    tools_handler=self.tools_handler, tools=self.tools, llm=self.llm
+                    tools_handler=self.tools_handler,
+                    tools=self.tools,
+                    tools_description=self.tools_description,
+                    tools_names=self.tools_names,
+                    llm=self.llm,
+                    task=self.task,
                 ).use(agent_action.log)
             else:
                 tool_run_kwargs = self.agent.tool_run_logging_kwargs()

src/crewai/tools/tool_usage.py

@@ -10,30 +10,55 @@
 from crewai.utilities import I18N, Printer
 
 
+class ToolUsageErrorException(Exception):
+    """Exception raised for errors in the tool usage."""
+
+    def __init__(self, message: str) -> None:
+        self.message = message
+        super().__init__(self.message)
+
+
 class ToolUsage:
     """
     Class that represents the usage of a tool by an agent.
 
     Attributes:
+        task: Task being executed.
         tools_handler: Tools handler that will manage the tool usage.
         tools: List of tools available for the agent.
+        tools_description: Description of the tools available for the agent.
+        tools_names: Names of the tools available for the agent.
         llm: Language model to be used for the tool usage.
     """
 
     def __init__(
-        self, tools_handler: ToolsHandler, tools: List[BaseTool], llm: Any
+        self,
+        tools_handler: ToolsHandler,
+        tools: List[BaseTool],
+        tools_description: str,
+        tools_names: str,
+        task: Any,
+        llm: Any,
     ) -> None:
         self._i18n: I18N = I18N()
         self._printer: Printer = Printer()
         self._telemetry: Telemetry = Telemetry()
         self._run_attempts: int = 1
         self._max_parsing_attempts: int = 3
+        self._remeber_format_after_usages: int = 3
+        self.tools_description = tools_description
+        self.tools_names = tools_names
         self.tools_handler = tools_handler
         self.tools = tools
+        self.task = task
         self.llm = llm
 
     def use(self, tool_string: str):
         calling = self._tool_calling(tool_string)
+        if isinstance(calling, ToolUsageErrorException):
+            error = calling.message
+            self._printer.print(content=f"\n\n{error}\n", color="yellow")
+            return error
         tool = self._select_tool(calling.function_name)
         return self._use(tool=tool, calling=calling)
 
@@ -57,6 +82,24 @@ def _use(self, tool: BaseTool, calling: ToolCalling) -> None:
         self._telemetry.tool_usage(
             llm=self.llm, tool_name=tool.name, attempts=self._run_attempts
         )
+
+        result = self._format_result(result=result)
+        return result
+
+    def _format_result(self, result: Any) -> None:
+        self.task.used_tools += 1
+        if self._should_remember_format():
+            result = self._remember_format(result=result)
+        return result
+
+    def _should_remember_format(self) -> None:
+        return self.task.used_tools % self._remeber_format_after_usages == 0
+
+    def _remember_format(self, result: str) -> None:
+        result = str(result)
+        result += "\n\n" + self._i18n.slice("tools").format(
+            tools=self.tools_description, tool_names=self.tools_names
+        )
         return result
 
     def _check_tool_repeated_usage(self, calling: ToolCalling) -> None:
@@ -102,11 +145,11 @@ def _tool_calling(self, tool_string: str) -> ToolCalling:
             chain = prompt | self.llm | parser
             calling = chain.invoke({"tool_string": tool_string})
 
-        except Exception as e:
+        except Exception:
             self._run_attempts += 1
             if self._run_attempts > self._max_parsing_attempts:
                 self._telemetry.tool_usage_error(llm=self.llm)
-                raise e
+                return ToolUsageErrorException(self._i18n.errors("tool_usage_error"))
             return self._tool_calling(tool_string)
 
         return calling

src/crewai/translations/el.json

@@ -16,7 +16,8 @@
   "errors": {
     "force_final_answer": "Στην πραγματικότητα, χρησιμοποίησα πάρα πολλά εργαλεία, οπότε θα σταματήσω τώρα και θα σας δώσω την απόλυτη ΚΑΛΥΤΕΡΗ τελική μου απάντηση ΤΩΡΑ, χρησιμοποιώντας την αναμενόμενη μορφή: ```\nΣκέφτηκα: Χρειάζεται να χρησιμοποιήσω ένα εργαλείο; Όχι\nΤελική απάντηση: [η απάντησή σας εδώ]```",
     "agent_tool_unexsiting_coworker": "\nΣφάλμα κατά την εκτέλεση του εργαλείου. Ο συνάδελφος που αναφέρεται στο Ενέργεια προς εισαγωγή δεν βρέθηκε, πρέπει να είναι μία από τις ακόλουθες επιλογές: {coworkers}.\n",
-    "task_repeated_usage": "Μόλις χρησιμοποίησα το εργαλείο {tool} με είσοδο {tool_input}. Άρα ξέρω ήδη το αποτέλεσμα αυτού και δεν χρειάζεται να το χρησιμοποιήσω ξανά τώρα.\n"
+    "task_repeated_usage": "Μόλις χρησιμοποίησα το εργαλείο {tool} με είσοδο {tool_input}. Άρα ξέρω ήδη το αποτέλεσμα αυτού και δεν χρειάζεται να το χρησιμοποιήσω ξανά τώρα.\n",
+    "tool_usage_error": "Φαίνεται ότι αντιμετωπίσαμε ένα απροσδόκητο σφάλμα κατά την προσπάθεια χρήσης του εργαλείου."
   },
   "tools": {
     "delegate_work": "Αναθέστε μια συγκεκριμένη εργασία σε έναν από τους παρακάτω συναδέλφους: {coworkers}. Η είσοδος σε αυτό το εργαλείο θα πρέπει να είναι ο ρόλος του συναδέλφου, η εργασία που θέλετε να κάνει και ΟΛΟ το απαραίτητο πλαίσιο για την εκτέλεση της εργασίας, δεν γνωρίζουν τίποτα για την εργασία, επομένως μοιραστείτε απολύτως όλα όσα γνωρίζετε, μην αναφέρετε πράγματα, αλλά αντί να τους εξηγήσεις.",

src/crewai/translations/en.json

@@ -14,9 +14,10 @@
     "expected_output": "Your final answer must be: {expected_output}"
   },
   "errors": {
-    "force_final_answer": "Actually, I used too many tools, so I'll stop now and give you my absolute BEST Final answer NOW, using the expected format: ```\nThought: Do I need to use a tool? No\nFinal Answer: [your response here]```",
+    "force_final_answer": "Actually, I used too many tools, so I'll stop now and give you my absolute BEST Final answer NOW, using exaclty the expected format bellow: \n```\nThought: Do I need to use a tool? No\nFinal Answer: [your response here]```",
     "agent_tool_unexsiting_coworker": "\nError executing tool. Co-worker mentioned on the Action Input not found, it must to be one of the following options: {coworkers}.\n",
-    "task_repeated_usage": "I just used the {tool} tool with input {tool_input}. So I already know the result of that and don't need to use it again now.\n"
+    "task_repeated_usage": "I just used the {tool} tool with input {tool_input}. So I already know the result of that and don't need to use it again now. \nI could give my final answer if I'm ready, using exaclty the expected format bellow: \n```\nThought: Do I need to use a tool? No\nFinal Answer: [your response here]```\n",
+    "tool_usage_error": "It seems we encountered an unexpected error while trying to use the tool."
   },
   "tools": {
     "delegate_work": "Delegate a specific task to one of the following co-workers: {coworkers}. The input to this tool should be the role of the coworker, the task you want them to do, and ALL necessary context to exectue the task, they know nothing about the task, so share absolute everything you know, don't reference things but instead explain them.",

tests/agent_test.py

@@ -10,6 +10,7 @@
 from crewai.agents.cache import CacheHandler
 from crewai.agents.executor import CrewAgentExecutor
 from crewai.tools.tool_calling import ToolCalling
+from crewai.tools.tool_usage import ToolUsage
 from crewai.utilities import RPMController
 
 
@@ -256,7 +257,7 @@ def get_final_answer(numbers) -> float:
 
     captured = capsys.readouterr()
     assert (
-        "I just used the get_final_answer tool with input {'numbers': 42}. So I already know the result of that and don't need to use it again now."
+        "I just used the get_final_answer tool with input {'numbers': 42}. So I already know the result of that and don't need to use it again now. \nI could give my final answer if I'm ready, using exaclty the expected format bellow: \n```\nThought: Do I need to use a tool? No\nFinal Answer: [your response here]```\n"
         in captured.out
     )
 
@@ -414,6 +415,78 @@ def get_final_answer(numbers) -> float:
         moveon.assert_called_once()
 
 
+@pytest.mark.vcr(filter_headers=["authorization"])
+def test_agent_error_on_parsing_tool(capsys):
+    from unittest.mock import patch
+
+    from langchain.tools import tool
+
+    @tool
+    def get_final_answer(numbers) -> float:
+        """Get the final answer but don't give it yet, just re-use this
+        tool non-stop."""
+        return 42
+
+    agent1 = Agent(
+        role="test role",
+        goal="test goal",
+        backstory="test backstory",
+        verbose=True,
+    )
+    tasks = [
+        Task(
+            description="Use the get_final_answer tool.",
+            agent=agent1,
+            tools=[get_final_answer],
+        )
+    ]
+
+    crew = Crew(agents=[agent1], tasks=tasks, verbose=2)
+
+    with patch.object(ToolUsage, "_render") as force_exception:
+        force_exception.side_effect = Exception("Error on parsing tool.")
+        crew.kickoff()
+        captured = capsys.readouterr()
+        assert (
+            "It seems we encountered an unexpected error while trying to use the tool"
+            in captured.out
+        )
+
+
+@pytest.mark.vcr(filter_headers=["authorization"])
+def test_agent_remembers_output_format_after_using_tools_too_many_times():
+    from unittest.mock import patch
+
+    from langchain.tools import tool
+
+    @tool
+    def get_final_answer(numbers) -> float:
+        """Get the final answer but don't give it yet, just re-use this
+        tool non-stop."""
+        return 42
+
+    agent1 = Agent(
+        role="test role",
+        goal="test goal",
+        backstory="test backstory",
+        max_iter=4,
+        verbose=True,
+    )
+    tasks = [
+        Task(
+            description="Never give the final answer. Use the get_final_answer tool in a loop.",
+            agent=agent1,
+            tools=[get_final_answer],
+        )
+    ]
+
+    crew = Crew(agents=[agent1], tasks=tasks, verbose=2)
+
+    with patch.object(ToolUsage, "_remember_format") as remember_format:
+        crew.kickoff()
+        remember_format.assert_called()
+
+
 @pytest.mark.vcr(filter_headers=["authorization"])
 def test_agent_use_specific_tasks_output_as_context(capsys):
     agent1 = Agent(role="test role", goal="test goal", backstory="test backstory")