From e2e4fce4c8402335e7d455ff162d5d1d98d896cc Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 18 Jun 2026 04:59:05 +0000 Subject: [PATCH] Fix live agent hanging when it announces a tool it can't use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `assembly live` told the model it could "search the web" in every session, but the Tavily search tool is only bound when a TAVILY_API_KEY is set. With no key, the model would narrate "I'll search for the latest news…" and then the turn would end with no result — the announced tool was never actually available to call, so the answer never came back. Build the live agent's system prompt from the tools actually resolved for the session: each capability clause (web search, URL fetch, docs lookup) is advertised only when a tool backing it is present, and with no tools at all the model is told to answer from its own knowledge and not promise an action it can't take. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01WvDSFotLTrHbtxeM9QaGWA --- aai_cli/agent_cascade/brain.py | 79 ++++++++++++++++++++++++++----- aai_cli/code_agent/web_search.py | 4 ++ tests/test_agent_cascade_brain.py | 57 ++++++++++++++++++++-- 3 files changed, 123 insertions(+), 17 deletions(-) diff --git a/aai_cli/agent_cascade/brain.py b/aai_cli/agent_cascade/brain.py index 7be2297..966e3e6 100644 --- a/aai_cli/agent_cascade/brain.py +++ b/aai_cli/agent_cascade/brain.py @@ -21,26 +21,77 @@ from aai_cli.agent_cascade.config import CascadeConfig from aai_cli.code_agent.agent import CompiledAgent +from aai_cli.code_agent.fetch_tool import FETCH_TOOL_NAME +from aai_cli.code_agent.web_search import WEB_SEARCH_TOOL_NAME if TYPE_CHECKING: from langchain_core.tools import BaseTool from openai.types.chat import ChatCompletionMessageParam -# Appended to the user's persona so the model knows it has tools and must keep replies -# spoken. The cascade's plain-LLM persona (CascadeConfig.system_prompt) says nothing -# about tools, so without this the agent would never reach for web search. -_TOOL_GUIDANCE = ( - "You can use tools to help answer: search the web for current or unfamiliar facts, " - "fetch a specific URL, and look up the AssemblyAI documentation. Reach for a tool " - "when a question needs fresh or external information; answer directly and instantly " - "when you already know. Your reply is read aloud, so keep it short and spoken — no " - "markdown, lists, code, or raw URLs." +# Closes every guidance variant: the reply is spoken, so it must stay short and plain. +_SPOKEN_TAIL = ( + "Your reply is read aloud, so keep it short and spoken — no markdown, lists, code, or raw URLs." ) +# When the session has *no* tools wired (e.g. no web search and the docs host is +# unreachable), the model must answer from its own knowledge — and crucially must not +# promise an action it can't take. Without this, telling it "you can search the web" while +# no search tool is bound makes it narrate "I'll search for that…" and then stop, so the +# answer never comes (the tool it announced was never actually available to call). +_NO_TOOLS_GUIDANCE = ( + "You have no external tools available, so answer from your own knowledge. Never say " + "you will search the web, look something up, or fetch a page — you can't do any of " + "that, so don't promise it; if a question needs information you don't have, say so " + f"briefly instead. {_SPOKEN_TAIL}" +) + + +def _join_clause(parts: list[str]) -> str: + """Join capability phrases into a readable clause: ``a``, ``a and b``, ``a, b, and c``.""" + *initial, last = parts + if not initial: + return last + # Oxford comma only once there are three-or-more items (two or more lead the last). + joiner = ", and " if initial[1:] else " and " + return f"{', '.join(initial)}{joiner}{last}" + -def build_system_prompt(persona: str) -> str: - """The live agent's system prompt: the user's persona plus the tool guidance.""" - return f"{persona}\n\n{_TOOL_GUIDANCE}" +def _tool_capabilities(tools: Sequence[BaseTool]) -> list[str]: + """The spoken-capability phrases backed by an actually-present tool. + + Derived from the resolved tool names so the prompt never advertises a capability the + agent can't perform: web search is present only with a ``TAVILY_API_KEY``, and the docs + tools are best-effort (absent when the docs host is unreachable). + """ + names = {tool.name for tool in tools} + capabilities: list[str] = [] + if WEB_SEARCH_TOOL_NAME in names: + capabilities.append("search the web for current or unfamiliar facts") + if FETCH_TOOL_NAME in names: + capabilities.append("fetch a specific URL") + if names - {WEB_SEARCH_TOOL_NAME, FETCH_TOOL_NAME}: + capabilities.append("look up the AssemblyAI documentation") + return capabilities + + +def build_system_prompt(persona: str, *, tools: Sequence[BaseTool]) -> str: + """The live agent's system prompt: the user's persona plus tool guidance. + + The guidance is tailored to ``tools`` so the model is only told about capabilities it + actually has — advertising a missing tool (web search without a ``TAVILY_API_KEY``) made + the agent announce an action it then couldn't take, leaving the turn hanging with no + answer. With no tools at all the model is told to answer from its own knowledge. + """ + capabilities = _tool_capabilities(tools) + if not capabilities: + return f"{persona}\n\n{_NO_TOOLS_GUIDANCE}" + guidance = ( + f"You can use tools to help answer: {_join_clause(capabilities)}. Reach for a " + "tool when a question needs fresh or external information; answer directly and " + "instantly when you already know. Only offer to do what these tools allow — don't " + f"say you'll search the web or look something up unless it's listed here. {_SPOKEN_TAIL}" + ) + return f"{persona}\n\n{guidance}" def build_live_tools() -> list[BaseTool]: @@ -83,7 +134,9 @@ def build_graph( ) resolved = build_live_tools() if tools is None else list(tools) return create_deep_agent( - model=model, tools=resolved, system_prompt=build_system_prompt(config.system_prompt) + model=model, + tools=resolved, + system_prompt=build_system_prompt(config.system_prompt, tools=resolved), ) diff --git a/aai_cli/code_agent/web_search.py b/aai_cli/code_agent/web_search.py index d06af99..71ed2bf 100644 --- a/aai_cli/code_agent/web_search.py +++ b/aai_cli/code_agent/web_search.py @@ -19,6 +19,10 @@ # agent a tool that will fail on first use for lack of a key. TAVILY_API_KEY_ENV = "TAVILY_API_KEY" +# The name ``TavilySearch`` registers itself under. Callers (e.g. the live agent's prompt +# builder) detect web-search availability by this name, so a test pins it against the tool. +WEB_SEARCH_TOOL_NAME = "tavily_search" + # A small result cap keeps search responses inside the model's context budget. _DEFAULT_MAX_RESULTS = 5 diff --git a/tests/test_agent_cascade_brain.py b/tests/test_agent_cascade_brain.py index acb9801..9f0509a 100644 --- a/tests/test_agent_cascade_brain.py +++ b/tests/test_agent_cascade_brain.py @@ -47,12 +47,61 @@ def _graph(model: BaseChatModel): # --- build_system_prompt ----------------------------------------------------- -def test_system_prompt_appends_tool_guidance(): - prompt = brain.build_system_prompt("You are a pirate.") - # The persona is preserved, and the tool guidance is appended so the model knows it - # can search the web (the plain cascade persona never mentions tools). +class _NamedTool: + """A stand-in tool exposing just the ``.name`` the prompt builder inspects.""" + + def __init__(self, name: str): + self.name = name + + +def test_system_prompt_appends_tool_guidance_for_present_tools(): + prompt = brain.build_system_prompt( + "You are a pirate.", + tools=[_NamedTool("tavily_search"), _NamedTool("fetch_url"), _NamedTool("docs_search")], + ) + # The persona is preserved, and the guidance advertises each capability that a present + # tool backs (the plain cascade persona never mentions tools). assert prompt.startswith("You are a pirate.") assert "search the web" in prompt + assert "fetch a specific URL" in prompt + assert "AssemblyAI documentation" in prompt + + +def test_system_prompt_omits_web_search_when_no_search_tool(): + # With no TAVILY_API_KEY the search tool is absent — the guidance must NOT promise web + # search, since announcing a missing tool makes the agent narrate "I'll search…" and + # then stall with no answer. The capabilities it *does* have still appear. + prompt = brain.build_system_prompt( + "persona", tools=[_NamedTool("fetch_url"), _NamedTool("docs_search")] + ) + assert "search the web for current or unfamiliar facts" not in prompt + assert "fetch a specific URL" in prompt + assert "AssemblyAI documentation" in prompt + + +def test_system_prompt_tells_model_not_to_promise_tools_when_none(): + # No tools at all: the model must answer from its own knowledge and explicitly not + # promise to search or look anything up (the bug that left replies never coming back). + prompt = brain.build_system_prompt("persona", tools=[]) + assert "search the web for current or unfamiliar facts" not in prompt + assert "your own knowledge" in prompt + assert "Never say" in prompt + + +def test_join_clause_grammar(): + # One/two/three capability phrases each render with natural conjunctions. + assert brain._join_clause(["a"]) == "a" + assert brain._join_clause(["a", "b"]) == "a and b" + assert brain._join_clause(["a", "b", "c"]) == "a, b, and c" + + +def test_web_search_tool_name_matches_built_tool(monkeypatch): + # The prompt builder detects search by WEB_SEARCH_TOOL_NAME, so pin it against the real + # tool's registered name — if langchain_tavily renames it, detection would silently break. + from aai_cli.code_agent import web_search + + monkeypatch.setenv(web_search.TAVILY_API_KEY_ENV, "tvly-x") + assert web_search.build_web_search_tool().name == web_search.WEB_SEARCH_TOOL_NAME # --- build_completer (driving the real graph with a fake model) --------------