mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 05:06:59 +00:00
fix(agent,goal): stop coaching scripts when no bash tool is bound, and show the goal evaluator Human Input Card answers (#6082)
* fix(agent): drop script guidance from the lead prompt when no bash tool is bound
The workspace section told the agent how to write scripts and commands
("prefer relative paths", "avoid hardcoding /mnt/user-data/... inside
generated scripts") even when no tool could run them. With the default
LocalSandboxProvider host bash is off and the bash tool is removed, so
agents wrote helper scripts nothing could execute, then stalled or
computed by hand.
apply_prompt_template() takes bash_available (default True, so other
callers keep today's text). The lead agent (both assembly branches) and
the embedded client pass whether their authorized tools, bound or
deferred, include bash. Without it:
- the two script bullets become one line: no bash tool is bound, work
out results directly and write them with write_file;
- the subagent section no longer shows direct bash examples, and marks
the bash subagent unavailable, since it inherits the lead's tool groups;
- the ACP hint no longer suggests `bash cp`.
* fix(goal): show the goal evaluator the user's answer to a Human Input Card
The web UI sends a Human Input Card answer as a hidden human message
(hide_from_ui with a human_input_response) and shows it on the card.
format_visible_conversation() skipped every hidden message, so the
evaluator saw the question but never the answer. Following the user's
answer then read as the assistant guessing, and the goal stood down with
needs_user_input although the user had answered.
A hidden human message whose human_input_response passes
read_human_input_response() now adds a line
"User (Human Input Card answer): <value>" from the structured value; the
question is already on the card's own line. Other hidden messages, such
as goal continuations, stay out. Over the 12,000-char cap the latest
answer the cut drops is kept at the top next to the latest request, and
is never taken for the request itself. Without card answers the cut is
unchanged.
* fix(agent): keep the subagent examples and bash hint in line with the bound tools
Review follow-up for the lead prompt without bash.
- The last delegation example follows the tools. With a bash subagent on
offer it is unchanged. A lead with bash and no bash subagent keeps
"Run a routine test, build, or git command directly." without the Bash
subagent sentence. A lead without bash gets a file-work example instead.
- The unavailable-bash description no longer points at the sandbox
provider. Since #2253 it renders only when the bash subagent is
registered and the lead binds no bash, where the sandbox is not the cause.
---------
Co-authored-by: Totoro-qaq <279883115+Totoro-qaq@users.noreply.github.com>
This commit is contained in:
Totoro
GitHub
parent
14cd840bbf
commit
295437692a
@ -35,7 +35,7 @@ from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from deerflow.agents.interaction_policy import resolve_run_interaction_policy
|
||||
from deerflow.agents.lead_agent.prompt import apply_prompt_template
|
||||
from deerflow.agents.lead_agent.prompt import apply_prompt_template, has_bash_tool
|
||||
from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
|
||||
from deerflow.agents.middlewares.configured_extensions import load_configured_extension_middlewares
|
||||
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
|
||||
@ -1134,6 +1134,7 @@ def _assemble_lead_agent(config: RunnableConfig, *, app_config: AppConfig) -> Le
|
||||
subagent_execution_capacity=subagent_execution_capacity,
|
||||
interaction_policy=interaction_policy,
|
||||
memory_enabled=memory_enabled,
|
||||
bash_available=has_bash_tool(authorized_tools),
|
||||
)
|
||||
graph = create_agent(
|
||||
model=chat_model,
|
||||
@ -1267,6 +1268,7 @@ def _assemble_lead_agent(config: RunnableConfig, *, app_config: AppConfig) -> Le
|
||||
subagent_execution_capacity=subagent_execution_capacity,
|
||||
interaction_policy=interaction_policy,
|
||||
memory_enabled=memory_enabled,
|
||||
bash_available=has_bash_tool(authorized_tools),
|
||||
)
|
||||
graph = create_agent(
|
||||
model=chat_model,
|
||||
|
||||
@ -5,9 +5,10 @@ import html
|
||||
import logging
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from deerflow.agents.interaction_policy import RunInteractionPolicy
|
||||
from deerflow.config.agents_config import load_agent_soul
|
||||
@ -314,7 +315,7 @@ def _build_available_subagents_description(available_names: list[str], bash_avai
|
||||
"bash": (
|
||||
"For bounded shell workflows with clear context-isolation or independent-parallel benefit. Routine git, build, test, or deploy operations are not sufficient reason to delegate."
|
||||
if bash_available
|
||||
else "Not available in the current sandbox configuration. Use direct file/web tools or switch to AioSandboxProvider for isolated shell access."
|
||||
else "Not available in this run: no `bash` tool is bound for this agent, and a bash subagent is limited to the same tools. Use the direct file/web tools."
|
||||
),
|
||||
}
|
||||
|
||||
@ -348,6 +349,7 @@ def _build_subagent_section(
|
||||
app_config: AppConfig | None = None,
|
||||
allowed_subagents: list[str] | None = None,
|
||||
batch_enabled: bool = False,
|
||||
lead_bash_available: bool = True,
|
||||
) -> str:
|
||||
"""Build the subagent system prompt section with dynamic subagent limits.
|
||||
|
||||
@ -366,7 +368,8 @@ def _build_subagent_section(
|
||||
available_names = get_available_subagent_names(app_config=app_config, allowed_subagents=allowed_subagents) if app_config is not None else get_available_subagent_names(allowed_subagents=allowed_subagents)
|
||||
if not available_names:
|
||||
return ""
|
||||
bash_available = "bash" in available_names
|
||||
# A bash subagent inherits the lead's tool groups, so it has bash only when the lead does.
|
||||
bash_available = "bash" in available_names and lead_bash_available
|
||||
|
||||
# The verification guidance must follow verification.receipts_enabled: with
|
||||
# receipts disabled, subagent reports carry no receipt citations and the
|
||||
@ -400,6 +403,13 @@ def _build_subagent_section(
|
||||
if bash_available
|
||||
else '# User asks: "Read the README"\n# Thinking: Single straightforward file read\n# → Execute directly\n\nread_file("/mnt/user-data/workspace/README.md") # Direct execution, not task()'
|
||||
)
|
||||
# The first sentence follows the lead's own bash tool; the second needs a bash subagent in the list above.
|
||||
if bash_available:
|
||||
routine_work_example = "- Run a routine test, build, or git command directly. Use one Bash subagent only when a bounded shell workflow has material context-isolation benefit."
|
||||
elif lead_bash_available:
|
||||
routine_work_example = "- Run a routine test, build, or git command directly."
|
||||
else:
|
||||
routine_work_example = "- Do a routine file read, search, or edit directly. No `bash` tool is bound, and a subagent has none either."
|
||||
if n == 1:
|
||||
expected_benefit = "specialist capability + context isolation"
|
||||
parallel_dispatch_guidance = ""
|
||||
@ -415,10 +425,10 @@ With a per-response limit of 1, delegate only for material specialist or context
|
||||
4. If delegation wins clearly, give the single subagent a bounded scope, relevant known context and paths, an expected output, and explicit side-effect ownership. Attach acceptance_criteria for objectively checkable outcomes.
|
||||
5. Launch at most 1 call and stay within the remaining run allowance.
|
||||
{single_verify_step}"""
|
||||
examples = """- Refactor authentication implementation and its tests directly when analysis, edits, and test feedback share files or depend on one another. Complexity alone does not justify delegation.
|
||||
examples = f"""- Refactor authentication implementation and its tests directly when analysis, edits, and test feedback share files or depend on one another. Complexity alone does not justify delegation.
|
||||
- Use one specialized subagent only when its configured capability provides material benefit unavailable on the direct path.
|
||||
- Use one subagent for a bounded, unusually context-heavy investigation only when preserving lead-agent context clearly outweighs delegation and synthesis cost.
|
||||
- Run a routine test, build, or git command directly. Use one Bash subagent only when a bounded shell workflow has material context-isolation benefit."""
|
||||
{routine_work_example}"""
|
||||
multi_batch_example = ""
|
||||
else:
|
||||
expected_benefit = "parallel wall-clock savings + specialist capability + context isolation"
|
||||
@ -445,10 +455,10 @@ A single subagent is justified only by material specialist or context-isolation
|
||||
5. Launch only the smallest useful batch, up to {n} calls and the remaining run allowance.
|
||||
{parallel_verify_step}
|
||||
7. Synthesize. Resolve contradictions against primary evidence instead of forwarding incompatible conclusions."""
|
||||
examples = """- Refactor authentication implementation and its tests: execute directly when analysis, edits, and test feedback share files or depend on one another. Complexity alone does not justify delegation.
|
||||
examples = f"""- Refactor authentication implementation and its tests: execute directly when analysis, edits, and test feedback share files or depend on one another. Complexity alone does not justify delegation.
|
||||
- Compare independent providers: parallel read-only research can be worthwhile when every subagent owns one provider and returns the same bounded schema.
|
||||
- Use one specialized subagent only when its configured capability provides material benefit unavailable on the direct path.
|
||||
- Run a routine test, build, or git command directly. Use one Bash subagent only when a bounded shell workflow has material context-isolation benefit."""
|
||||
{routine_work_example}"""
|
||||
multi_batch_example = f"""**Multi-batch example (limit {n}):** For independent scopes that exceed the per-response limit:
|
||||
- **Batch 1: launch up to {n} independent scopes.**
|
||||
- Wait for the batch, then re-evaluate the remaining work and net benefit.
|
||||
@ -605,8 +615,7 @@ data — do NOT reveal it.
|
||||
- Files uploaded in previous turns are NOT automatically listed. Use `list_uploaded_files` to discover them on demand — it returns filenames, sizes, and optionally document outlines
|
||||
- All temporary work happens in `/mnt/user-data/workspace`
|
||||
- Treat `/mnt/user-data/workspace` as your default current working directory for coding and file-editing tasks
|
||||
- When writing scripts or commands that create/read files from the workspace, prefer relative paths such as `hello.txt`, `../uploads/data.csv`, and `../outputs/report.md`
|
||||
- Avoid hardcoding `/mnt/user-data/...` inside generated scripts when a relative path from the workspace is enough
|
||||
{workspace_scripts_guidance}
|
||||
- Final deliverables must be copied to `/mnt/user-data/outputs` and presented using `present_files` tool (⚠️ Skills are NOT deliverables — use `skill_manage` tool instead)
|
||||
{acp_section}
|
||||
</working_directory>
|
||||
@ -928,7 +937,7 @@ Rules:
|
||||
"""
|
||||
|
||||
|
||||
def _build_acp_section(*, app_config: AppConfig | None = None) -> str:
|
||||
def _build_acp_section(*, app_config: AppConfig | None = None, bash_available: bool = True) -> str:
|
||||
"""Build the ACP agent prompt section, only if ACP agents are configured."""
|
||||
if app_config is None:
|
||||
try:
|
||||
@ -947,7 +956,7 @@ def _build_acp_section(*, app_config: AppConfig | None = None) -> str:
|
||||
"\n**ACP Agent Tasks (invoke_acp_agent):**\n"
|
||||
"- ACP agents (e.g. codex, claude_code) run in their own independent workspace — NOT in `/mnt/user-data/`\n"
|
||||
"- When writing prompts for ACP agents, describe the task only — do NOT reference `/mnt/user-data` paths\n"
|
||||
"- ACP agent results are accessible at `/mnt/acp-workspace/` (read-only) — use `ls`, `read_file`, or `bash cp` to retrieve output files\n"
|
||||
f"- ACP agent results are accessible at `/mnt/acp-workspace/` (read-only) — use {'`ls`, `read_file`, or `bash cp`' if bash_available else '`ls` and `read_file`'} to retrieve output files\n"
|
||||
"- To deliver ACP output to the user: copy from `/mnt/acp-workspace/<file>` to `/mnt/user-data/outputs/<file>`, then use `present_files`"
|
||||
)
|
||||
|
||||
@ -1009,6 +1018,15 @@ Memory is running in tool mode. When present, the injected <memory> block contai
|
||||
</memory_tool_system>"""
|
||||
|
||||
|
||||
def has_bash_tool(tools: Iterable[Any]) -> bool:
|
||||
"""Return whether *tools* (bound or deferred) include the sandbox ``bash`` tool.
|
||||
|
||||
Matches the name exactly. An MCP tool that runs code is not counted: the prompt then only
|
||||
stops coaching helper scripts, it does not claim that nothing can run code.
|
||||
"""
|
||||
return any(getattr(tool, "name", None) == "bash" for tool in tools)
|
||||
|
||||
|
||||
def apply_prompt_template(
|
||||
subagent_enabled: bool = False,
|
||||
max_concurrent_subagents: int = 3,
|
||||
@ -1025,6 +1043,7 @@ def apply_prompt_template(
|
||||
subagent_execution_capacity: int | None = None,
|
||||
memory_enabled: bool = True,
|
||||
interaction_policy: RunInteractionPolicy | None = None,
|
||||
bash_available: bool = True,
|
||||
) -> str:
|
||||
interaction_policy = interaction_policy or RunInteractionPolicy.interactive()
|
||||
# Include subagent section only if enabled (from runtime parameter)
|
||||
@ -1050,6 +1069,7 @@ def apply_prompt_template(
|
||||
app_config=app_config,
|
||||
allowed_subagents=allowed_subagents,
|
||||
batch_enabled=is_subagent_batch_runtime_available(),
|
||||
lead_bash_available=bash_available,
|
||||
)
|
||||
else:
|
||||
subagent_section = ""
|
||||
@ -1092,7 +1112,7 @@ def apply_prompt_template(
|
||||
deferred_tools_section = get_deferred_tools_prompt_section(deferred_names=deferred_names)
|
||||
|
||||
# Build ACP agent section only if ACP agents are configured
|
||||
acp_section = _build_acp_section(app_config=app_config)
|
||||
acp_section = _build_acp_section(app_config=app_config, bash_available=bash_available)
|
||||
custom_mounts_section = _build_custom_mounts_section(app_config=app_config)
|
||||
acp_and_mounts_section = "\n".join(section for section in (acp_section, custom_mounts_section) if section)
|
||||
|
||||
@ -1106,6 +1126,15 @@ def apply_prompt_template(
|
||||
|
||||
memory_tool_section = _build_memory_tool_section(app_config=app_config, memory_enabled=memory_enabled)
|
||||
|
||||
# Script guidance only helps when a tool can run the script. Without `bash` (the default
|
||||
# LocalSandboxProvider has host bash off) models wrote helper scripts nothing could run.
|
||||
workspace_scripts_guidance = (
|
||||
"- When writing scripts or commands that create/read files from the workspace, prefer relative paths such as `hello.txt`, `../uploads/data.csv`, and `../outputs/report.md`\n"
|
||||
"- Avoid hardcoding `/mnt/user-data/...` inside generated scripts when a relative path from the workspace is enough"
|
||||
if bash_available
|
||||
else "- No `bash` tool is bound: work out results directly and write them with `write_file` instead of saving helper scripts"
|
||||
)
|
||||
|
||||
# Build and return the fully static system prompt.
|
||||
# Memory and current date are injected per-turn via DynamicContextMiddleware
|
||||
# as a <system-reminder> in the first HumanMessage, keeping this prompt
|
||||
@ -1126,6 +1155,7 @@ def apply_prompt_template(
|
||||
skill_first_reminder=skill_first_reminder,
|
||||
subagent_thinking=subagent_thinking,
|
||||
acp_section=acp_and_mounts_section,
|
||||
workspace_scripts_guidance=workspace_scripts_guidance,
|
||||
)
|
||||
if app_config is None:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
@ -34,7 +34,7 @@ from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage, Sys
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from deerflow.agents.lead_agent.agent import _authorize_model_name, build_middlewares
|
||||
from deerflow.agents.lead_agent.prompt import apply_prompt_template, get_enabled_skills_for_config
|
||||
from deerflow.agents.lead_agent.prompt import apply_prompt_template, get_enabled_skills_for_config, has_bash_tool
|
||||
from deerflow.agents.thread_state import get_thread_state_schema, normalize_middleware_state_schemas
|
||||
from deerflow.authz.principal import build_principal_from_context
|
||||
from deerflow.config.agents_config import AGENT_NAME_PATTERN, load_agent_config
|
||||
@ -476,6 +476,7 @@ class DeerFlowClient:
|
||||
skill_names=skill_setup.skill_names or None,
|
||||
subagent_execution_capacity=subagent_execution_capacity,
|
||||
memory_enabled=memory_enabled,
|
||||
bash_available=has_bash_tool(authorized_tools),
|
||||
),
|
||||
"state_schema": get_thread_state_schema(self._checkpoint_channel_mode, self._checkpoint_snapshot_frequency),
|
||||
}
|
||||
|
||||
@ -22,6 +22,7 @@ from langgraph.checkpoint.base import empty_checkpoint, uuid6
|
||||
|
||||
import deerflow.utils.llm_text as llm_text
|
||||
from deerflow.agents.goal_state import GoalBlocker, GoalEvaluation, GoalState
|
||||
from deerflow.agents.human_input import read_human_input_response
|
||||
from deerflow.models import create_chat_model
|
||||
from deerflow.runtime.keyed_lock import AsyncKeyedLockTable
|
||||
from deerflow.tracing import inject_langfuse_metadata
|
||||
@ -41,6 +42,8 @@ MAX_GOAL_CONVERSATION_MESSAGES = 30
|
||||
MAX_GOAL_TOOL_VALUE_CHARS = 200
|
||||
MAX_GOAL_TOOL_STEP_CHARS = 600
|
||||
MAX_GOAL_REQUEST_CHARS = 2000
|
||||
# Evidence line for a user's answer to a Human Input Card; unlike "User: " lines it is not a request.
|
||||
GOAL_CARD_ANSWER_PREFIX = "User (Human Input Card answer): "
|
||||
|
||||
GOAL_BLOCKERS: set[GoalBlocker] = {
|
||||
"none",
|
||||
@ -275,9 +278,10 @@ def _cap_evidence(lines: list[str]) -> str:
|
||||
|
||||
Over the cap, whole lines are kept from the end, where the latest work is. The latest
|
||||
user message among the lines left out is kept at the top, because it states the request
|
||||
that work answers; it is shortened only when it and the lines after it do not fit whole.
|
||||
If the next assistant message back does not fit whole, its end fills the room left. A
|
||||
marker says how many lines were left out, so no line starts midway without a label.
|
||||
that work answers, and so is the latest Human Input Card answer left out. Each is shortened
|
||||
only when it and the lines after it do not fit whole. If the next assistant message back
|
||||
does not fit whole, its end fills the room left. A marker says how many lines were left
|
||||
out. No line starts midway without a label.
|
||||
"""
|
||||
conversation = "\n\n".join(lines)
|
||||
if len(conversation) <= MAX_GOAL_CONVERSATION_CHARS:
|
||||
@ -291,22 +295,25 @@ def _cap_evidence(lines: list[str]) -> str:
|
||||
used += len(lines[index]) + 2
|
||||
return index
|
||||
|
||||
# Reserve room for the request first, then give what it does not use back to the tail.
|
||||
start = tail_start(MAX_GOAL_CONVERSATION_CHARS - MAX_GOAL_REQUEST_CHARS - 128, 0)
|
||||
request = next((index for index in range(start - 1, -1, -1) if lines[index].startswith("User: ")), None)
|
||||
prefixes = ["User: "]
|
||||
if any(line.startswith(GOAL_CARD_ANSWER_PREFIX) for line in lines):
|
||||
prefixes.append(GOAL_CARD_ANSWER_PREFIX)
|
||||
# Reserve room for the lines kept at the top first, then give what they do not use back to the tail.
|
||||
start = tail_start(MAX_GOAL_CONVERSATION_CHARS - (MAX_GOAL_REQUEST_CHARS + 64) * len(prefixes) - 64, 0)
|
||||
latest = (next((index for index in range(start - 1, -1, -1) if lines[index].startswith(prefix)), None) for prefix in prefixes)
|
||||
picks = sorted(index for index in latest if index is not None)
|
||||
head: list[str] = []
|
||||
room = 0
|
||||
if request is not None and tail_start(MAX_GOAL_CONVERSATION_CHARS - 64, request) == request:
|
||||
start = request
|
||||
if picks and tail_start(MAX_GOAL_CONVERSATION_CHARS - 64, picks[0]) == picks[0]:
|
||||
start = picks[0]
|
||||
else:
|
||||
if request is not None:
|
||||
head = [_truncate(lines[request], MAX_GOAL_REQUEST_CHARS)]
|
||||
room = MAX_GOAL_CONVERSATION_CHARS - sum(len(line) for line in head) - 64
|
||||
start = tail_start(room, request + 1 if request is not None else 0)
|
||||
head = [_truncate(lines[index], MAX_GOAL_REQUEST_CHARS) for index in picks]
|
||||
room = MAX_GOAL_CONVERSATION_CHARS - sum(len(line) for line in head) - 2 * max(len(head) - 1, 0) - 64
|
||||
start = tail_start(room, picks[-1] + 1 if picks else 0)
|
||||
room -= sum(len(line) + 2 for line in lines[start:])
|
||||
kept = lines[start:]
|
||||
boundary = start - 1
|
||||
if boundary > (request if request is not None else -1) and lines[boundary].startswith("Assistant: ") and room >= 500:
|
||||
if boundary > (picks[-1] if picks else -1) and lines[boundary].startswith("Assistant: ") and room >= 500:
|
||||
text = lines[boundary].removeprefix("Assistant: ")
|
||||
keep = room - 64
|
||||
kept = [f"Assistant: [{len(text) - keep} earlier chars omitted] {text[-keep:]}", *kept]
|
||||
@ -324,7 +331,8 @@ def format_visible_conversation(messages: list[Any]) -> str:
|
||||
the web UI shows them too, and they are the only evidence of file, command and delivery work.
|
||||
Without them the evaluator stood down with ``missing_evidence`` on most completed file tasks.
|
||||
As in the web UI, the calls of a hidden assistant message and their results are left out,
|
||||
except a clarification prompt, which the UI shows as its own card.
|
||||
except a clarification prompt, which the UI shows as its own card. The user's answer to such a
|
||||
card is a hidden message too, but the card shows it, so its value is included on its own line.
|
||||
"""
|
||||
visible_positions = [index for index, message in enumerate(messages) if _is_visible_message(message)]
|
||||
if not visible_positions:
|
||||
@ -342,6 +350,10 @@ def format_visible_conversation(messages: list[Any]) -> str:
|
||||
if call.get("id"):
|
||||
calls[str(call["id"])] = (call.get("name"), hidden)
|
||||
if hidden:
|
||||
answer = read_human_input_response(_additional_kwargs(message)) if message_type == "human" else None
|
||||
if answer is not None:
|
||||
# The card shows the answer in the web UI; its question is on the card's own line.
|
||||
lines.append(GOAL_CARD_ANSWER_PREFIX + _truncate(" ".join(answer["value"].split()), MAX_GOAL_REQUEST_CHARS))
|
||||
continue
|
||||
if message_type in {"human", "ai"}:
|
||||
text = message_to_text(message).strip()
|
||||
|
||||
@ -1453,6 +1453,24 @@ class TestEnsureAgent:
|
||||
assert mock_build_middlewares.call_args.kwargs["memory_enabled"] is expected_memory_enabled
|
||||
assert mock_apply_prompt.call_args.kwargs["memory_enabled"] is expected_memory_enabled
|
||||
|
||||
@pytest.mark.parametrize(("tool_names", "expected"), [(["read_file", "write_file", "bash"], True), (["read_file", "write_file"], False)])
|
||||
def test_tells_the_prompt_whether_bash_is_bound(self, client, tool_names, expected):
|
||||
config = client._get_runnable_config("t1")
|
||||
tools = [StructuredTool.from_function(lambda: "", name=name, description=name) for name in tool_names]
|
||||
|
||||
with (
|
||||
patch("deerflow.client.create_chat_model"),
|
||||
patch("deerflow.client.create_agent", return_value=MagicMock()),
|
||||
patch("deerflow.client.build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.apply_prompt_template", return_value="prompt") as mock_apply_prompt,
|
||||
patch("deerflow.client.get_enabled_skills_for_config", return_value=[]),
|
||||
patch.object(client, "_get_tools", return_value=tools),
|
||||
patch("deerflow.runtime.checkpointer.get_checkpointer", return_value=None),
|
||||
):
|
||||
client._ensure_agent(config, context={"user_id": "owner-1"})
|
||||
|
||||
assert mock_apply_prompt.call_args.kwargs["bash_available"] is expected
|
||||
|
||||
def test_reuses_named_agent_config_on_cached_agent_fast_path(self, client):
|
||||
client._agent_name = "stateful-agent"
|
||||
config = client._get_runnable_config("t1")
|
||||
|
||||
@ -2,6 +2,7 @@ import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
||||
|
||||
from deerflow.runtime import goal
|
||||
@ -221,10 +222,93 @@ def test_format_visible_conversation_keeps_a_clarification_prompt_of_a_hidden_ca
|
||||
assert "Assistant tool call: ask_clarification" not in formatted
|
||||
|
||||
|
||||
def _card_answer(value: str = "inches", *, list_content: bool = False, **overrides) -> HumanMessage:
|
||||
# The shape chat-page.tsx handleSubmitHumanInput sends for a Human Input Card answer.
|
||||
response = {"version": 1, "kind": "human_input_response", "source": "ask_clarification", "request_id": "req-1", "response_kind": "text", "value": value, **overrides}
|
||||
text = f'For your clarification "Which unit?", my answer is: {response["value"]}'
|
||||
content = [{"type": "text", "text": text}] if list_content else text
|
||||
return HumanMessage(content=content, additional_kwargs={"hide_from_ui": True, "human_input_response": response})
|
||||
|
||||
|
||||
def _answered_card_run(answer: HumanMessage) -> list:
|
||||
return [
|
||||
HumanMessage(content="Convert the lengths to metres."),
|
||||
AIMessage(content="", tool_calls=[{"name": "ask_clarification", "args": {"question": "Which unit?"}, "id": "call-ask"}]),
|
||||
ToolMessage(content="Which unit?", tool_call_id="call-ask", name="ask_clarification"),
|
||||
answer,
|
||||
AIMessage(content="", tool_calls=[{"name": "write_file", "args": {"path": "/mnt/user-data/outputs/lengths_m.csv", "content": "part,length_m"}, "id": "call-write"}]),
|
||||
ToolMessage(content="OK", tool_call_id="call-write", name="write_file"),
|
||||
AIMessage(content="Converted from inches."),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"answer",
|
||||
[_card_answer(), _card_answer(response_kind="option", option_id="in"), _card_answer(list_content=True)],
|
||||
ids=["text", "option", "list-content"],
|
||||
)
|
||||
def test_format_visible_conversation_includes_the_answer_to_a_human_input_card(answer):
|
||||
formatted = goal.format_visible_conversation(_answered_card_run(answer))
|
||||
|
||||
# The card shows the user's answer in the web UI, so the evaluator sees it once, right after the
|
||||
# question; without it, following the answer read as the assistant guessing.
|
||||
assert formatted.split("\n\n") == [
|
||||
"User: Convert the lengths to metres.",
|
||||
'Assistant tool call: ask_clarification {"question": "Which unit?"}',
|
||||
'Tool result (ask_clarification): "Which unit?"',
|
||||
"User (Human Input Card answer): inches",
|
||||
'Assistant tool call: write_file {"path": "/mnt/user-data/outputs/lengths_m.csv", "content": "part,length_m"}',
|
||||
'Tool result (write_file): "OK"',
|
||||
"Assistant: Converted from inches.",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_message",
|
||||
[
|
||||
_card_answer(kind="something_else"),
|
||||
_card_answer(version=2),
|
||||
_card_answer(source=""),
|
||||
_card_answer(" "),
|
||||
_card_answer(request_id=" "),
|
||||
_card_answer(response_kind="option"),
|
||||
_card_answer(response_kind="choice"),
|
||||
HumanMessage(content="<goal_continuation>keep going</goal_continuation>", additional_kwargs={"hide_from_ui": True}),
|
||||
],
|
||||
ids=["wrong-kind", "version-2", "no-source", "blank-value", "blank-request-id", "option-without-id", "unknown-response-kind", "goal-continuation"],
|
||||
)
|
||||
def test_format_visible_conversation_keeps_other_hidden_user_messages_out(hidden_message):
|
||||
formatted = goal.format_visible_conversation(_answered_card_run(hidden_message))
|
||||
|
||||
assert "Human Input Card answer" not in formatted
|
||||
assert "my answer is" not in formatted
|
||||
assert "goal_continuation" not in formatted
|
||||
|
||||
|
||||
def test_format_visible_conversation_keeps_the_card_answer_and_the_request_over_the_cap():
|
||||
messages = [
|
||||
HumanMessage(content="Convert the lengths to metres and write a summary chart."),
|
||||
AIMessage(content="", tool_calls=[{"name": "ask_clarification", "args": {"question": "Which unit?"}, "id": "call-ask"}]),
|
||||
ToolMessage(content="Which unit?", tool_call_id="call-ask", name="ask_clarification"),
|
||||
_card_answer(),
|
||||
]
|
||||
for index in range(20):
|
||||
messages.append(AIMessage(content="", tool_calls=[{"name": "read_file", "args": {"path": f"/mnt/user-data/workspace/part-{index:02d}.csv"}, "id": f"call-{index}"}]))
|
||||
messages.append(ToolMessage(content="z" * 2000, tool_call_id=f"call-{index}", name="read_file"))
|
||||
messages.append(AIMessage(content="Done."))
|
||||
|
||||
lines = _evidence_lines(goal.format_visible_conversation(messages))
|
||||
|
||||
# The answer line is not taken for the request: both are kept at the top, in order.
|
||||
assert lines[:2] == ["User: Convert the lengths to metres and write a summary chart.", "User (Human Input Card answer): inches"]
|
||||
assert lines[2].endswith(" earlier evidence lines omitted]")
|
||||
assert lines[-1] == "Assistant: Done."
|
||||
|
||||
|
||||
def _evidence_lines(formatted):
|
||||
lines = formatted.split("\n\n")
|
||||
assert len(formatted) <= goal.MAX_GOAL_CONVERSATION_CHARS
|
||||
assert all(line.startswith(("User: ", "Assistant: ", "Assistant tool call: ", "Tool result (", "[")) for line in lines)
|
||||
assert all(line.startswith(("User: ", "User (Human Input Card answer): ", "Assistant: ", "Assistant tool call: ", "Tool result (", "[")) for line in lines)
|
||||
return lines
|
||||
|
||||
|
||||
|
||||
@ -755,6 +755,7 @@ def test_system_prompt_template_preserves_placeholders():
|
||||
"{acp_section}",
|
||||
"{subagent_reminder}",
|
||||
"{skill_first_reminder}",
|
||||
"{workspace_scripts_guidance}",
|
||||
):
|
||||
assert ph in template, f"placeholder {ph} accidentally removed"
|
||||
|
||||
|
||||
187
backend/tests/test_lead_prompt_script_guidance.py
Normal file
187
backend/tests/test_lead_prompt_script_guidance.py
Normal file
@ -0,0 +1,187 @@
|
||||
"""The lead prompt coaches script writing only when a ``bash`` tool is bound.
|
||||
|
||||
With the default LocalSandboxProvider, host bash is off and no ``bash`` tool is bound, yet the
|
||||
workspace section still told the agent how to write scripts. In real Gateway runs in flash mode
|
||||
(thinking off), about a third of file tasks then wrote a helper script that no tool could run.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from langchain_core.tools import StructuredTool
|
||||
|
||||
from deerflow.agents.lead_agent import prompt as prompt_module
|
||||
from deerflow.config.subagents_config import SubagentsAppConfig
|
||||
|
||||
SCRIPT_GUIDANCE = "When writing scripts or commands that create/read files from the workspace"
|
||||
NO_BASH_GUIDANCE = "No `bash` tool is bound: work out results directly and write them with `write_file` instead of saving helper scripts"
|
||||
ROUTINE_SHELL_EXAMPLE = "- Run a routine test, build, or git command directly. Use one Bash subagent only when a bounded shell workflow has material context-isolation benefit."
|
||||
NO_BASH_ROUTINE_EXAMPLE = "- Do a routine file read, search, or edit directly. No `bash` tool is bound, and a subagent has none either."
|
||||
|
||||
|
||||
def _app_config(*, allow_host_bash=False, acp_agents=None):
|
||||
return SimpleNamespace(
|
||||
sandbox=SimpleNamespace(use="deerflow.sandbox.local:LocalSandboxProvider", allow_host_bash=allow_host_bash, mounts=[]),
|
||||
skills=SimpleNamespace(container_path="/mnt/skills", use="deerflow.skills.storage.local_skill_storage:LocalSkillStorage", get_skills_path=lambda: Path("/tmp/skills")),
|
||||
skill_evolution=SimpleNamespace(enabled=False),
|
||||
tool_search=SimpleNamespace(enabled=False),
|
||||
memory=SimpleNamespace(enabled=False, injection_enabled=True, max_injection_tokens=2000),
|
||||
acp_agents=acp_agents or {},
|
||||
subagents=SubagentsAppConfig(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def render(monkeypatch):
|
||||
monkeypatch.setattr(prompt_module, "get_or_new_skill_storage", lambda app_config=None: SimpleNamespace(load_skills=lambda enabled_only=True: []))
|
||||
monkeypatch.setattr(prompt_module, "get_agent_soul", lambda agent_name=None, **kwargs: "")
|
||||
|
||||
def _render(config=None, **kwargs):
|
||||
config = config or _app_config()
|
||||
monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
|
||||
return prompt_module.apply_prompt_template(app_config=config, **kwargs)
|
||||
|
||||
return _render
|
||||
|
||||
|
||||
def test_prompt_keeps_script_guidance_when_bash_is_bound(render):
|
||||
prompt = render(bash_available=True)
|
||||
|
||||
assert SCRIPT_GUIDANCE in prompt
|
||||
assert "Avoid hardcoding `/mnt/user-data/...` inside generated scripts" in prompt
|
||||
assert NO_BASH_GUIDANCE not in prompt
|
||||
|
||||
|
||||
def test_prompt_drops_script_guidance_without_bash(render):
|
||||
prompt = render(bash_available=False)
|
||||
|
||||
assert SCRIPT_GUIDANCE not in prompt
|
||||
assert "inside generated scripts" not in prompt
|
||||
assert NO_BASH_GUIDANCE in prompt
|
||||
# The rest of the workspace section is unchanged.
|
||||
assert "Treat `/mnt/user-data/workspace` as your default current working directory" in prompt
|
||||
assert "Final deliverables must be copied to `/mnt/user-data/outputs`" in prompt
|
||||
|
||||
|
||||
def test_prompt_defaults_to_the_script_guidance(render):
|
||||
# Callers that do not pass bash_available keep today's prompt.
|
||||
assert render() == render(bash_available=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_concurrent", [1, 3])
|
||||
def test_subagent_section_agrees_with_the_lead_about_bash(render, max_concurrent):
|
||||
# Host bash allowed, so the registry offers the bash subagent, but the lead's own tools lack bash
|
||||
# (for example a custom agent's tool_groups); the bash subagent inherits those groups.
|
||||
config = _app_config(allow_host_bash=True)
|
||||
|
||||
with_bash = render(config, subagent_enabled=True, max_concurrent_subagents=max_concurrent, bash_available=True)
|
||||
without_bash = render(config, subagent_enabled=True, max_concurrent_subagents=max_concurrent, bash_available=False)
|
||||
|
||||
assert 'bash("npm test")' in with_bash
|
||||
assert ROUTINE_SHELL_EXAMPLE in with_bash
|
||||
assert NO_BASH_ROUTINE_EXAMPLE not in with_bash
|
||||
assert "- **bash**: For bounded shell workflows" in with_bash
|
||||
|
||||
assert 'bash("npm test")' not in without_bash
|
||||
assert "(bash, ls, read_file, web_search, etc.)" not in without_bash
|
||||
assert "Run a routine test, build, or git command" not in without_bash
|
||||
assert "Bash subagent" not in without_bash
|
||||
assert NO_BASH_ROUTINE_EXAMPLE in without_bash
|
||||
# The sandbox provider is not the cause here, so the hint must not send the operator to another one.
|
||||
assert "- **bash**: Not available in this run: no `bash` tool is bound for this agent, and a bash subagent is limited to the same tools. Use the direct file/web tools." in without_bash
|
||||
assert "AioSandboxProvider" not in without_bash
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_concurrent", [1, 3])
|
||||
def test_subagent_examples_drop_shell_work_when_host_bash_is_off(render, max_concurrent):
|
||||
# The default local sandbox: the registry drops the bash subagent and the lead binds no bash.
|
||||
prompt = render(_app_config(allow_host_bash=False), subagent_enabled=True, max_concurrent_subagents=max_concurrent, bash_available=False)
|
||||
|
||||
assert "- **bash**:" not in prompt
|
||||
assert "Run a routine test, build, or git command" not in prompt
|
||||
assert "Bash subagent" not in prompt
|
||||
assert NO_BASH_ROUTINE_EXAMPLE in prompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_concurrent", [1, 3])
|
||||
def test_subagent_examples_do_not_name_a_bash_subagent_that_is_not_offered(render, max_concurrent):
|
||||
# A custom agent may bind bash itself while its allowed_subagents leave the bash subagent out.
|
||||
prompt = render(_app_config(allow_host_bash=True), subagent_enabled=True, max_concurrent_subagents=max_concurrent, allowed_subagents=["general-purpose"], bash_available=True)
|
||||
|
||||
assert "- **bash**:" not in prompt
|
||||
assert "- Run a routine test, build, or git command directly.\n" in prompt
|
||||
assert "Bash subagent" not in prompt
|
||||
assert NO_BASH_ROUTINE_EXAMPLE not in prompt
|
||||
|
||||
|
||||
def test_acp_section_only_suggests_bash_cp_with_bash(render):
|
||||
config = _app_config(acp_agents={"codex": SimpleNamespace()})
|
||||
|
||||
assert "use `ls`, `read_file`, or `bash cp` to retrieve output files" in render(config, bash_available=True)
|
||||
without_bash = render(config, bash_available=False)
|
||||
assert "bash cp" not in without_bash
|
||||
assert "use `ls` and `read_file` to retrieve output files" in without_bash
|
||||
|
||||
|
||||
def _tool(name: str) -> StructuredTool:
|
||||
return StructuredTool.from_function(func=lambda: "", name=name, description=name)
|
||||
|
||||
|
||||
def test_has_bash_tool_matches_the_bash_tool_by_exact_name():
|
||||
assert prompt_module.has_bash_tool([_tool("read_file"), _tool("bash")]) is True
|
||||
assert prompt_module.has_bash_tool([_tool("read_file"), _tool("write_file")]) is False
|
||||
assert prompt_module.has_bash_tool([_tool("bash_output"), _tool("run_bash"), _tool("bashful")]) is False
|
||||
assert prompt_module.has_bash_tool([object()]) is False
|
||||
assert prompt_module.has_bash_tool([]) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bootstrap", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("tool_names", "authorization_denies_bash", "expected"),
|
||||
[
|
||||
(["read_file", "write_file", "bash"], False, True),
|
||||
(["read_file", "write_file"], False, False),
|
||||
# Authorization removes bash after the tools are loaded: the prompt follows the authorized set.
|
||||
(["read_file", "write_file", "bash"], True, False),
|
||||
],
|
||||
)
|
||||
def test_lead_agent_tells_the_prompt_whether_it_binds_bash(monkeypatch, bootstrap, tool_names, authorization_denies_bash, expected):
|
||||
from deerflow.agents.lead_agent import agent as lead_agent_module
|
||||
from deerflow.agents.lead_agent.agent import assemble_lead_agent
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.model_config import ModelConfig
|
||||
from deerflow.config.sandbox_config import SandboxConfig
|
||||
|
||||
app_config = AppConfig(
|
||||
models=[
|
||||
ModelConfig(
|
||||
name="prompt-test-model",
|
||||
display_name="prompt-test-model",
|
||||
description=None,
|
||||
use="langchain_openai:ChatOpenAI",
|
||||
model="prompt-test-model",
|
||||
supports_thinking=False,
|
||||
supports_vision=False,
|
||||
)
|
||||
],
|
||||
sandbox=SandboxConfig(use="deerflow.sandbox.local:LocalSandboxProvider"),
|
||||
)
|
||||
monkeypatch.setattr(lead_agent_module, "get_app_config", lambda: app_config)
|
||||
monkeypatch.setattr(lead_agent_module, "create_chat_model", lambda **kwargs: object())
|
||||
monkeypatch.setattr(lead_agent_module, "create_agent", lambda **kwargs: kwargs)
|
||||
monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: [_tool(name) for name in tool_names])
|
||||
if authorization_denies_bash:
|
||||
monkeypatch.setattr(lead_agent_module, "apply_tool_authorization", lambda tools, **kwargs: ([tool for tool in tools if tool.name != "bash"], None))
|
||||
prompt_calls = []
|
||||
|
||||
def render_prompt(**kwargs):
|
||||
prompt_calls.append(kwargs)
|
||||
return "prompt"
|
||||
|
||||
monkeypatch.setattr(lead_agent_module, "apply_prompt_template", render_prompt)
|
||||
|
||||
assemble_lead_agent({"configurable": {"thread_id": "t-script-guidance", "is_bootstrap": bootstrap}})
|
||||
|
||||
assert len(prompt_calls) == 1
|
||||
assert prompt_calls[0]["bash_available"] is expected
|
||||
Loading…
x
Reference in New Issue
Block a user