From 66dfba31e1791aa11bfb91a05c3d414910f93feb Mon Sep 17 00:00:00 2001 From: ZJPex <144683084+ZJPex@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:07:59 +0800 Subject: [PATCH] fix(agents): bound structured tool-output previews (#6422) * fix(agents): bound structured tool-output previews * docs(agents): keep synopsis guidance within context budget --- README.md | 2 + .../deerflow/agents/middlewares/AGENTS.md | 3 +- .../middlewares/tool_output_synopsis.py | 40 +++-- .../tests/test_tool_output_synopsis_budget.py | 151 ++++++++++++++++++ 4 files changed, 180 insertions(+), 16 deletions(-) create mode 100644 backend/tests/test_tool_output_synopsis_budget.py diff --git a/README.md b/README.md index 0d362023a6..4a5e235b35 100644 --- a/README.md +++ b/README.md @@ -2154,6 +2154,8 @@ request the binary capability retain the legacy JSON/base64 frame protocol. **Summarization**: Within a session, DeerFlow manages context aggressively — summarizing completed sub-tasks, offloading intermediate results to the filesystem, compressing what's no longer immediately relevant. This lets it stay sharp across long, multi-step tasks without blowing the context window. +**Tool Output Preview Budget**: Structured synopses of externalized results have a character budget. Long JSON keys and paths are shortened with `...`, and oversized synopsis bodies include a truncation notice. Raw head/tail samples, the full file reference, and `read_file` guidance are preserved; read the saved file for exact keys or complete output. + **Strict Tool-Call Recovery**: When a provider or middleware interrupts a tool-call loop, DeerFlow now strips provider-level raw tool-call metadata on forced-stop assistant messages and injects placeholder tool results for dangling calls before the next model invocation. This keeps OpenAI-compatible reasoning models that strictly validate `tool_call_id` sequences from failing with malformed history errors. **Visible Tool-Run Completion**: For interactive turns, DeerFlow retries an empty post-tool final response once, then surfaces a visible error instead of reporting a silent successful run. diff --git a/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md b/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md index b8297a7644..08d710ddd6 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md +++ b/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md @@ -53,7 +53,8 @@ No match: skip inspection, log ID. Tests: `test_read_mark_tool_call_correlation. 1. **InputSanitizationMiddleware** - First, so it is the outermost `wrap_model_call` wrapper; every inner middleware (including LLM retries) sees sanitized messages. `additional_kwargs.original_user_content` is server-owned provenance: Gateway strips caller-supplied values for non-internal run requests, trusted IM calls may carry the string they captured before adding transport/file context, and the middleware replaces any non-string value before wrapping. Uploads and sanitization retain first-writer-wins only for validated strings. Caller markers are marked `untrusted_input`, never stripped; scope is every turn. **KnowledgeScopeMiddleware** follows input sanitization: it exposes only Gateway-admitted execution scope, removes scope/display data from model messages, and blocks `knowledge_search` when disabled without reading storage or RAGFlow. -2. **ToolOutputBudgetMiddleware** - Budgets model-bound text/JSON with configured limits and exemptions; externalizes to thread outputs (`tool_output.storage_subdir`, default `.tool-results`, `TOOL_RESULTS_DIRNAME`) with a synopsis + `read_file` reference. Preserve media in mixed JSON results. Blob refs restore across hosts; hash/write failures use inline fallback. Internal outputs bypass workspace/delivery scans. Publication, write elision and bash exit marker: [contract](TOOL_OUTPUT_PERSISTENCE.md). Elision uses shared `tool_call_args`; retain `keep_recent_writes` newest writes. Controls: `elide_superseded_writes`, `superseded_write_min_chars`. +2. **ToolOutputBudgetMiddleware** - Budgets model-bound text/JSON with limits/exemptions; externalizes to thread outputs (`tool_output.storage_subdir`, default `.tool-results`, `TOOL_RESULTS_DIRNAME`) with synopsis/`read_file` refs. Preserve mixed-result media. Restore blob refs across hosts; hash/write failures fall back inline. Internal files bypass workspace/delivery scans. Publication/elision/bash exit marker: [contract](TOOL_OUTPUT_PERSISTENCE.md). Elision: `tool_call_args`; retain newest `keep_recent_writes` writes. Controls: `elide_superseded_writes`, `superseded_write_min_chars`. + Tests: `tests/test_tool_output_synopsis_budget.py`. 3. **ToolResultSanitizationMiddleware** - Neutralizes framework/injection tags (e.g. ``) and boundary markers in *remote-content* tool results (`web_fetch`/`web_search`/`image_search`/`web_capture`) so attacker-controlled fetched pages cannot forge trusted framework context. Mirrors `InputSanitizationMiddleware`'s user-input guardrail for the other untrusted-content entry point; sits inner of `ToolOutputBudgetMiddleware` (neutralizes the raw output, then the budget truncates). Local tool output (bash/read_file) is left untouched. Scope is a name-based allowlist for the first-party web tools, plus every MCP-sourced tool via its `deerflow_mcp` metadata tag, so an MCP server naming its fetcher `fetch_url` is still covered Result-rewriting middlewares between the raw callable boundary and the diff --git a/backend/packages/harness/deerflow/agents/middlewares/tool_output_synopsis.py b/backend/packages/harness/deerflow/agents/middlewares/tool_output_synopsis.py index 244dac093e..61464ca85f 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/tool_output_synopsis.py +++ b/backend/packages/harness/deerflow/agents/middlewares/tool_output_synopsis.py @@ -22,6 +22,7 @@ ToolOutputKind = Literal["json", "csv", "tsv", "yaml", "xml", "code", "text", "u _KEY_LIMIT = 12 _SCALAR_LIMIT = 6 +_LABEL_CHARS = 80 _TABLE_SAMPLE_ROWS = 50 _TABLE_COLUMN_LIMIT = 18 _TEXT_HEADER_LIMIT = 16 @@ -150,12 +151,12 @@ def render_tool_output_preview( # appended (avoids duplicating head/tail bytes in both places). if synopsis.kind == "text" and head_budget + tail_budget > 0 and len(content) > head_budget + tail_budget: synopsis = _summarize_text(content, tool_name=tool_name, include_excerpts=False) - lines = [ + tool_name = _one_line(tool_name, _LABEL_CHARS) + header = [ f"[Full {tool_name} output saved to {virtual_path} ({total} chars, ~{total // 4} tokens).]", f"[Preview kind: {synopsis.kind}. This is a structured synopsis, not a raw head/tail truncation.]", - "", - f"{synopsis.title}:", ] + lines = [f"{synopsis.title}:"] lines.extend(f"- {item}" for item in synopsis.summary) if synopsis.structure: @@ -168,6 +169,15 @@ def render_tool_output_preview( lines.append("Notable items:") lines.extend(f"- {item}" for item in synopsis.notable_items) + # Reuse the head/tail budget, with a floor of two existing short excerpts. + # Keep the file reference and read guidance outside the synopsis budget. + synopsis_budget = max(_TEXT_EXCERPT_CHARS * 2, head_budget + tail_budget) + body = "\n".join(lines) + if len(body) > synopsis_budget: + marker = "\n[Synopsis truncated; use read_file for full output.]" + body = body[: synopsis_budget - len(marker)] + marker + lines = [*header, "", body] + raw_sample = _build_raw_sample(content, head_budget=head_budget, tail_budget=tail_budget, existing=synopsis.sample) if raw_sample: lines.append("") @@ -260,15 +270,15 @@ def _type_name(value: Any) -> str: def _short_value(value: Any) -> str: if isinstance(value, str): - return json.dumps(_clip(value, 80), ensure_ascii=False) - return _clip(repr(value), 80) + return json.dumps(_clip(value, _LABEL_CHARS), ensure_ascii=False) + return _clip(repr(value), _LABEL_CHARS) def _json_shape(value: Any, *, depth: int = 0) -> str: if depth >= _JSON_SHAPE_MAX_DEPTH: return "..." if isinstance(value, dict): - keys = [str(key) for key in list(value.keys())[:_KEY_LIMIT]] + keys = [_one_line(str(key), _LABEL_CHARS) for key in list(value.keys())[:_KEY_LIMIT]] suffix = f": {', '.join(keys)}" if keys else "" return f"object(keys={len(value)}{suffix})" if isinstance(value, list): @@ -279,15 +289,15 @@ def _json_shape(value: Any, *, depth: int = 0) -> str: def _json_path(parent: str, key: Any) -> str: - key_text = str(key) + key_text = _clip(str(key), _LABEL_CHARS) if re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key_text): - return f"{parent}.{key_text}" - return f"{parent}[{json.dumps(key_text, ensure_ascii=False)}]" + return _clip(f"{parent}.{key_text}", _TEXT_EXCERPT_CHARS) + return _clip(f"{parent}[{json.dumps(key_text, ensure_ascii=False)}]", _TEXT_EXCERPT_CHARS) def _json_container_description(value: Any) -> str: if isinstance(value, dict): - keys = [str(key) for key in list(value.keys())[:_KEY_LIMIT]] + keys = [_one_line(str(key), _LABEL_CHARS) for key in list(value.keys())[:_KEY_LIMIT]] suffix = f"; keys {', '.join(keys)}" if keys else "" return f"object keys {len(value)}{suffix}" if isinstance(value, list): @@ -325,7 +335,7 @@ def _json_container_paths(value: Any, *, limit: int = _JSON_STRUCTURE_LIMIT) -> if isinstance(node, list) and node: first = node[0] if isinstance(first, (dict, list)): - walk(first, f"{current_path}[]", depth + 1) + walk(first, _clip(f"{current_path}[]", _TEXT_EXCERPT_CHARS), depth + 1) walk(value, "$", 0) return paths @@ -339,13 +349,13 @@ def _scalar_examples(value: Any, *, path: str = "$", limit: int = _SCALAR_LIMIT) return if isinstance(node, dict): for key, child in list(node.items())[:_KEY_LIMIT]: - walk(child, f"{current}.{key}", depth + 1) + walk(child, _json_path(current, key), depth + 1) if len(examples) >= limit: break return if isinstance(node, list): for index, child in enumerate(node[:2]): - walk(child, f"{current}[{index}]", depth + 1) + walk(child, _clip(f"{current}[{index}]", _TEXT_EXCERPT_CHARS), depth + 1) if len(examples) >= limit: break return @@ -376,8 +386,8 @@ def _try_json(content: str) -> ToolOutputSynopsis | None: # who relied on the old preview to only expose head/tail snippets should # review their tool outputs for sensitive mid-document values. if isinstance(value, dict): - keys = [str(key) for key in value.keys()] - summary.append(f"JSON object with {len(keys)} top-level keys.") + keys = [_one_line(str(key), _LABEL_CHARS) for key in list(value.keys())[:_KEY_LIMIT]] + summary.append(f"JSON object with {len(value)} top-level keys.") summary.append(f"Top-level keys: {', '.join(keys[:_KEY_LIMIT]) or '(none)'}") elif isinstance(value, list): summary.append(f"JSON array with {len(value)} items.") diff --git a/backend/tests/test_tool_output_synopsis_budget.py b/backend/tests/test_tool_output_synopsis_budget.py new file mode 100644 index 0000000000..71fb3652e5 --- /dev/null +++ b/backend/tests/test_tool_output_synopsis_budget.py @@ -0,0 +1,151 @@ +"""Verify structured preview budgets and access to the complete raw output.""" + +import json +from types import SimpleNamespace + +import pytest +from langchain_core.messages import ToolMessage +from langgraph.types import Command + +from deerflow.agents.middlewares import tool_output_synopsis as synopsis_module +from deerflow.agents.middlewares.tool_output_budget_middleware import ToolOutputBudgetMiddleware, _build_preview, _patch_model_messages +from deerflow.agents.middlewares.tool_output_synopsis import ToolOutputSynopsis, build_tool_output_synopsis +from deerflow.config.tool_output_config import ToolOutputConfig + + +@pytest.mark.parametrize("key", ["x" * 20000, "字段🙂" * 5000, '"\\\n' * 5000], ids=["ascii", "unicode", "escaped"]) +@pytest.mark.parametrize("nested", [False, True]) +def test_long_json_labels_and_paths_are_bounded(key, nested): + value = {key: {key: {key: 1}}} if nested else {key: 1} + content = json.dumps(value, ensure_ascii=False) + synopsis = build_tool_output_synopsis(content) + assert synopsis.kind == "json" + for item in synopsis.summary + synopsis.structure + synopsis.notable_items: + assert len(item) <= 1500 + preview = _build_preview(content, tool_name="bash", virtual_path="/mnt/result.json", head_chars=1000, tail_chars=500) + assert len(preview) < 4000 + assert "..." in preview + assert preview.endswith("Use read_file on /mnt/result.json with start_line and end_line to inspect the raw output.") + preview.encode("utf-8") + + +@pytest.mark.parametrize("head,tail", [(0, 0), (1, 0), (0, 1), (1000, 500), (2000, 1000)]) +@pytest.mark.parametrize("kind", ["json", "xml", "csv", "yaml", "text", "unknown"]) +def test_renderer_bounds_aggregate_synopsis_and_keeps_samples(monkeypatch, head, tail, kind): + # Check every format at the shared renderer so no parser bypasses the budget. + monkeypatch.setattr(synopsis_module, "build_tool_output_synopsis", lambda *args, **kwargs: ToolOutputSynopsis(kind, "title", ["s" * 10000], ["p" * 10000] * 24, ["v" * 10000] * 6)) + content = "H" * 5000 + "T" * 5000 + preview = _build_preview(content, tool_name="bash", virtual_path="/mnt/result.txt", head_chars=head, tail_chars=tail) + # Use two existing short excerpts as the synopsis floor; retain raw samples separately. + assert len(preview) <= max(840, head + tail) + head + tail + 600 + if kind != "text" or head + tail == 0: + assert "Synopsis truncated" in preview + assert preview.endswith("Use read_file on /mnt/result.txt with start_line and end_line to inspect the raw output.") + if head: + assert "H" * head in preview + if tail: + assert "T" * tail in preview + + +@pytest.mark.parametrize("offset", [-1, 0, 1]) +def test_synopsis_exact_budget_boundary(monkeypatch, offset): + prefix = "title:\n- " + summary = "中" * (840 - len(prefix) + offset) + monkeypatch.setattr(synopsis_module, "build_tool_output_synopsis", lambda *args, **kwargs: ToolOutputSynopsis("json", "title", [summary], [], [])) + preview = _build_preview("{}", tool_name="bash", virtual_path="/mnt/result.txt", head_chars=0, tail_chars=0) + body = preview.split("\n\n", 1)[1].rsplit("\n\nAccess:", 1)[0] + assert len(body) <= 840 + assert ("Synopsis truncated" in body) == (offset > 0) + if offset <= 0: + assert body == prefix + summary + + +@pytest.mark.parametrize("mode", ["sync", "async", "command"]) +@pytest.mark.asyncio +async def test_model_visible_preview_preserves_externalized_json(tmp_path, mode): + content = json.dumps({"字段🙂" * 5000: {"child": 1}}, ensure_ascii=False) + config = ToolOutputConfig() + middleware = ToolOutputBudgetMiddleware(config=config) + request = SimpleNamespace(tool_call={"name": "bash", "id": "call-json"}, runtime=SimpleNamespace(state={"thread_data": {"outputs_path": str(tmp_path)}})) + message = ToolMessage(content=content, name="bash", tool_call_id="call-json") + result = Command(update={"messages": [message]}) if mode == "command" else message + if mode == "async": + + async def handler(_): + return result + + visible = await middleware.awrap_tool_call(request, handler) + else: + visible = middleware.wrap_tool_call(request, lambda _: result) + if mode == "command": + visible = visible.update["messages"][0] + assert len(visible.content) < 7000 + assert "Preview kind: json" in visible.content + assert "read_file" in visible.content + files = list((tmp_path / ".tool-results").iterdir()) + assert len(files) == 1 + assert files[0].read_text(encoding="utf-8") == content + virtual_path = f"/mnt/user-data/outputs/.tool-results/{files[0].name}" + assert visible.content.count(virtual_path) == 2 + assert _patch_model_messages([visible], config) is None + assert message.content == content + + +def test_short_json_keeps_complete_paths_and_values(): + content = '{"meta":{"source":"unit"},"items":[{"id":1}]}' + preview = _build_preview(content, tool_name="bash", virtual_path="/mnt/result.json", head_chars=0, tail_chars=0) + assert "Top-level keys: meta, items" in preview + assert '$.meta.source: "unit"' in preview + assert "$.items[0].id: 1" in preview + assert "Synopsis truncated" not in preview + + +@pytest.mark.parametrize( + "kind,content", + [ + ("json", json.dumps({str(i): {"field" * 100: list(range(20))} for i in range(12)})), + ("xml", "<" + "x" * 10000 + "/>"), + ("yaml", "---\n" + "key" * 300 + ": 1\n"), + ("csv", ("column" * 1000 + ",id\n") + "value,1\n" * 6), + ("text", '{"invalid":' + "x" * 20000), + ("unknown", "\x00" + "x" * 20000), + ("unknown", "x" * 5000001), + ], + ids=["json", "xml", "yaml", "csv", "invalid-json", "binary", "oversized"], +) +def test_actual_format_and_parse_fallback_stay_bounded(kind, content): + preview = _build_preview(content, tool_name="bash", virtual_path="/mnt/result.txt", head_chars=1000, tail_chars=500) + assert f"Preview kind: {kind}" in preview + assert len(preview) <= 3600 + assert preview.endswith("Use read_file on /mnt/result.txt with start_line and end_line to inspect the raw output.") + + +@pytest.mark.parametrize("size", [79, 80, 81]) +def test_json_label_boundary(size): + key = "k" * size + result = build_tool_output_synopsis(json.dumps({key: 1})) + keys = result.summary[1].removeprefix("Top-level keys: ") + assert len(keys) <= 80 + assert (keys.endswith("...")) == (size > 80) + if size <= 80: + assert f"$.{key}: 1" in result.notable_items + + +def test_storage_failure_keeps_existing_fallback_budget(): + content = json.dumps({"x" * 40000: 1}) + config = ToolOutputConfig(fallback_max_chars=1000) + middleware = ToolOutputBudgetMiddleware(config=config) + request = SimpleNamespace(tool_call={"name": "bash", "id": "call-fallback"}, runtime=SimpleNamespace(state={})) + result = middleware.wrap_tool_call(request, lambda _: ToolMessage(content=content, name="bash", tool_call_id="call-fallback")) + assert len(result.content) <= 1000 + assert "Persistent storage unavailable" in result.content + assert "output saved to" not in result.content + + +def test_long_reference_is_preserved_outside_synopsis_budget(): + path = "/mnt/" + "目录/" * 200 + "result.json" + content = json.dumps({"x" * 20000: 1}) + preview = _build_preview(content, tool_name="tool" * 10000, virtual_path=path, head_chars=0, tail_chars=0) + assert preview.count(path) == 2 + assert len(preview) <= 840 + 2 * len(path) + 600 + assert preview.endswith(f"Use read_file on {path} with start_line and end_line to inspect the raw output.")