From 3b592c2053a43b433789a8a3f552ee94561a0b89 Mon Sep 17 00:00:00 2001 From: Zeren Wang <53075619+Vanzeren@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:56:56 +0200 Subject: [PATCH] feat(harness): subagent receipt citation verification (#5076) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(harness): subagent receipt citation verification - add receipt citation verification core - harvest subagent tool receipts at terminal status - transport subagent receipts and citation verdict via status contract - verify subagent report citations at task write-back - render citation verdicts in the delegation ledger * fix(gateway): strip forged receipt verdicts from the delegations channel normalize_input() and the checkpoint-state mutation sanitizer only stripped server-owned metadata from message-shaped values, so an external caller could submit a delegation entry carrying a forged receipt_verdict that render_delegation_ledger would present as runtime-owned execution evidence. Strip receipt_verdict from caller-supplied delegation entries on both the run-creation and thread-state mutation paths, with regression coverage for each. * fix(harness): close silent-pass gaps in the zero-citation heuristic The action-claim detector missed the most common completion verbs (fixed/added/tested/changed/...) and had no CJK coverage at all, so reports like 'I fixed the bug and added tests.' or '我已经创建了文件并运行了测试。' were treated as claim-free: citation_resolved=True with no ledger warning. Broaden the verb lists and add a language-independent safety net: when the run harvested receipts and a nontrivial (>=240 char) report cites none of them, flag it UNVERIFIED. Short claim-free confirmations remain a vacuous pass. --- backend/app/gateway/services.py | 45 +- .../deerflow/agents/middlewares/AGENTS.md | 4 +- .../agents/middlewares/delegation_ledger.py | 9 + .../middlewares/receipt_verification.py | 172 +++++++ .../agents/middlewares/tool_receipt.py | 131 ++++- .../middlewares/tool_receipt_middleware.py | 49 +- .../harness/deerflow/agents/thread_state.py | 3 + .../harness/deerflow/subagents/AGENTS.md | 2 +- .../harness/deerflow/subagents/executor.py | 132 ++++- .../deerflow/subagents/status_contract.py | 30 ++ .../packages/harness/deerflow/tools/AGENTS.md | 1 + .../deerflow/tools/builtins/task_tool.py | 20 + backend/tests/test_delegation_ledger.py | 77 +++ backend/tests/test_gateway_services.py | 64 ++- backend/tests/test_message_provenance.py | 32 ++ backend/tests/test_receipt_verification.py | 164 ++++++ backend/tests/test_subagent_executor.py | 471 +++++++++++++++++- .../tests/test_subagent_status_contract.py | 74 +++ backend/tests/test_task_tool_core_logic.py | 133 ++++- backend/tests/test_tool_receipt.py | 74 +++ backend/tests/test_tool_receipt_middleware.py | 44 +- 21 files changed, 1682 insertions(+), 49 deletions(-) create mode 100644 backend/packages/harness/deerflow/agents/middlewares/receipt_verification.py create mode 100644 backend/tests/test_receipt_verification.py diff --git a/backend/app/gateway/services.py b/backend/app/gateway/services.py index e77974b6b..0ec631234 100644 --- a/backend/app/gateway/services.py +++ b/backend/app/gateway/services.py @@ -35,7 +35,7 @@ from app.gateway.utils import sanitize_log_param from app.mcp_tasks.errors import PermanentNotificationError from deerflow.agents.middlewares.dynamic_context_middleware import _DYNAMIC_CONTEXT_REMINDER_KEY, _REMINDER_DATE_KEY from deerflow.agents.middlewares.input_sanitization_middleware import frame_untrusted_text -from deerflow.agents.middlewares.tool_receipt import TOOL_RECEIPT_KEY +from deerflow.agents.middlewares.tool_receipt import TOOL_RECEIPT_KEY, TOOL_RECEIPT_LEDGER_KEY from deerflow.agents.middlewares.tool_transform_meta import TOOL_TRANSFORMS_KEY from deerflow.agents.middlewares.view_image_middleware import _IMAGE_CONTEXT_MESSAGE_MARKER_KEY from deerflow.config.app_config import get_app_config @@ -75,6 +75,7 @@ from deerflow.runtime.secret_context import ( ) from deerflow.runtime.stream_modes import normalize_stream_modes from deerflow.runtime.user_context import reset_current_user, set_current_user +from deerflow.subagents.status_contract import SUBAGENT_RECEIPT_VERDICT_KEY, SUBAGENT_TOOL_RECEIPTS_KEY from deerflow.utils.messages import ORIGINAL_USER_CONTENT_KEY from deerflow.utils.thread_id import validate_thread_id @@ -115,7 +116,10 @@ _SERVER_OWNED_MESSAGE_METADATA_KEYS = ( _REMINDER_DATE_KEY, _IMAGE_CONTEXT_MESSAGE_MARKER_KEY, TOOL_RECEIPT_KEY, + TOOL_RECEIPT_LEDGER_KEY, TOOL_TRANSFORMS_KEY, + SUBAGENT_TOOL_RECEIPTS_KEY, + SUBAGENT_RECEIPT_VERDICT_KEY, } ) | PROVENANCE_KEYS @@ -267,6 +271,21 @@ def _strip_external_metadata_from_message_like(item: Any) -> Any: return item +def _strip_external_delegation_verdict(entry: Any) -> Any: + """Remove the runtime-stamped receipt verdict from a caller-supplied + delegation-ledger entry. + + ``receipt_verdict`` is server-owned execution evidence stamped at task + write-back. Ledger entries are plain dicts, not messages, so the + message-metadata stripper never sees them; without this a caller can + persist a forged verdict that ``render_delegation_ledger`` would present + as fact. + """ + if isinstance(entry, dict) and "receipt_verdict" in entry: + return {key: value for key, value in entry.items() if key != "receipt_verdict"} + return entry + + def strip_server_owned_state_metadata(values: Mapping[str, Any]) -> dict[str, Any]: """Remove server-owned message metadata from caller-supplied state values. @@ -282,7 +301,9 @@ def strip_server_owned_state_metadata(values: Mapping[str, Any]) -> dict[str, An """ stripped: dict[str, Any] = {} for channel, value in values.items(): - if isinstance(value, list): + if channel == "delegations" and isinstance(value, list): + stripped[channel] = [_strip_external_delegation_verdict(item) for item in value] + elif isinstance(value, list): stripped[channel] = [_strip_external_metadata_from_message_like(item) for item in value] else: stripped[channel] = _strip_external_metadata_from_message_like(value) @@ -304,12 +325,16 @@ def normalize_input(raw_input: dict[str, Any] | None, *, trusted_internal: bool validation errors are the right shape for clients to retry against. ``original_user_content``, dynamic-context reminder markers, the - transient view-image context marker, and tool receipts are server-owned. - External callers cannot supply them; trusted internal channel calls may - preserve metadata they added before invoking this boundary. + transient view-image context marker, tool receipts, and delegated receipt + metadata/verdicts are server-owned. External callers cannot supply them; + trusted internal channel calls may preserve metadata they added before + invoking this boundary. The same applies to the ``delegations`` channel: + a caller-supplied ledger entry's ``receipt_verdict`` is a forgery and is + stripped before the graph runs. """ if raw_input is None: return {} + result = raw_input messages = raw_input.get("messages") if messages and isinstance(messages, list): converted: list[Any] = [] @@ -328,8 +353,14 @@ def normalize_input(raw_input: dict[str, Any] | None, *, trusted_internal: bool converted.append(msg) if not trusted_internal: converted = [_strip_external_message_metadata(message) for message in converted] - return {**raw_input, "messages": converted} - return raw_input + result = {**raw_input, "messages": converted} + if not trusted_internal: + delegations = result.get("delegations") + if isinstance(delegations, list): + cleaned = [_strip_external_delegation_verdict(entry) for entry in delegations] + if cleaned != delegations: + result = {**result, "delegations": cleaned} + return result _DEFAULT_ASSISTANT_ID = "lead_agent" diff --git a/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md b/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md index 6870deef3..f1d0c3426 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md +++ b/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md @@ -1,5 +1,7 @@ ### Middleware Chain +Persisted delegation verdicts are untrusted durable context; ledger rendering revalidates them and ignores malformed values. + Lead-agent middlewares are assembled in strict order across three functions: the shared base in `packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py` (`_build_runtime_middlewares`, exposed via `build_lead_runtime_middlewares`), then the lead-only middlewares appended in `packages/harness/deerflow/agents/lead_agent/agent.py` (`build_middlewares`). Items marked *(optional)* are appended only when their config/runtime condition holds, so the live chain length varies. **Message provenance.** A middleware that injects or rewrites a message stamps @@ -61,7 +63,7 @@ it to that middleware's declaration in the same change. 10. **SandboxAuditMiddleware** - Audits sandboxed shell/file operations for security logging before tool execution. Command classification is **defense-in-depth and audit, not a security boundary** — the sandbox itself is the isolation boundary. Command substitution is judged by *position*, not by the presence of `$(`: a substitution in **command position** (`$(curl url)`, `` `curl url` ``, the word after a `|`/`&&`/`;`, or any `eval`/`source` argument) executes fetched or interpreted content and is blocked, while **value position** (`x=$(curl url)`, `echo $(curl url)`, an argument, a `for` word list) only captures output and passes (#4611). `_HIGH_RISK_COMMAND_POSITION_PATTERNS` is therefore matched anchored against each split sub-command, never against the whole compound string, and `_split_compound_command(split_pipes=True)` supplies those sub-commands; rules that span a pipe (`| sh`, `base64 -d | ...`) still rely on `_classify_command`'s whole-command Pass 1. `_COMMAND_POSITION_PREFIX` extends the anchor over leading variable assignments and exec wrappers (`FOO=1 $(curl url)`, `env`/`command`/`builtin`/`exec`/`nohup`/`time`/`sudo`/`doas`), which are still command position; its assignment branch requires whitespace before the substitution, which is exactly what keeps `x=$(curl url)` in value position. Two execution contexts are deliberately **position-blind** and matched against the whole command in Pass 1, because they execute what they receive wherever they appear (including as an argument to something else, e.g. `xargs sh -c "$(curl url)"`): an `eval`/`source` argument, and an interpreter's **code-string flag** — `-c` (shells, `python`), `-e` (`perl`/`ruby`/`node`), `-p` (`perl`/`node`), `-r` (`php`) — plus the here-string (`<<<`) that reaches the same place through stdin. All three substitution spellings (`$(cmd`, `<(cmd`, `` `cmd ``) share one `_RISKY_SUBSTITUTION` opener so a rule cannot cover one spelling and miss another. An unquoted newline splits like `;`, because it separates statements the same way: leaving it joined let `echo hi\n$(curl url)` evade the anchored rules that its `;` spelling triggers. A heredoc body is data rather than statements, so `_split_compound_command` records headers (`<