mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-09 21:49:37 +00:00
feat(harness): subagent receipt citation verification (#5076)
* feat(harness): subagent receipt citation verification - add receipt citation verification core - harvest subagent tool receipts at terminal status - transport subagent receipts and citation verdict via status contract - verify subagent report citations at task write-back - render citation verdicts in the delegation ledger * fix(gateway): strip forged receipt verdicts from the delegations channel normalize_input() and the checkpoint-state mutation sanitizer only stripped server-owned metadata from message-shaped values, so an external caller could submit a delegation entry carrying a forged receipt_verdict that render_delegation_ledger would present as runtime-owned execution evidence. Strip receipt_verdict from caller-supplied delegation entries on both the run-creation and thread-state mutation paths, with regression coverage for each. * fix(harness): close silent-pass gaps in the zero-citation heuristic The action-claim detector missed the most common completion verbs (fixed/added/tested/changed/...) and had no CJK coverage at all, so reports like 'I fixed the bug and added tests.' or '我已经创建了文件并运行了测试。' were treated as claim-free: citation_resolved=True with no ledger warning. Broaden the verb lists and add a language-independent safety net: when the run harvested receipts and a nontrivial (>=240 char) report cites none of them, flag it UNVERIFIED. Short claim-free confirmations remain a vacuous pass.
This commit is contained in:
Zeren Wang
GitHub
parent
b41354d75f
commit
3b592c2053
@ -35,7 +35,7 @@ from app.gateway.utils import sanitize_log_param
|
||||
from app.mcp_tasks.errors import PermanentNotificationError
|
||||
from deerflow.agents.middlewares.dynamic_context_middleware import _DYNAMIC_CONTEXT_REMINDER_KEY, _REMINDER_DATE_KEY
|
||||
from deerflow.agents.middlewares.input_sanitization_middleware import frame_untrusted_text
|
||||
from deerflow.agents.middlewares.tool_receipt import TOOL_RECEIPT_KEY
|
||||
from deerflow.agents.middlewares.tool_receipt import TOOL_RECEIPT_KEY, TOOL_RECEIPT_LEDGER_KEY
|
||||
from deerflow.agents.middlewares.tool_transform_meta import TOOL_TRANSFORMS_KEY
|
||||
from deerflow.agents.middlewares.view_image_middleware import _IMAGE_CONTEXT_MESSAGE_MARKER_KEY
|
||||
from deerflow.config.app_config import get_app_config
|
||||
@ -75,6 +75,7 @@ from deerflow.runtime.secret_context import (
|
||||
)
|
||||
from deerflow.runtime.stream_modes import normalize_stream_modes
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
from deerflow.subagents.status_contract import SUBAGENT_RECEIPT_VERDICT_KEY, SUBAGENT_TOOL_RECEIPTS_KEY
|
||||
from deerflow.utils.messages import ORIGINAL_USER_CONTENT_KEY
|
||||
from deerflow.utils.thread_id import validate_thread_id
|
||||
|
||||
@ -115,7 +116,10 @@ _SERVER_OWNED_MESSAGE_METADATA_KEYS = (
|
||||
_REMINDER_DATE_KEY,
|
||||
_IMAGE_CONTEXT_MESSAGE_MARKER_KEY,
|
||||
TOOL_RECEIPT_KEY,
|
||||
TOOL_RECEIPT_LEDGER_KEY,
|
||||
TOOL_TRANSFORMS_KEY,
|
||||
SUBAGENT_TOOL_RECEIPTS_KEY,
|
||||
SUBAGENT_RECEIPT_VERDICT_KEY,
|
||||
}
|
||||
)
|
||||
| PROVENANCE_KEYS
|
||||
@ -267,6 +271,21 @@ def _strip_external_metadata_from_message_like(item: Any) -> Any:
|
||||
return item
|
||||
|
||||
|
||||
def _strip_external_delegation_verdict(entry: Any) -> Any:
|
||||
"""Remove the runtime-stamped receipt verdict from a caller-supplied
|
||||
delegation-ledger entry.
|
||||
|
||||
``receipt_verdict`` is server-owned execution evidence stamped at task
|
||||
write-back. Ledger entries are plain dicts, not messages, so the
|
||||
message-metadata stripper never sees them; without this a caller can
|
||||
persist a forged verdict that ``render_delegation_ledger`` would present
|
||||
as fact.
|
||||
"""
|
||||
if isinstance(entry, dict) and "receipt_verdict" in entry:
|
||||
return {key: value for key, value in entry.items() if key != "receipt_verdict"}
|
||||
return entry
|
||||
|
||||
|
||||
def strip_server_owned_state_metadata(values: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Remove server-owned message metadata from caller-supplied state values.
|
||||
|
||||
@ -282,7 +301,9 @@ def strip_server_owned_state_metadata(values: Mapping[str, Any]) -> dict[str, An
|
||||
"""
|
||||
stripped: dict[str, Any] = {}
|
||||
for channel, value in values.items():
|
||||
if isinstance(value, list):
|
||||
if channel == "delegations" and isinstance(value, list):
|
||||
stripped[channel] = [_strip_external_delegation_verdict(item) for item in value]
|
||||
elif isinstance(value, list):
|
||||
stripped[channel] = [_strip_external_metadata_from_message_like(item) for item in value]
|
||||
else:
|
||||
stripped[channel] = _strip_external_metadata_from_message_like(value)
|
||||
@ -304,12 +325,16 @@ def normalize_input(raw_input: dict[str, Any] | None, *, trusted_internal: bool
|
||||
validation errors are the right shape for clients to retry against.
|
||||
|
||||
``original_user_content``, dynamic-context reminder markers, the
|
||||
transient view-image context marker, and tool receipts are server-owned.
|
||||
External callers cannot supply them; trusted internal channel calls may
|
||||
preserve metadata they added before invoking this boundary.
|
||||
transient view-image context marker, tool receipts, and delegated receipt
|
||||
metadata/verdicts are server-owned. External callers cannot supply them;
|
||||
trusted internal channel calls may preserve metadata they added before
|
||||
invoking this boundary. The same applies to the ``delegations`` channel:
|
||||
a caller-supplied ledger entry's ``receipt_verdict`` is a forgery and is
|
||||
stripped before the graph runs.
|
||||
"""
|
||||
if raw_input is None:
|
||||
return {}
|
||||
result = raw_input
|
||||
messages = raw_input.get("messages")
|
||||
if messages and isinstance(messages, list):
|
||||
converted: list[Any] = []
|
||||
@ -328,8 +353,14 @@ def normalize_input(raw_input: dict[str, Any] | None, *, trusted_internal: bool
|
||||
converted.append(msg)
|
||||
if not trusted_internal:
|
||||
converted = [_strip_external_message_metadata(message) for message in converted]
|
||||
return {**raw_input, "messages": converted}
|
||||
return raw_input
|
||||
result = {**raw_input, "messages": converted}
|
||||
if not trusted_internal:
|
||||
delegations = result.get("delegations")
|
||||
if isinstance(delegations, list):
|
||||
cleaned = [_strip_external_delegation_verdict(entry) for entry in delegations]
|
||||
if cleaned != delegations:
|
||||
result = {**result, "delegations": cleaned}
|
||||
return result
|
||||
|
||||
|
||||
_DEFAULT_ASSISTANT_ID = "lead_agent"
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
### Middleware Chain
|
||||
|
||||
Persisted delegation verdicts are untrusted durable context; ledger rendering revalidates them and ignores malformed values.
|
||||
|
||||
Lead-agent middlewares are assembled in strict order across three functions: the shared base in `packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py` (`_build_runtime_middlewares`, exposed via `build_lead_runtime_middlewares`), then the lead-only middlewares appended in `packages/harness/deerflow/agents/lead_agent/agent.py` (`build_middlewares`). Items marked *(optional)* are appended only when their config/runtime condition holds, so the live chain length varies.
|
||||
|
||||
**Message provenance.** A middleware that injects or rewrites a message stamps
|
||||
@ -61,7 +63,7 @@ it to that middleware's declaration in the same change.
|
||||
10. **SandboxAuditMiddleware** - Audits sandboxed shell/file operations for security logging before tool execution. Command classification is **defense-in-depth and audit, not a security boundary** — the sandbox itself is the isolation boundary. Command substitution is judged by *position*, not by the presence of `$(`: a substitution in **command position** (`$(curl url)`, `` `curl url` ``, the word after a `|`/`&&`/`;`, or any `eval`/`source` argument) executes fetched or interpreted content and is blocked, while **value position** (`x=$(curl url)`, `echo $(curl url)`, an argument, a `for` word list) only captures output and passes (#4611). `_HIGH_RISK_COMMAND_POSITION_PATTERNS` is therefore matched anchored against each split sub-command, never against the whole compound string, and `_split_compound_command(split_pipes=True)` supplies those sub-commands; rules that span a pipe (`| sh`, `base64 -d | ...`) still rely on `_classify_command`'s whole-command Pass 1. `_COMMAND_POSITION_PREFIX` extends the anchor over leading variable assignments and exec wrappers (`FOO=1 $(curl url)`, `env`/`command`/`builtin`/`exec`/`nohup`/`time`/`sudo`/`doas`), which are still command position; its assignment branch requires whitespace before the substitution, which is exactly what keeps `x=$(curl url)` in value position. Two execution contexts are deliberately **position-blind** and matched against the whole command in Pass 1, because they execute what they receive wherever they appear (including as an argument to something else, e.g. `xargs sh -c "$(curl url)"`): an `eval`/`source` argument, and an interpreter's **code-string flag** — `-c` (shells, `python`), `-e` (`perl`/`ruby`/`node`), `-p` (`perl`/`node`), `-r` (`php`) — plus the here-string (`<<<`) that reaches the same place through stdin. All three substitution spellings (`$(cmd`, `<(cmd`, `` `cmd ``) share one `_RISKY_SUBSTITUTION` opener so a rule cannot cover one spelling and miss another. An unquoted newline splits like `;`, because it separates statements the same way: leaving it joined let `echo hi\n$(curl url)` evade the anchored rules that its `;` spelling triggers. A heredoc body is data rather than statements, so `_split_compound_command` records headers (`<<EOF`, `<<-EOF`, `<<'EOF'`) and consumes their bodies verbatim at the newline that starts them — otherwise a body line beginning with `$(curl url)` would be promoted to a command position the shell never creates. Two things that look like headers must not open one, or a body that never terminates swallows every following statement: `<<<` is a here-string (both a lookahead and a lookbehind are needed, or the trailing `<<` of `<<< "text"` reads as a heredoc with delimiter `text`), and a `<<` inside `$(( ... ))` / `(( ... ))` is a bit shift, so arithmetic depth is tracked alongside the quote flags. That is a heuristic, not shell parsing: it exists only to avoid manufacturing command positions *and* to avoid destroying real ones. An unterminated body consumes the rest of the string; an unclosed `((` only disables heredoc detection, so newlines keep splitting and the failure direction stays towards seeing more command positions rather than fewer. Known, deliberate gaps: process substitution outside `eval`/`source` (`. <(curl u)`) is not detected — closing it would require real shell parsing, which is out of scope for this layer. Two-step forms (`x=$(curl u); eval "$x"`) are inherent rather than incidental: any rule that allows output capture allows the first statement, and connecting it to the later `eval` needs dataflow analysis, not pattern matching. There is currently no config gate: the middleware is appended unconditionally in `_build_runtime_middlewares`, so it applies to both the lead agent and subagents.
|
||||
11. **ReadBeforeWriteMiddleware** - *(optional, if `read_before_write.enabled`, default on)* Outermost write gate (issue #3857): `read_file` stamps a content hash onto its ToolMessage; `write_file` (append/overwrite-existing) and `str_replace` are blocked unless the newest mark for that path matches the file's current hash. Sits outside ToolProgressMiddleware and ToolErrorHandlingMiddleware so a blocked write returns immediately without consuming a ToolProgress slot. Blocked results call `normalize_tool_result` directly to stamp `deerflow_tool_meta` (`recoverable_by_model=True`) before returning, keeping the result well-formed for any outer consumer. Marks live on messages, so summarization dropping the read result invalidates the gate automatically; writes never refresh marks, forcing a re-read between consecutive edits. Gate check + tool execution are serialized per (thread, path) so same-turn parallel writes cannot reuse one stale mark; on sandboxes whose `read_file` reports failures as `"Error: ..."` strings instead of raising (AIO/E2B), uninspectable targets fail open (creation proceeds, no mark stamped)
|
||||
12. **ToolProgressMiddleware** - *(optional, if `tool_progress.enabled`)* State-machine-based stagnation guard (RFC #3177). Outer wrapper around ToolErrorHandlingMiddleware so its `wrap_tool_call` receives results already stamped with `deerflow_tool_meta`. Tracks per-(thread, tool) consecutive "no-new-info" calls across three error categories: (a) `recoverable_by_model=True` (no_results, not_found, permission, Jaccard-duplicate success): ACTIVE → WARNED (terminal — hint re-injected on each subsequent problem); (b) `recoverable_by_model=False, action≠stop` (rate_limited, transient): ACTIVE → WARNED → BLOCKED after `warn_escalation_count` more problems; (c) `recoverable_by_model=False, action=stop` (auth, config, internal): immediately BLOCKED on first occurrence. **Division of labor with LoopDetectionMiddleware:** ToolProgressMiddleware is a result-quality guard — fires after tool execution and blocks specific tools that stop producing new information; LoopDetectionMiddleware is a call-pattern guard — fires after the model responds and hard-stops the whole turn when the model repeatedly issues identical tool_calls. Both can inject HumanMessage hints in the same model call without conflict; neither reads the other's internal state.
|
||||
13. **ToolReceiptMiddleware + ToolErrorHandlingMiddleware** - `ToolReceiptMiddleware` is *(optional, if `verification.receipts_enabled`, default on)*. It is the **outermost `wrap_tool_call` layer** — registered ahead of entries 9-12 — because Guardrail/SandboxAudit/ReadBeforeWrite/ToolProgress can short-circuit a call with their own ToolMessage (and SandboxAudit rebuilds medium-risk results); an inner receipt layer would silently gap the ledger on those results (ordering constraints in `deerflow.extensions.ordering`). Normal results still carry the `deerflow_tool_meta` status ToolErrorHandlingMiddleware stamps on the inner return path; short-circuit messages self-stamp meta or fall back to `message.status`. It stamps deterministic provenance (tool name, status, args/output hashes, byte count, timestamp) onto direct `ToolMessage` results and every matching `ToolMessage` carried in `Command.update.messages`, including delegated `task`, `present_file`, `view_image`, and `tool_search` results; before model calls it derives a hidden receipt ledger (display ids r1..rN) from message state, and when the 2,000-character budget is exceeded the newest receipts are retained in chronological order with their original ids plus an older-receipts omission marker. `ToolErrorHandlingMiddleware` receives `AppConfig`, converts tool exceptions into error `ToolMessage`s so the run can continue instead of aborting, stamps every result with `deerflow_tool_meta` (status / error_type / recoverable_by_model / recommended_next_action / source) via `tool_result_meta.normalize_tool_result`, stamps structured metadata for task exception wrappers, and stamps skill-read metadata for downstream durable-context capture. Task tool result text is generated from the same status/result/error inputs as the structured metadata so callers do not hand-write a second protocol string.
|
||||
13. **ToolReceiptMiddleware + ToolErrorHandlingMiddleware** - `ToolReceiptMiddleware` is *(optional, if `verification.receipts_enabled`, default on)*. It is the **outermost `wrap_tool_call` layer** — registered ahead of entries 9-12 — because Guardrail/SandboxAudit/ReadBeforeWrite/ToolProgress can short-circuit a call with their own ToolMessage (and SandboxAudit rebuilds medium-risk results); an inner receipt layer would silently gap the ledger on those results (ordering constraints in `deerflow.extensions.ordering`). Normal results still carry the `deerflow_tool_meta` status ToolErrorHandlingMiddleware stamps on the inner return path; short-circuit messages self-stamp meta or fall back to `message.status`. It stamps deterministic provenance (tool name, status, args/output hashes, byte count, timestamp) onto direct `ToolMessage` results and every matching `ToolMessage` carried in `Command.update.messages`, including delegated `task`, `present_file`, `view_image`, and `tool_search` results; before model calls it derives a hidden receipt ledger (display ids r1..rN) from message state, and when the 2,000-character budget is exceeded the newest receipts are retained in chronological order with their original ids plus an older-receipts omission marker. Rendering returns both the text and its retained receipt subset; every response that received a ledger carries only that exact server-owned subset, never omitted receipts. Snapshot validation accepts a strictly consecutive positive original-id range (for example `r24`–`r30`) rather than requiring `r1`, so subagent terminal citation verification resolves ids against evidence present in the citing turn even when later summarization drops and renumbers tool messages. Model-generated citation IDs are digit-bounded before integer conversion; oversized IDs are ignored as malformed input rather than raising through task write-back. Citation parsing deduplicates exact `(id, anchor)` pairs, not IDs alone, so repeated identical references stay compact while every distinct anchor claim is verified. Gateway strips delegated receipts/verdicts from external messages. `ToolErrorHandlingMiddleware` receives `AppConfig`, converts tool exceptions into error `ToolMessage`s so the run can continue instead of aborting, stamps every result with `deerflow_tool_meta` (status / error_type / recoverable_by_model / recommended_next_action / source) via `tool_result_meta.normalize_tool_result`, stamps structured metadata for task exception wrappers, and stamps skill-read metadata for downstream durable-context capture. Task tool result text is generated from the same status/result/error inputs as the structured metadata so callers do not hand-write a second protocol string.
|
||||
|
||||
Authorization identity plumbing is independent of whether authorization enforcement is enabled. Gateway removes client-supplied `is_internal` / `authz_attributes` / `channel_user_id`, derives `is_internal` only from the server-owned `request.state.auth_source`, and accepts `channel_user_id` only from an internally authenticated IM caller's top-level `body.context`; free-form `body.config` can never supply it. `build_principal_from_context` is the shared Principal builder for assembly-time authorization and `GuardrailAuthorizationAdapter`; it applies `default_role`, strict-boolean internal provenance, and copy-on-read `authz_attributes`. The built-in RBAC provider validates `authorization.default_role` during provider resolution so an unknown fallback role fails agent construction instead of degrading into an empty tool set. Task delegation carries `is_internal` plus copied attributes through `SubagentExecutor`, while `GuardrailMiddleware` maps the same runtime fields into `GuardrailRequest`. Phase 1B applies Layer 1 before deferred-tool assembly on the lead, native-subagent, and embedded-client paths, then passes the same provider instance into Layer 2. Framework-provided `describe_skill` and memory tools are included in Layer 1 but restored to their legacy post-`tool_search` ordering afterward. `DeerFlowClient.stream()` treats its in-process caller as trusted and accepts the same identity fields as keyword overrides; it includes the complete Principal in its agent cache key and deep-copies nested attributes so caller mutation cannot make a stale tool set look current.
|
||||
|
||||
|
||||
@ -9,6 +9,7 @@ from typing import Any
|
||||
|
||||
from langchain_core.messages import AIMessage, AnyMessage, ToolMessage
|
||||
|
||||
from deerflow.agents.middlewares.receipt_verification import render_citation_verdict, validate_receipt_verdict
|
||||
from deerflow.agents.thread_state import DelegationEntry
|
||||
from deerflow.subagents.status_contract import (
|
||||
read_subagent_result_metadata,
|
||||
@ -135,6 +136,9 @@ def extract_delegations(messages: list[AnyMessage]) -> list[DelegationEntry]:
|
||||
stop_reason = structured.get("stop_reason")
|
||||
if stop_reason:
|
||||
entry["stop_reason"] = stop_reason
|
||||
receipt_verdict = structured.get("receipt_verdict")
|
||||
if receipt_verdict:
|
||||
entry["receipt_verdict"] = receipt_verdict
|
||||
result_text = structured.get("result_brief") or structured.get("error") or _STATUS_ONLY_RESULT_BRIEFS.get(structured["status"])
|
||||
if result_text:
|
||||
result_sha256 = structured.get("result_sha256") or hashlib.sha256(result_text.encode("utf-8")).hexdigest()
|
||||
@ -161,6 +165,11 @@ def _render_entry_line(entry: DelegationEntry) -> str:
|
||||
result_brief = entry.get("result_brief")
|
||||
if result_brief:
|
||||
line += f" -> {_escape_context_text(_bound_text(result_brief, _LEDGER_ENTRY_RESULT_RENDER_CAP))}"
|
||||
receipt_verdict = validate_receipt_verdict(entry.get("receipt_verdict"))
|
||||
if receipt_verdict is not None:
|
||||
segment = render_citation_verdict(receipt_verdict)
|
||||
if segment:
|
||||
line += f" · {segment}"
|
||||
return line
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,172 @@
|
||||
"""Parent-side verification of subagent report citations against tool receipts.
|
||||
|
||||
Layer 1 consumption (RFC #4651 PR2): a subagent's final report cites receipt
|
||||
ids (``[rN]``, optionally anchored ``[rN tool_name]``); this module
|
||||
cross-checks those citations against the execution record harvested from the
|
||||
child's own message stream. Pure functions only — no IO, no LLM calls.
|
||||
|
||||
Vocabulary layering: the summary boolean is ``citation_resolved``, never
|
||||
``satisfied``/``verified``/``passed`` — strong-positive words are reserved
|
||||
for the runtime hard gate so the model never conflates advisory execution
|
||||
evidence with task acceptance.
|
||||
|
||||
Display ids are positional over the ledger shown for one model call. The
|
||||
receipt middleware stamps that exact ledger on the resulting assistant
|
||||
message, and terminal harvest uses the snapshot rather than renumbering the
|
||||
post-compaction tool-message tail. The verdict remains advisory, not a gate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import TypedDict
|
||||
|
||||
from deerflow.agents.middlewares.tool_receipt import ToolReceipt, parse_citations
|
||||
|
||||
VERDICT_SOURCE = "receipt_citations"
|
||||
VERDICT_REQUIREMENT = "cited_ids_in_execution_record"
|
||||
|
||||
#: Zero-citation heuristic: a completed report making action claims with no
|
||||
#: receipt citations is a weak-negative signal, not a clean bill. Verb list
|
||||
#: plus path-with-extension patterns; false positives cost one UNVERIFIED
|
||||
#: line, false negatives let false success through — biased toward sensitive.
|
||||
_ACTION_VERB_RE = re.compile(
|
||||
r"\b(wrote|written|created|saved|generated|ran|executed|uploaded|downloaded"
|
||||
r"|deleted|modified|updated|installed|deployed|fetched|built|compiled"
|
||||
r"|produced|exported|fixed|added|changed|removed|implemented|patched"
|
||||
r"|refactored|renamed|moved|merged|committed|edited|replaced|tested"
|
||||
r"|verified|cleaned|configured)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
#: CJK reports have no word boundaries, so the English list never fires on
|
||||
#: them; match common action verbs directly instead.
|
||||
_CJK_ACTION_VERB_RE = re.compile(
|
||||
r"创建|生成|写入|保存|修改|更新|删除|运行|执行|安装|部署|上传|下载"
|
||||
r"|修复|添加|新增|编写|编译|构建|导出|测试|提交|移动|重命名|配置|替换|清理"
|
||||
)
|
||||
_FILE_PATH_RE = re.compile(r"(?:/[\w.\-]+){2,}|\b[\w.\-]+\.(?:py|md|txt|json|ya?ml|csv|html|js|ts|sh|log|pdf|png|jpe?g)\b")
|
||||
|
||||
#: Safety net beyond any verb list: a harvested non-empty receipt ledger means
|
||||
#: the subagent demonstrably executed tools, so a paragraph-length report that
|
||||
#: cites none of them is UNVERIFIED regardless of language or phrasing. Short
|
||||
#: status confirmations stay a vacuous pass.
|
||||
_NONTRIVIAL_REPORT_MIN_CHARS = 240
|
||||
|
||||
#: Anti-automation-bias: model-visible verdict text always states its boundary.
|
||||
_LIMITATION = "execution evidence only, does not validate claim correctness"
|
||||
|
||||
|
||||
class CitationFailure(TypedDict):
|
||||
id: str
|
||||
reason: str
|
||||
|
||||
|
||||
class ReceiptVerdict(TypedDict):
|
||||
source: str
|
||||
requirement: str
|
||||
citation_resolved: bool
|
||||
cited: list[str]
|
||||
resolved: list[str]
|
||||
failed: list[CitationFailure]
|
||||
unknown: list[str]
|
||||
no_citation_claims: bool
|
||||
|
||||
|
||||
def _has_action_claims(report_text: str) -> bool:
|
||||
return bool(_ACTION_VERB_RE.search(report_text) or _CJK_ACTION_VERB_RE.search(report_text) or _FILE_PATH_RE.search(report_text))
|
||||
|
||||
|
||||
def verify_receipt_citations(report_text: str, receipts: list[ToolReceipt]) -> ReceiptVerdict:
|
||||
"""Cross-check every citation in the report against the harvested ledger."""
|
||||
by_id = {receipt["id"]: receipt for receipt in receipts}
|
||||
cited: list[str] = []
|
||||
resolved: list[str] = []
|
||||
failed: list[CitationFailure] = []
|
||||
unknown: list[str] = []
|
||||
for rid, anchor in parse_citations(report_text):
|
||||
cited.append(rid)
|
||||
receipt = by_id.get(rid)
|
||||
if receipt is None:
|
||||
unknown.append(rid)
|
||||
continue
|
||||
if receipt["status"] != "success":
|
||||
failed.append({"id": rid, "reason": f"receipt status={receipt['status']}"})
|
||||
continue
|
||||
if anchor is not None and anchor != receipt["tool_name"]:
|
||||
failed.append({"id": rid, "reason": f"anchor mismatch: cited as {anchor}, receipt {rid} is {receipt['tool_name']}"})
|
||||
continue
|
||||
resolved.append(rid)
|
||||
no_citation_claims = not cited and (_has_action_claims(report_text) or bool(receipts) and len(report_text.strip()) >= _NONTRIVIAL_REPORT_MIN_CHARS)
|
||||
if cited:
|
||||
citation_resolved = not failed and not unknown
|
||||
else:
|
||||
# Claim-free report with nothing to check: vacuous pass, renders nothing.
|
||||
citation_resolved = not no_citation_claims
|
||||
return ReceiptVerdict(
|
||||
source=VERDICT_SOURCE,
|
||||
requirement=VERDICT_REQUIREMENT,
|
||||
citation_resolved=citation_resolved,
|
||||
cited=cited,
|
||||
resolved=resolved,
|
||||
failed=failed,
|
||||
unknown=unknown,
|
||||
no_citation_claims=no_citation_claims,
|
||||
)
|
||||
|
||||
|
||||
def validate_receipt_verdict(value: object) -> ReceiptVerdict | None:
|
||||
"""Structural check for a persisted verdict (read side trusts nothing)."""
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
source = value.get("source")
|
||||
requirement = value.get("requirement")
|
||||
citation_resolved = value.get("citation_resolved")
|
||||
no_citation_claims = value.get("no_citation_claims")
|
||||
if not isinstance(source, str) or not isinstance(requirement, str):
|
||||
return None
|
||||
if not isinstance(citation_resolved, bool) or not isinstance(no_citation_claims, bool):
|
||||
return None
|
||||
|
||||
def _str_list(key: str) -> list[str] | None:
|
||||
items = value.get(key)
|
||||
if not isinstance(items, list) or any(not isinstance(item, str) for item in items):
|
||||
return None
|
||||
return list(items)
|
||||
|
||||
cited = _str_list("cited")
|
||||
resolved = _str_list("resolved")
|
||||
unknown = _str_list("unknown")
|
||||
if cited is None or resolved is None or unknown is None:
|
||||
return None
|
||||
raw_failed = value.get("failed")
|
||||
if not isinstance(raw_failed, list):
|
||||
return None
|
||||
failed: list[CitationFailure] = []
|
||||
for entry in raw_failed:
|
||||
if not isinstance(entry, dict) or not isinstance(entry.get("id"), str) or not isinstance(entry.get("reason"), str):
|
||||
return None
|
||||
failed.append({"id": entry["id"], "reason": entry["reason"]})
|
||||
return ReceiptVerdict(
|
||||
source=source,
|
||||
requirement=requirement,
|
||||
citation_resolved=citation_resolved,
|
||||
cited=cited,
|
||||
resolved=resolved,
|
||||
failed=failed,
|
||||
unknown=unknown,
|
||||
no_citation_claims=no_citation_claims,
|
||||
)
|
||||
|
||||
|
||||
def render_citation_verdict(verdict: ReceiptVerdict) -> str:
|
||||
"""Render the verdict as the delegation-ledger citation segment."""
|
||||
if verdict["no_citation_claims"]:
|
||||
return "citations: UNVERIFIED — action claims without receipt citations"
|
||||
if not verdict["cited"]:
|
||||
return ""
|
||||
parts = [f"{len(verdict['resolved'])} resolved"]
|
||||
if verdict["failed"]:
|
||||
parts.append(f"{len(verdict['failed'])} failed")
|
||||
if verdict["unknown"]:
|
||||
parts.append(f"{len(verdict['unknown'])} unknown")
|
||||
return f"citations: {', '.join(parts)} — {_LIMITATION}"
|
||||
@ -34,18 +34,61 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from typing import TypedDict
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
|
||||
from deerflow.agents.middlewares.tool_result_meta import TOOL_META_KEY
|
||||
|
||||
TOOL_RECEIPT_KEY = "deerflow_tool_receipt"
|
||||
TOOL_RECEIPT_LEDGER_KEY = "deerflow_tool_receipt_ledger"
|
||||
|
||||
_HASH_LEN = 16
|
||||
_RENDER_CHAR_BUDGET = 2000
|
||||
|
||||
#: Single source of truth for the citation wire format. Id assignment, ledger
|
||||
#: validation, the model-facing prompt example, and the parent-side verifier
|
||||
#: all derive from these — the format changes in exactly one place.
|
||||
RECEIPT_ID_PREFIX = "r"
|
||||
_MAX_RECEIPT_ID_DIGITS = 10
|
||||
|
||||
|
||||
def receipt_id(position: int) -> str:
|
||||
"""Display id for the ``position``-th receipt (1-based): ``r1``..``rN``."""
|
||||
return f"{RECEIPT_ID_PREFIX}{position}"
|
||||
|
||||
|
||||
#: ``[r2]`` bare or ``[r2 write_file]`` anchored. The optional label lets the
|
||||
#: verifier sanity-check claim-evidence coherence.
|
||||
CITATION_RE = re.compile(rf"\[{RECEIPT_ID_PREFIX}(\d+)(?:\s+([A-Za-z_][\w.-]*))?\]")
|
||||
|
||||
|
||||
def format_citation(rid: str, tool_name: str | None = None) -> str:
|
||||
"""Canonical model-facing citation (``[r2]`` bare, ``[r2 write_file]`` anchored)."""
|
||||
return f"[{rid} {tool_name}]" if tool_name else f"[{rid}]"
|
||||
|
||||
|
||||
def parse_citations(text: str) -> list[tuple[str, str | None]]:
|
||||
"""Pull ``(id, anchor)`` pairs out of report prose, deduped first-seen."""
|
||||
seen: set[tuple[str, str | None]] = set()
|
||||
citations: list[tuple[str, str | None]] = []
|
||||
for match in CITATION_RE.finditer(text):
|
||||
digits = match.group(1)
|
||||
# Model output is untrusted. Bound the decimal string before int()
|
||||
# so an enormous citation id cannot trip Python's conversion limit and
|
||||
# turn an otherwise successful task into a verification exception.
|
||||
if len(digits) > _MAX_RECEIPT_ID_DIGITS:
|
||||
continue
|
||||
rid = receipt_id(int(digits))
|
||||
citation = (rid, match.group(2))
|
||||
if citation in seen:
|
||||
continue
|
||||
seen.add(citation)
|
||||
citations.append(citation)
|
||||
return citations
|
||||
|
||||
|
||||
class ToolReceipt(TypedDict):
|
||||
id: str # display id, assigned by extract_tool_receipts ("r1"..)
|
||||
@ -93,11 +136,11 @@ def extract_tool_receipts(messages: list) -> list[ToolReceipt]:
|
||||
if not isinstance(message, ToolMessage):
|
||||
continue
|
||||
receipt = (message.additional_kwargs or {}).get(TOOL_RECEIPT_KEY)
|
||||
if not _is_valid_receipt(receipt):
|
||||
if not is_valid_receipt(receipt):
|
||||
continue
|
||||
receipts.append(
|
||||
ToolReceipt(
|
||||
id=f"r{len(receipts) + 1}",
|
||||
id=receipt_id(len(receipts) + 1),
|
||||
tool_call_id=receipt["tool_call_id"],
|
||||
tool_name=receipt["tool_name"],
|
||||
status=receipt["status"],
|
||||
@ -110,10 +153,55 @@ def extract_tool_receipts(messages: list) -> list[ToolReceipt]:
|
||||
return receipts
|
||||
|
||||
|
||||
def extract_citing_turn_receipts(messages: list) -> list[ToolReceipt] | None:
|
||||
"""Return the ledger snapshot shown to the last citing assistant turn.
|
||||
|
||||
``ToolReceiptMiddleware`` stamps this runtime-owned snapshot on each model
|
||||
response that received a receipt ledger. Unlike a terminal re-scan of tool
|
||||
messages, these positional display ids remain the ids the response could
|
||||
actually cite even if summarization later compacts and renumbers history.
|
||||
"""
|
||||
for message in reversed(messages):
|
||||
if not isinstance(message, AIMessage):
|
||||
continue
|
||||
raw_ledger = (message.additional_kwargs or {}).get(TOOL_RECEIPT_LEDGER_KEY)
|
||||
if raw_ledger is None:
|
||||
continue
|
||||
if not isinstance(raw_ledger, list):
|
||||
return None
|
||||
receipts: list[ToolReceipt] = []
|
||||
first_position: int | None = None
|
||||
for index, receipt in enumerate(raw_ledger):
|
||||
if not is_valid_receipt(receipt):
|
||||
return None
|
||||
rid = receipt.get("id")
|
||||
match = re.fullmatch(rf"{re.escape(RECEIPT_ID_PREFIX)}([1-9]\d*)", rid) if isinstance(rid, str) else None
|
||||
if match is None:
|
||||
return None
|
||||
if first_position is None:
|
||||
first_position = int(match.group(1))
|
||||
if rid != receipt_id(first_position + index):
|
||||
return None
|
||||
receipts.append(
|
||||
ToolReceipt(
|
||||
id=receipt["id"],
|
||||
tool_call_id=receipt["tool_call_id"],
|
||||
tool_name=receipt["tool_name"],
|
||||
status=receipt["status"],
|
||||
args_sha256=receipt["args_sha256"],
|
||||
output_sha256=receipt["output_sha256"],
|
||||
output_bytes=receipt["output_bytes"],
|
||||
created_at=receipt["created_at"],
|
||||
)
|
||||
)
|
||||
return receipts
|
||||
return None
|
||||
|
||||
|
||||
_RECEIPT_STR_FIELDS = ("tool_call_id", "tool_name", "status", "args_sha256", "output_sha256", "created_at")
|
||||
|
||||
|
||||
def _is_valid_receipt(receipt: object) -> bool:
|
||||
def is_valid_receipt(receipt: object) -> bool:
|
||||
"""Structural check for a persisted receipt (types only, not provenance)."""
|
||||
if not isinstance(receipt, dict):
|
||||
return False
|
||||
@ -123,13 +211,25 @@ def _is_valid_receipt(receipt: object) -> bool:
|
||||
return isinstance(output_bytes, int) and not isinstance(output_bytes, bool)
|
||||
|
||||
|
||||
def render_tool_receipts(receipts: list[ToolReceipt], *, max_chars: int = _RENDER_CHAR_BUDGET) -> str:
|
||||
"""Render the receipt ledger as model-visible context (empty -> "")."""
|
||||
def render_tool_receipts_with_snapshot(
|
||||
receipts: list[ToolReceipt],
|
||||
*,
|
||||
max_chars: int = _RENDER_CHAR_BUDGET,
|
||||
) -> tuple[str, list[ToolReceipt]]:
|
||||
"""Render a ledger and return the exact receipt subset visible in it.
|
||||
|
||||
The retained receipts keep their original display ids. Callers that
|
||||
persist a citing-turn ledger must use this snapshot rather than the full
|
||||
input list, otherwise a citation could resolve against an entry omitted by
|
||||
the model context budget.
|
||||
"""
|
||||
if not receipts:
|
||||
return ""
|
||||
return "", []
|
||||
lines = [
|
||||
"## Tool receipts (execution record)",
|
||||
"Cite receipt ids (e.g. [r1]) in your final report for every claim about an action you took.",
|
||||
# The example is generated from format_citation/parse_citations' shared
|
||||
# format so the instruction can never drift from the verifier.
|
||||
f"Cite receipt ids (e.g. {format_citation(receipt_id(1), 'write_file')}) in your final report for every claim about an action you took.",
|
||||
# Anti-automation-bias (design rule 4): the ledger always states its
|
||||
# evidence boundary so the model never reads provenance as endorsement.
|
||||
"Execution evidence only — receipts record that a call happened and its status; they do not validate claim correctness or task acceptance.",
|
||||
@ -137,14 +237,27 @@ def render_tool_receipts(receipts: list[ToolReceipt], *, max_chars: int = _RENDE
|
||||
receipt_lines = [f"- [{receipt['id']}] {receipt['tool_name']} status={receipt['status']} args_sha256={receipt['args_sha256']} output_sha256={receipt['output_sha256']} bytes={receipt['output_bytes']}" for receipt in receipts]
|
||||
if len("\n".join([*lines, *receipt_lines])) <= max_chars:
|
||||
lines.extend(receipt_lines)
|
||||
retained_receipts = receipts
|
||||
else:
|
||||
omission = "- ... older receipts omitted (context budget)"
|
||||
retained: list[str] = []
|
||||
retained_count = 0
|
||||
for line in reversed(receipt_lines):
|
||||
candidate = [*lines, omission, line, *retained]
|
||||
if len("\n".join(candidate)) > max_chars:
|
||||
break
|
||||
retained.insert(0, line)
|
||||
retained_count += 1
|
||||
lines.extend([omission, *retained])
|
||||
retained_receipts = receipts[-retained_count:] if retained_count else []
|
||||
rendered = "\n".join(lines)
|
||||
return rendered if len(rendered) <= max_chars else rendered[: max(0, max_chars - 4)] + "\n..."
|
||||
if len(rendered) > max_chars:
|
||||
rendered = rendered[: max(0, max_chars - 4)] + "\n..."
|
||||
retained_receipts = []
|
||||
return rendered, retained_receipts
|
||||
|
||||
|
||||
def render_tool_receipts(receipts: list[ToolReceipt], *, max_chars: int = _RENDER_CHAR_BUDGET) -> str:
|
||||
"""Render the receipt ledger as model-visible context (empty -> "")."""
|
||||
rendered, _ = render_tool_receipts_with_snapshot(receipts, max_chars=max_chars)
|
||||
return rendered
|
||||
|
||||
@ -23,12 +23,19 @@ from typing import override
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain.agents.middleware.types import ModelCallResult, ModelRequest, ModelResponse
|
||||
from langchain_core.messages import HumanMessage, ToolMessage
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
from langgraph.prebuilt.tool_node import ToolCallRequest
|
||||
from langgraph.types import Command
|
||||
|
||||
from deerflow.agents.middlewares.message_utils import insert_after_leading_system_messages, is_genuine_user_message
|
||||
from deerflow.agents.middlewares.tool_receipt import TOOL_RECEIPT_KEY, extract_tool_receipts, make_tool_receipt, render_tool_receipts
|
||||
from deerflow.agents.middlewares.tool_receipt import (
|
||||
TOOL_RECEIPT_KEY,
|
||||
TOOL_RECEIPT_LEDGER_KEY,
|
||||
ToolReceipt,
|
||||
extract_tool_receipts,
|
||||
make_tool_receipt,
|
||||
render_tool_receipts_with_snapshot,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -125,11 +132,7 @@ class ToolReceiptMiddleware(AgentMiddleware[AgentState]):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _inject(self, request: ModelRequest) -> ModelRequest:
|
||||
if not self._should_render(request):
|
||||
return request
|
||||
receipts = extract_tool_receipts(list(request.messages))
|
||||
ledger = render_tool_receipts(receipts)
|
||||
def _inject(self, request: ModelRequest, ledger: str) -> ModelRequest:
|
||||
if not ledger:
|
||||
return request
|
||||
ledger_message = HumanMessage(
|
||||
@ -139,13 +142,40 @@ class ToolReceiptMiddleware(AgentMiddleware[AgentState]):
|
||||
messages = insert_after_leading_system_messages(list(request.messages), [ledger_message])
|
||||
return request.override(messages=messages)
|
||||
|
||||
def _prepare_model_call(self, request: ModelRequest) -> tuple[ModelRequest, list[ToolReceipt] | None]:
|
||||
if not self._should_render(request):
|
||||
return request, None
|
||||
receipts = extract_tool_receipts(list(request.messages))
|
||||
ledger, rendered_receipts = render_tool_receipts_with_snapshot(receipts)
|
||||
return self._inject(request, ledger), rendered_receipts
|
||||
|
||||
@staticmethod
|
||||
def _stamp_citing_ledger(result: ModelCallResult, receipts: list[ToolReceipt] | None) -> ModelCallResult:
|
||||
if receipts is None:
|
||||
return result
|
||||
if isinstance(result, AIMessage):
|
||||
messages = [result]
|
||||
else:
|
||||
response = getattr(result, "model_response", result)
|
||||
messages = getattr(response, "result", [])
|
||||
for message in messages:
|
||||
if not isinstance(message, AIMessage):
|
||||
continue
|
||||
kwargs = dict(message.additional_kwargs or {})
|
||||
# Runtime-owned and always overwritten so provider output cannot
|
||||
# forge the ledger against which its citations will be checked.
|
||||
kwargs[TOOL_RECEIPT_LEDGER_KEY] = [dict(receipt) for receipt in receipts]
|
||||
message.additional_kwargs = kwargs
|
||||
return result
|
||||
|
||||
@override
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelCallResult:
|
||||
return handler(self._inject(request))
|
||||
prepared, receipts = self._prepare_model_call(request)
|
||||
return self._stamp_citing_ledger(handler(prepared), receipts)
|
||||
|
||||
@override
|
||||
async def awrap_model_call(
|
||||
@ -153,4 +183,5 @@ class ToolReceiptMiddleware(AgentMiddleware[AgentState]):
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelCallResult:
|
||||
return await handler(self._inject(request))
|
||||
prepared, receipts = self._prepare_model_call(request)
|
||||
return self._stamp_citing_ledger(await handler(prepared), receipts)
|
||||
|
||||
@ -177,6 +177,9 @@ class DelegationEntry(TypedDict):
|
||||
# turn_capped / loop_capped. The status stays completed/failed; this field
|
||||
# is the additive signal that distinguishes a capped run from a clean one.
|
||||
stop_reason: NotRequired[str]
|
||||
# RFC #4651 PR2: parent-side citation-check verdict (advisory execution
|
||||
# evidence), stamped at task write-back; absent on legacy history.
|
||||
receipt_verdict: NotRequired[dict]
|
||||
created_at: str
|
||||
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
**Date context (#4781)**: Every built-in subagent execution registers `SubagentDateContextMiddleware` immediately before `SystemMessageCoalescingMiddleware`. Its one-time `before_agent` hook adds a hidden framework-owned `SystemMessage` containing only `<current_date>` before the first model call; it does not read `AppConfig.memory`, call the memory manager, rewrite the task `HumanMessage`, or inherit the lead agent's frozen-conversation/midnight lifecycle. The coalescer merges that reminder with the subagent's static prompt so strict providers still receive exactly one leading `SystemMessage`. The lead-only `DynamicContextMiddleware` registration and its date, optional-memory, and midnight-update behavior remain unchanged.
|
||||
**Execution**: Ordinary and durable-batch native subagents submit coroutines directly to one persistent isolated event loop. Gateway/embedded startup installs one process-wide async FIFO admission controller (default 3 running, bounded queue). Direct `create_deerflow_agent` callers can instead pass a caller-owned `SubagentRuntime`; reuse the same instance across graphs so its bound `task`, optional batch tools/service, middleware limits, and `SubagentExecutor` all share one controller without reading global YAML. An owned batch service must be started before graph construction and stopped at application shutdown. Waiters hold no scheduler thread, and cancellation/timeout release queue/slot ownership.
|
||||
**Concurrency and total delegation cap**: Ordinary `task` concurrency is resolved once as the minimum of the per-run request, the startup-frozen `subagent_runtime.max_running`, and the schema safety ceiling (1-64), then shared by the lead prompt and `SubagentLimitMiddleware`. Hot reloads must not make either layer advertise more capacity than the already-created process controller; a changed startup-only value takes effect only after restart. The same middleware separately enforces `subagents.max_total_per_run` (default 6, config schema 1-50, runtime override `max_total_subagents` clamped to the same range) against current-run entries in the durable delegation ledger, so a long lead-agent run cannot bypass concurrency limits by launching repeated legal-sized batches at each planning checkpoint, but historical delegations from previous runs in the same thread do not consume the new run's budget. Explicit `batch_task` work does not consume or relax that ordinary-run ledger: its persisted total/live/running limits live under `subagent_batches`. Gateway `run_agent()` and embedded `DeerFlowClient.stream()` both provide a per-invocation `run_id` in runtime context; `DeerFlowClient.stream()` also tags its input `HumanMessage` with that same id so durable-context capture can identify the current request boundary. Gateway resume paths may not append a new `HumanMessage`, so the worker also exposes the pre-run checkpoint's message ids in runtime context; durable-context capture uses that as the current-run boundary and never re-tags older task calls as the resumed run. When no delegation slots remain, task calls are stripped, provider raw tool-call metadata is synced, `finish_reason` is forced to `stop`, and a visible "subagent delegation limit" note is appended so the agent can synthesize already-collected results. Default subagent timeout `subagents.timeout_seconds=1800` (30 min) and built-in `general-purpose` `max_turns=150`.
|
||||
**Flow**: Ordinary `task()` → `SubagentExecutor` → shared process slot → result polling/SSE. Explicit `batch_task()` → durable batch/item rows → lease-based batch service (`subagents/batch_service.py`, started by Gateway or an explicit direct runtime) → the same `SubagentExecutor`/process slots → bounded stored result and owner-scoped API/JSONL export. Batch mode is selected only by the explicit tool, never inferred from prompt size. Executor queue rejection/timeout occurs before model execution and therefore releases the durable lease without consuming an item attempt; real execution failure and expired leases still consume the retry budget. User cancellation terminalizes every nonterminal item immediately and clears its lease, fencing any stale worker completion. Direct runtimes provide the tools and worker but not Gateway's HTTP/UI surface. `task_started` carries the resolved effective model name. The per-subagent `SubagentTokenCollector` publishes a cumulative usage snapshot to the shared `SubagentResult` after every completed LLM response; the next `task_running` event carries that snapshot, so collapsed workspace cards can update without re-accounting parent-run totals. Terminal ToolMessage metadata (`subagent_model_name`, `subagent_token_usage`) and the persisted `subagent.end` event retain the model/usage after reload; absent provider usage stays absent rather than being estimated as zero.
|
||||
**Flow**: Ordinary `task()` → `SubagentExecutor` → shared process slot → result polling/SSE. Explicit `batch_task()` → durable batch/item rows → lease-based batch service (`subagents/batch_service.py`, started by Gateway or an explicit direct runtime) → the same `SubagentExecutor`/process slots → bounded stored result and owner-scoped API/JSONL export. Batch mode is selected only by the explicit tool, never inferred from prompt size. Executor queue rejection/timeout occurs before model execution and therefore releases the durable lease without consuming an item attempt; real execution failure and expired leases still consume the retry budget. User cancellation terminalizes every nonterminal item immediately and clears its lease, fencing any stale worker completion. Background cancellation resolves the result/future under `_background_tasks_lock` but calls `Future.cancel()` only after releasing it, because cancellation may synchronously invoke the completion callback that reacquires the registry lock. Direct runtimes provide the tools and worker but not Gateway's HTTP/UI surface. `task_started` carries the resolved effective model name. The per-subagent `SubagentTokenCollector` publishes a cumulative usage snapshot to the shared `SubagentResult` after every completed LLM response; the next `task_running` event carries that snapshot, so collapsed workspace cards can update without re-accounting parent-run totals. Terminal ToolMessage metadata (`subagent_model_name`, `subagent_token_usage`) and the persisted `subagent.end` event retain the model/usage after reload; absent provider usage stays absent rather than being estimated as zero. The executor caches one resolved `AppConfig` snapshot (explicit or `get_app_config()` fallback) for agent assembly, deferred setup, and receipt harvesting, so `verification.receipts_enabled=false` remains authoritative on both construction paths. Terminal tool receipts are harvested before `try_set_terminal` and committed with the other payload fields under the same state lock, so status polling cannot observe a terminal result before its receipt metadata is available. Each yielded values chunk becomes the latest terminal-harvest state and immediately publishes its harvested receipts to the shared result before cooperative cancellation is checked. Tool-ended cancellation/failure evidence uses the current ToolMessage scan, but a completed result always uses the bounded ledger snapshot attached to the assistant text being returned—even when a max-turn partial ends on a later tool chunk—so omitted receipts cannot validate its citations; a missing/malformed completed snapshot fails closed with no receipts. Therefore direct task cancellation and both execution/polling timeouts retain the latest execution evidence even when cancellation interrupts before another stream boundary.
|
||||
**Events**: `task_started`, `task_running`, `task_completed`/`task_failed`/`task_timed_out`
|
||||
**Handled LLM failures**: `LLMErrorHandlingMiddleware` deliberately converts provider/model exceptions into an `AIMessage` so the graph can end cleanly, stamping `additional_kwargs.deerflow_error_fallback=true` plus error metadata. Clean graph termination does not imply subagent success: `SubagentExecutor` inspects the last assistant message at terminalization and maps a marked fallback to `SubagentStatus.FAILED`, which then emits `task_failed` and the existing structured `subagent_error`. Only the marker is authoritative — error-looking assistant prose without it remains a normal completed result, so neither the executor nor frontend parses display text as a status protocol.
|
||||
**Guardrail caps & `stop_reason` (#3875 Phase 2)**: three independent axes can end a subagent run early, and all now surface *why* through one additive field rather than a new status enum. **Turn axis**: `recursion_limit` on the subagent `run_config` equals `max_turns`, so exhausting the turn budget raises `GraphRecursionError` from `agent.astream`; `executor.py::_aexecute` catches it specifically (before the generic `except Exception`). **Token axis**: `TokenBudgetMiddleware` is attached per-agent via `build_subagent_runtime_middlewares` from `subagents.token_budget` (default `max_tokens` **coupled to `summarization.enabled`** — 1,000,000 when subagent summarization is on, 2,000,000 when off, warn at 0.7, hard-stop at 1.0; a user-set budget always wins regardless of the switch — #3875 Phase 3; a backstop against a subagent that burns tokens on trivial work). It does *not* raise: at the hard-stop threshold it strips the in-flight turn's tool calls, forces `finish_reason="stop"`, and lets the run complete naturally with a final answer. **Loop axis**: `LoopDetectionMiddleware` (attached at the same point) catches repeated identical tool-call sets — or one tool *type* called many times with varying args — and its hard-stop likewise strips `tool_calls` and forces a final answer without raising, recording `loop_capped`. Each guard exposes its cap on a per-`run_id` `consume_stop_reason(run_id)` accessor; `_aexecute` collects **every** middleware with that method (duck-typed via `hasattr`, so the executor has no import coupling to the guard classes) and surfaces the first non-`None` reason — adding a future guard needs no executor change. **Surfacing**: whichever axis fired, `_aexecute` stamps a normal status plus an additive reason — `completed` + `stop_reason=token_capped|turn_capped|loop_capped` when a usable final answer (or partial recovered from the last streamed chunk via `_extract_final_result` → `utils/messages.py::message_content_to_text`, returning a `"No response Generated"` sentinel when no text survived) was produced; `failed` + `stop_reason=turn_capped` when nothing usable survived. `SubagentResult.stop_reason` flows through `task_tool.py::_task_result_command` → `format_subagent_result_message` (renders `Task Succeeded (capped: ...)` / `Task failed (capped: ...)`) and `make_subagent_additional_kwargs`, which stamps the additive `subagent_stop_reason` key alongside the normal `subagent_status`. **Why additive, not an enum**: a new status value would break v1 consumers; an optional field is ignored by older frontends and ledger readers, so the cross-language contract (`contracts/subagent_status_contract.json` v2 + `subagents/status_contract.py` + `frontend/.../subtask-result.ts`, pinned by `test_status_values_match_contract` / `test_stop_reason_values_match_contract`) stays backward-compatible. The durable delegation ledger captures `stop_reason` onto the entry and renders model-facing guidance ("hit a guardrail cap with a partial result; reuse it, retry tighter, or raise the per-agent budget (`max_turns` / `token_budget`)") so the lead reuses a capped completion knowingly instead of mistaking it for a clean one. (Phase 1 shipped this surfacing as a `MAX_TURNS_REACHED` status enum in #3949; Phase 2 replaced that enum with the additive `stop_reason` field per the agreed design — the `max_turns_reached` status value and `SubagentStatus.MAX_TURNS_REACHED` are gone.)
|
||||
|
||||
@ -102,6 +102,11 @@ class SubagentResult:
|
||||
completed_at: When execution completed.
|
||||
ai_messages: List of complete AI messages (as dicts) generated during execution.
|
||||
admission_failure: Whether capacity rejected/timed out before execution started.
|
||||
tool_receipts: The child's tool receipts harvested from its terminal
|
||||
message stream (RFC #4651 PR2). ``None`` when the run ended before
|
||||
streaming produced a state (e.g. pre-stream cancellation), when
|
||||
receipts are disabled, or when harvesting failed; an empty list
|
||||
means the stream carried no stamped receipts (zero tool calls).
|
||||
"""
|
||||
|
||||
task_id: str
|
||||
@ -117,6 +122,7 @@ class SubagentResult:
|
||||
token_usage_records: list[dict[str, int | str | None]] = field(default_factory=list)
|
||||
usage_reported: bool = False
|
||||
admission_failure: bool = False
|
||||
tool_receipts: list[dict[str, Any]] | None = field(default=None, kw_only=True)
|
||||
cancel_event: threading.Event = field(default_factory=threading.Event, repr=False)
|
||||
_state_lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False)
|
||||
|
||||
@ -131,6 +137,21 @@ class SubagentResult:
|
||||
if not self.status.is_terminal:
|
||||
self.token_usage_records = list(records)
|
||||
|
||||
def update_tool_receipts(self, receipts: list[dict[str, Any]] | None) -> None:
|
||||
"""Publish receipts from the latest yielded state while still running."""
|
||||
if receipts is None:
|
||||
return
|
||||
with self._state_lock:
|
||||
if not self.status.is_terminal:
|
||||
self.tool_receipts = [dict(receipt) for receipt in receipts]
|
||||
|
||||
def snapshot_tool_receipts(self) -> list[dict[str, Any]] | None:
|
||||
"""Copy the latest published receipts for a racing terminal writer."""
|
||||
with self._state_lock:
|
||||
if self.tool_receipts is None:
|
||||
return None
|
||||
return [dict(receipt) for receipt in self.tool_receipts]
|
||||
|
||||
def try_set_terminal(
|
||||
self,
|
||||
status: SubagentStatus,
|
||||
@ -142,6 +163,7 @@ class SubagentResult:
|
||||
ai_messages: list[dict[str, Any]] | None = None,
|
||||
token_usage_records: list[dict[str, int | str | None]] | None = None,
|
||||
admission_failure: bool = False,
|
||||
tool_receipts: list[dict[str, Any]] | None = None,
|
||||
) -> bool:
|
||||
"""Set a terminal status exactly once.
|
||||
|
||||
@ -166,6 +188,8 @@ class SubagentResult:
|
||||
self.ai_messages = ai_messages
|
||||
if token_usage_records is not None:
|
||||
self.token_usage_records = token_usage_records
|
||||
if tool_receipts is not None:
|
||||
self.tool_receipts = [dict(receipt) for receipt in tool_receipts]
|
||||
self.admission_failure = admission_failure
|
||||
self.completed_at = completed_at or datetime.now()
|
||||
self.status = status
|
||||
@ -272,6 +296,51 @@ _background_tasks_lock = threading.Lock()
|
||||
|
||||
_background_futures: dict[str, Future[SubagentResult]] = {}
|
||||
|
||||
|
||||
def _harvest_tool_receipts(
|
||||
final_state: Any,
|
||||
*,
|
||||
prefer_citing_turn: bool = False,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
"""Harvest the child's tool receipts from its terminal message stream.
|
||||
|
||||
Lazy import: the executor package is imported in cycles with
|
||||
``deerflow.agents``; resolving ``tool_receipt`` at call time keeps module
|
||||
init order-independent. Failure-isolated: a harvest error can never
|
||||
change the run's outcome — the parent simply gets no receipts.
|
||||
"""
|
||||
if not final_state:
|
||||
return None
|
||||
messages = final_state.get("messages") if isinstance(final_state, dict) else None
|
||||
if not messages:
|
||||
return None
|
||||
try:
|
||||
from deerflow.agents.middlewares.tool_receipt import extract_citing_turn_receipts, extract_tool_receipts
|
||||
|
||||
message_list = list(messages)
|
||||
# Completed result text comes from the latest assistant turn even when
|
||||
# a max-turn chunk ends in a ToolMessage. Its bounded ledger remains
|
||||
# authoritative for citation verification. Tool-ended running,
|
||||
# cancelled, or failed evidence instead prefers the latest tool scan so
|
||||
# newly executed calls are not lost merely because no later assistant
|
||||
# turn was produced.
|
||||
citing_messages = message_list if prefer_citing_turn else [message_list[-1]]
|
||||
citing_turn_receipts = extract_citing_turn_receipts(citing_messages) if prefer_citing_turn or isinstance(message_list[-1], AIMessage) else None
|
||||
if prefer_citing_turn:
|
||||
# Missing/malformed completed-turn snapshots fail closed. Falling
|
||||
# back to the current ToolMessage scan can renumber compacted
|
||||
# receipts or reintroduce entries omitted from the model's budget.
|
||||
receipts = citing_turn_receipts
|
||||
else:
|
||||
receipts = citing_turn_receipts if citing_turn_receipts is not None else extract_tool_receipts(message_list)
|
||||
if receipts is None:
|
||||
return None
|
||||
return [dict(receipt) for receipt in receipts]
|
||||
except Exception:
|
||||
logger.warning("Failed to harvest subagent tool receipts", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
# Persistent event loop for isolated subagent executions triggered from an
|
||||
# already-running parent loop. Reusing one long-lived loop avoids creating a
|
||||
# fresh loop per execution and then closing async resources bound to it.
|
||||
@ -502,6 +571,7 @@ class SubagentExecutor:
|
||||
"""
|
||||
self.config = config
|
||||
self.app_config = app_config
|
||||
self._resolved_app_config = app_config
|
||||
self.parent_model = parent_model
|
||||
# Resolve eagerly only when it does not require loading config.yaml; otherwise defer
|
||||
# to _create_agent (which already loads app_config) so unit tests can construct
|
||||
@ -567,6 +637,12 @@ class SubagentExecutor:
|
||||
|
||||
logger.info(f"[trace={self.trace_id}] SubagentExecutor initialized: {config.name} with {len(self.tools)} tools")
|
||||
|
||||
def _get_resolved_app_config(self) -> AppConfig:
|
||||
"""Return the one AppConfig snapshot used throughout this execution."""
|
||||
if self._resolved_app_config is None:
|
||||
self._resolved_app_config = get_app_config()
|
||||
return self._resolved_app_config
|
||||
|
||||
def _create_agent(
|
||||
self,
|
||||
tools: list[BaseTool] | None = None,
|
||||
@ -580,7 +656,7 @@ class SubagentExecutor:
|
||||
deferred MCP tool names + catalog hash so the subagent gets the same
|
||||
DeferredToolFilterMiddleware the lead agent has. ``None`` is a no-op.
|
||||
"""
|
||||
app_config = self.app_config or get_app_config()
|
||||
app_config = self._get_resolved_app_config()
|
||||
if self.model_name is None:
|
||||
self.model_name = resolve_subagent_model_name(self.config, self.parent_model, app_config=app_config)
|
||||
model = create_chat_model(name=self.model_name, thinking_enabled=False, app_config=app_config, attach_tracing=False)
|
||||
@ -792,7 +868,7 @@ class SubagentExecutor:
|
||||
self._assembled_skills = list(skills)
|
||||
self._available_skill_names = {skill.name for skill in skills}
|
||||
|
||||
resolved_app_config = self.app_config or get_app_config()
|
||||
resolved_app_config = self._get_resolved_app_config()
|
||||
|
||||
from deerflow.skills.describe import build_skill_search_setup, get_skill_index_prompt_section
|
||||
|
||||
@ -989,6 +1065,14 @@ class SubagentExecutor:
|
||||
processed_message_count = 0
|
||||
|
||||
collector: SubagentTokenCollector | None = None
|
||||
final_state = None
|
||||
verification_cfg = getattr(self._get_resolved_app_config(), "verification", None)
|
||||
|
||||
def terminal_receipts(*, prefer_citing_turn: bool = False) -> list[dict[str, Any]] | None:
|
||||
if not getattr(verification_cfg, "receipts_enabled", True):
|
||||
return None
|
||||
return _harvest_tool_receipts(final_state, prefer_citing_turn=prefer_citing_turn)
|
||||
|
||||
try:
|
||||
if task_info is not None and task_store is not None:
|
||||
await notify_task_start(
|
||||
@ -998,7 +1082,11 @@ class SubagentExecutor:
|
||||
timeout=_EXTENSION_TASK_NOTIFY_TIMEOUT_SECONDS,
|
||||
)
|
||||
if result.cancel_event.is_set():
|
||||
result.try_set_terminal(SubagentStatus.CANCELLED, error="Cancelled by user")
|
||||
result.try_set_terminal(
|
||||
SubagentStatus.CANCELLED,
|
||||
error="Cancelled by user",
|
||||
tool_receipts=terminal_receipts(),
|
||||
)
|
||||
return result
|
||||
|
||||
state, final_tools, deferred_setup = await self._build_initial_state(task)
|
||||
@ -1085,7 +1173,6 @@ class SubagentExecutor:
|
||||
|
||||
# Use stream instead of invoke to get real-time updates
|
||||
# This allows us to collect AI messages as they are generated
|
||||
final_state = None
|
||||
|
||||
# Pre-check: bail out immediately if already cancelled before streaming starts
|
||||
if result.cancel_event.is_set():
|
||||
@ -1094,10 +1181,18 @@ class SubagentExecutor:
|
||||
SubagentStatus.CANCELLED,
|
||||
error="Cancelled by user",
|
||||
token_usage_records=collector.snapshot_records(),
|
||||
tool_receipts=terminal_receipts(),
|
||||
)
|
||||
return result
|
||||
|
||||
async for chunk in agent.astream(state, config=run_config, context=context, stream_mode="values"): # type: ignore[arg-type]
|
||||
# A yielded values chunk is already executed state. Retain it
|
||||
# before observing cooperative cancellation so terminal receipt
|
||||
# harvesting includes a tool result that completed while the
|
||||
# cancellation request was in flight.
|
||||
final_state = chunk
|
||||
result.update_tool_receipts(terminal_receipts())
|
||||
|
||||
# Cooperative cancellation: check if parent requested stop.
|
||||
# Note: cancellation is only detected at astream iteration boundaries,
|
||||
# so long-running tool calls within a single iteration will not be
|
||||
@ -1108,10 +1203,10 @@ class SubagentExecutor:
|
||||
SubagentStatus.CANCELLED,
|
||||
error="Cancelled by user",
|
||||
token_usage_records=collector.snapshot_records(),
|
||||
tool_receipts=terminal_receipts(),
|
||||
)
|
||||
return result
|
||||
|
||||
final_state = chunk
|
||||
result.update_token_usage_records(collector.snapshot_records())
|
||||
|
||||
# Capture every step message (assistant turns AND tool outputs)
|
||||
@ -1133,6 +1228,7 @@ class SubagentExecutor:
|
||||
SubagentStatus.FAILED,
|
||||
error=llm_error,
|
||||
token_usage_records=token_usage_records,
|
||||
tool_receipts=terminal_receipts(),
|
||||
)
|
||||
else:
|
||||
final_result = _extract_final_result(final_state, trace_id=self.trace_id, name=self.config.name)
|
||||
@ -1150,6 +1246,7 @@ class SubagentExecutor:
|
||||
result=final_result,
|
||||
stop_reason=stop_reason,
|
||||
token_usage_records=token_usage_records,
|
||||
tool_receipts=terminal_receipts(prefer_citing_turn=True),
|
||||
)
|
||||
|
||||
except GraphRecursionError:
|
||||
@ -1187,6 +1284,7 @@ class SubagentExecutor:
|
||||
error=llm_error,
|
||||
stop_reason=stop_reason,
|
||||
token_usage_records=records,
|
||||
tool_receipts=terminal_receipts(),
|
||||
)
|
||||
else:
|
||||
messages = (final_state or {}).get("messages", [])
|
||||
@ -1203,6 +1301,7 @@ class SubagentExecutor:
|
||||
result=usable_partial,
|
||||
stop_reason=stop_reason,
|
||||
token_usage_records=records,
|
||||
tool_receipts=terminal_receipts(prefer_citing_turn=True),
|
||||
)
|
||||
else:
|
||||
result.try_set_terminal(
|
||||
@ -1210,6 +1309,7 @@ class SubagentExecutor:
|
||||
error=f"Reached max_turns={max_turns}",
|
||||
stop_reason=stop_reason,
|
||||
token_usage_records=records,
|
||||
tool_receipts=terminal_receipts(),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
@ -1218,6 +1318,7 @@ class SubagentExecutor:
|
||||
SubagentStatus.FAILED,
|
||||
error=str(e),
|
||||
token_usage_records=collector.snapshot_records() if collector is not None else None,
|
||||
tool_receipts=terminal_receipts(),
|
||||
)
|
||||
|
||||
finally:
|
||||
@ -1356,11 +1457,16 @@ class SubagentExecutor:
|
||||
result.try_set_terminal(
|
||||
SubagentStatus.TIMED_OUT,
|
||||
error=f"Execution timed out after {self.config.timeout_seconds} seconds",
|
||||
tool_receipts=result.snapshot_tool_receipts(),
|
||||
)
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
result.cancel_event.set()
|
||||
result.try_set_terminal(SubagentStatus.CANCELLED, error="Cancelled by user")
|
||||
result.try_set_terminal(
|
||||
SubagentStatus.CANCELLED,
|
||||
error="Cancelled by user",
|
||||
tool_receipts=result.snapshot_tool_receipts(),
|
||||
)
|
||||
return result
|
||||
except Exception as exc:
|
||||
logger.exception("[trace=%s] Subagent %s async execution failed", self.trace_id, self.config.name)
|
||||
@ -1395,12 +1501,14 @@ def request_cancel_background_task(execution_id: str) -> None:
|
||||
"""
|
||||
with _background_tasks_lock:
|
||||
result = _background_tasks.get(execution_id)
|
||||
if result is not None:
|
||||
result.cancel_event.set()
|
||||
future = _background_futures.get(execution_id)
|
||||
if future is not None:
|
||||
future.cancel()
|
||||
logger.info("Requested cancellation for background execution %s", execution_id)
|
||||
future = _background_futures.get(execution_id) if result is not None else None
|
||||
if result is not None:
|
||||
result.cancel_event.set()
|
||||
# Future.cancel() may invoke forget_future synchronously; keep it out of
|
||||
# _background_tasks_lock because that callback acquires the same lock.
|
||||
if future is not None:
|
||||
future.cancel()
|
||||
logger.info("Requested cancellation for background execution %s", execution_id)
|
||||
|
||||
|
||||
def get_background_task_result(execution_id: str) -> SubagentResult | None:
|
||||
|
||||
@ -19,6 +19,12 @@ consumers read the structured facts carried inside
|
||||
by this delegated run.
|
||||
- ``subagent_token_usage`` (optional): final cumulative ``input_tokens`` /
|
||||
``output_tokens`` / ``total_tokens`` snapshot when the provider reported it.
|
||||
- ``subagent_tool_receipts`` (optional): the child's harvested tool receipts
|
||||
(RFC #4651 PR2), transported in full; present on terminal statuses when
|
||||
the run produced stamped receipts.
|
||||
- ``subagent_receipt_verdict`` (optional, ``completed`` only): the
|
||||
parent-side citation-check verdict — advisory execution evidence; the
|
||||
``citation_resolved`` vocabulary never claims task acceptance.
|
||||
|
||||
The shared fixture at ``contracts/subagent_status_contract.json`` pins
|
||||
the enum values across Python and TypeScript.
|
||||
@ -31,6 +37,9 @@ import re
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Literal, NotRequired, TypedDict
|
||||
|
||||
from deerflow.agents.middlewares.receipt_verification import ReceiptVerdict, validate_receipt_verdict
|
||||
from deerflow.agents.middlewares.tool_receipt import is_valid_receipt
|
||||
|
||||
SUBAGENT_STATUS_KEY = "subagent_status"
|
||||
SUBAGENT_STOP_REASON_KEY = "subagent_stop_reason"
|
||||
SUBAGENT_ERROR_KEY = "subagent_error"
|
||||
@ -38,6 +47,8 @@ SUBAGENT_RESULT_BRIEF_KEY = "subagent_result_brief"
|
||||
SUBAGENT_RESULT_SHA256_KEY = "subagent_result_sha256"
|
||||
SUBAGENT_MODEL_NAME_KEY = "subagent_model_name"
|
||||
SUBAGENT_TOKEN_USAGE_KEY = "subagent_token_usage"
|
||||
SUBAGENT_TOOL_RECEIPTS_KEY = "subagent_tool_receipts"
|
||||
SUBAGENT_RECEIPT_VERDICT_KEY = "subagent_receipt_verdict"
|
||||
SUBAGENT_METADATA_TEXT_MAX_CHARS = 2000
|
||||
|
||||
#: The producer always emits ``hashlib.sha256(...).hexdigest()`` — 64
|
||||
@ -111,6 +122,8 @@ class StructuredSubagentResult(TypedDict):
|
||||
result_brief: NotRequired[str]
|
||||
result_sha256: NotRequired[str]
|
||||
error: NotRequired[str]
|
||||
tool_receipts: NotRequired[list[dict[str, Any]]]
|
||||
receipt_verdict: NotRequired[ReceiptVerdict]
|
||||
|
||||
|
||||
def _bound_metadata_text(text: str, cap: int = SUBAGENT_METADATA_TEXT_MAX_CHARS) -> str:
|
||||
@ -135,6 +148,8 @@ def make_subagent_additional_kwargs(
|
||||
stop_reason: SubagentStopReasonValue | None = None,
|
||||
model_name: str | None = None,
|
||||
token_usage: Mapping[str, object] | None = None,
|
||||
tool_receipts: list[dict[str, Any]] | None = None,
|
||||
receipt_verdict: Mapping[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Build the ``additional_kwargs`` payload the middleware stamps.
|
||||
|
||||
@ -168,6 +183,13 @@ def make_subagent_additional_kwargs(
|
||||
normalized_usage = normalize_token_usage(token_usage)
|
||||
if normalized_usage is not None:
|
||||
payload[SUBAGENT_TOKEN_USAGE_KEY] = normalized_usage
|
||||
if isinstance(tool_receipts, list):
|
||||
cleaned_receipts = [dict(receipt) for receipt in tool_receipts if is_valid_receipt(receipt)]
|
||||
if cleaned_receipts:
|
||||
payload[SUBAGENT_TOOL_RECEIPTS_KEY] = cleaned_receipts
|
||||
validated_verdict = validate_receipt_verdict(receipt_verdict)
|
||||
if validated_verdict is not None:
|
||||
payload[SUBAGENT_RECEIPT_VERDICT_KEY] = validated_verdict
|
||||
return payload
|
||||
|
||||
|
||||
@ -283,4 +305,12 @@ def read_subagent_result_metadata(
|
||||
payload["stop_reason"] = raw_stop_reason
|
||||
elif legacy_stop_reason is not None:
|
||||
payload["stop_reason"] = legacy_stop_reason
|
||||
raw_receipts = additional_kwargs.get(SUBAGENT_TOOL_RECEIPTS_KEY)
|
||||
if isinstance(raw_receipts, list):
|
||||
cleaned_receipts = [dict(receipt) for receipt in raw_receipts if is_valid_receipt(receipt)]
|
||||
if cleaned_receipts:
|
||||
payload["tool_receipts"] = cleaned_receipts
|
||||
validated_verdict = validate_receipt_verdict(additional_kwargs.get(SUBAGENT_RECEIPT_VERDICT_KEY))
|
||||
if validated_verdict is not None:
|
||||
payload["receipt_verdict"] = validated_verdict
|
||||
return payload
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
- `update_agent` - Custom-agent-only: persist self-updates to the current agent's `SOUL.md` / `config.yaml` from inside a normal chat (partial update + atomic write). Bound when `agent_name` is set and `is_bootstrap=False`.
|
||||
4. **Subagent tool** (if enabled):
|
||||
- `task` - Delegate to subagent (description, prompt, subagent_type)
|
||||
Polling safety timeouts carry the latest published tool receipts into the terminal task metadata before requesting background cancellation.
|
||||
- `batch_task`, `batch_status`, `cancel_batch` - Explicit durable batch submission/progress/cancellation. Added only while the startup SQL-backed batch submitter is installed; large results stay in the owner-scoped API/JSONL export rather than the lead context.
|
||||
- Direct `create_deerflow_agent` integrations receive cloned tools bound to their explicit `SubagentRuntime`. The bound `task` forwards that runtime's exact execution controller and optional caller-owned `AppConfig` into registry/model/tool resolution and `SubagentExecutor`; bound batch tools use the same config snapshot and resolve only that runtime's submitter before falling back to no other application's active worker. Keep the original tool name/schema unchanged so model contracts and user-tool deduplication remain stable.
|
||||
|
||||
|
||||
@ -13,6 +13,7 @@ from langchain_core.messages import ToolMessage
|
||||
from langgraph.config import get_stream_writer
|
||||
from langgraph.types import Command
|
||||
|
||||
from deerflow.agents.middlewares.receipt_verification import verify_receipt_citations
|
||||
from deerflow.authz.principal import normalize_authz_attributes
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.extensions import resolve_run_extensions
|
||||
@ -232,6 +233,8 @@ def _task_result_command(
|
||||
stop_reason: SubagentStopReasonValue | None = None,
|
||||
model_name: str | None = None,
|
||||
usage: dict[str, int] | None = None,
|
||||
tool_receipts: list[dict] | None = None,
|
||||
receipt_verdict: dict | None = None,
|
||||
) -> Command:
|
||||
content, metadata_error = format_subagent_result_message(status, result=result, error=error, stop_reason=stop_reason)
|
||||
return Command(
|
||||
@ -248,6 +251,8 @@ def _task_result_command(
|
||||
stop_reason=stop_reason,
|
||||
model_name=model_name,
|
||||
token_usage=usage,
|
||||
tool_receipts=tool_receipts,
|
||||
receipt_verdict=receipt_verdict,
|
||||
),
|
||||
)
|
||||
]
|
||||
@ -555,6 +560,15 @@ async def task_tool(
|
||||
# stop_reason carries a guardrail cap (token_capped / turn_capped)
|
||||
# when the run was ended early but still produced a final answer
|
||||
# — the work survives on result_brief like a clean success.
|
||||
# RFC #4651 PR2: cross-check the report's [rN] citations
|
||||
# against the harvested receipts once, here — the only point
|
||||
# holding the full (untruncated) report text. receipts=None
|
||||
# means no harvest happened (receipts_enabled=false, or the
|
||||
# run ended before streaming): skip, keeping disabled
|
||||
# deployments exactly pre-PR2. An empty list is a real
|
||||
# harvest (zero stamped calls) and still gets a verdict.
|
||||
receipts = getattr(result, "tool_receipts", None)
|
||||
receipt_verdict = verify_receipt_citations(result.result or "", receipts) if receipts is not None else None
|
||||
return _task_result_command(
|
||||
tool_call_id=tool_call_id,
|
||||
status="completed",
|
||||
@ -562,6 +576,8 @@ async def task_tool(
|
||||
stop_reason=result.stop_reason,
|
||||
model_name=effective_model,
|
||||
usage=usage,
|
||||
tool_receipts=receipts,
|
||||
receipt_verdict=receipt_verdict,
|
||||
)
|
||||
elif result.status == SubagentStatus.FAILED:
|
||||
_report_subagent_usage(runtime, result)
|
||||
@ -587,6 +603,7 @@ async def task_tool(
|
||||
stop_reason=result.stop_reason,
|
||||
model_name=effective_model,
|
||||
usage=usage,
|
||||
tool_receipts=getattr(result, "tool_receipts", None),
|
||||
)
|
||||
elif result.status == SubagentStatus.CANCELLED:
|
||||
_report_subagent_usage(runtime, result)
|
||||
@ -608,6 +625,7 @@ async def task_tool(
|
||||
error=result.error,
|
||||
model_name=effective_model,
|
||||
usage=usage,
|
||||
tool_receipts=getattr(result, "tool_receipts", None),
|
||||
)
|
||||
elif result.status == SubagentStatus.TIMED_OUT:
|
||||
_report_subagent_usage(runtime, result)
|
||||
@ -629,6 +647,7 @@ async def task_tool(
|
||||
error=result.error,
|
||||
model_name=effective_model,
|
||||
usage=usage,
|
||||
tool_receipts=getattr(result, "tool_receipts", None),
|
||||
)
|
||||
|
||||
# Still running, wait before next poll
|
||||
@ -664,6 +683,7 @@ async def task_tool(
|
||||
error=message,
|
||||
model_name=effective_model,
|
||||
usage=usage,
|
||||
tool_receipts=getattr(result, "tool_receipts", None),
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
# Signal the background subagent thread to stop cooperatively.
|
||||
|
||||
@ -458,3 +458,80 @@ class TestRenderDelegationLedger:
|
||||
assert "task 10" in out
|
||||
assert "task 0" not in out
|
||||
assert "omitted from this model view" in out
|
||||
|
||||
|
||||
def _verdict(*, resolved=("r1",), failed=(), unknown=(), no_claims=False):
|
||||
return {
|
||||
"source": "receipt_citations",
|
||||
"requirement": "cited_ids_in_execution_record",
|
||||
"citation_resolved": not failed and not unknown and not no_claims,
|
||||
"cited": [*resolved, *failed, *unknown],
|
||||
"resolved": list(resolved),
|
||||
"failed": [{"id": rid, "reason": "receipt status=error"} for rid in failed],
|
||||
"unknown": list(unknown),
|
||||
"no_citation_claims": no_claims,
|
||||
}
|
||||
|
||||
|
||||
def _completed_task_message(tool_call_id: str, verdict: dict | None) -> ToolMessage:
|
||||
from deerflow.subagents.status_contract import make_subagent_additional_kwargs
|
||||
|
||||
receipts = [
|
||||
{
|
||||
"id": "r1",
|
||||
"tool_call_id": "tc-1",
|
||||
"tool_name": "write_file",
|
||||
"status": "success",
|
||||
"args_sha256": "a" * 16,
|
||||
"output_sha256": "b" * 16,
|
||||
"output_bytes": 10,
|
||||
"created_at": "2026-08-24T00:00:00+00:00",
|
||||
}
|
||||
]
|
||||
return ToolMessage(
|
||||
content="Task Succeeded. Result: done [r1]",
|
||||
tool_call_id=tool_call_id,
|
||||
name="task",
|
||||
additional_kwargs=make_subagent_additional_kwargs("completed", result="done [r1]", tool_receipts=receipts, receipt_verdict=verdict),
|
||||
)
|
||||
|
||||
|
||||
class TestReceiptVerdictRendering:
|
||||
def test_entry_carries_verdict_and_renders_counts(self):
|
||||
messages = [_ai_task_call("c1", "write report"), _completed_task_message("c1", _verdict())]
|
||||
entries = extract_delegations(messages)
|
||||
assert entries[0]["receipt_verdict"]["citation_resolved"] is True
|
||||
|
||||
rendered = render_delegation_ledger(entries)
|
||||
assert "citations: 1 resolved — execution evidence only, does not validate claim correctness" in rendered
|
||||
|
||||
def test_renders_failed_and_unknown_counts(self):
|
||||
verdict = _verdict(resolved=("r1",), failed=("r2",), unknown=("r9",))
|
||||
messages = [_ai_task_call("c1", "write report"), _completed_task_message("c1", verdict)]
|
||||
rendered = render_delegation_ledger(extract_delegations(messages))
|
||||
assert "citations: 1 resolved, 1 failed, 1 unknown" in rendered
|
||||
|
||||
def test_renders_unverified_for_uncited_action_claims(self):
|
||||
verdict = _verdict(resolved=(), no_claims=True)
|
||||
verdict["citation_resolved"] = False
|
||||
messages = [_ai_task_call("c1", "write report"), _completed_task_message("c1", verdict)]
|
||||
rendered = render_delegation_ledger(extract_delegations(messages))
|
||||
assert "citations: UNVERIFIED — action claims without receipt citations" in rendered
|
||||
|
||||
def test_legacy_messages_without_verdict_render_unchanged(self):
|
||||
messages = [_ai_task_call("c1", "write report"), _completed_task_message("c1", None)]
|
||||
entries = extract_delegations(messages)
|
||||
assert "receipt_verdict" not in entries[0]
|
||||
rendered = render_delegation_ledger(entries)
|
||||
assert "citations:" not in rendered
|
||||
|
||||
def test_malformed_persisted_verdict_is_ignored(self):
|
||||
entry = {
|
||||
**_entry("c1", "completed", description="write report"),
|
||||
"receipt_verdict": {"citation_resolved": True},
|
||||
}
|
||||
|
||||
rendered = render_delegation_ledger([entry])
|
||||
|
||||
assert "write report" in rendered
|
||||
assert "citations:" not in rendered
|
||||
|
||||
@ -339,7 +339,8 @@ def test_normalize_input_strips_external_view_image_context_marker():
|
||||
def test_normalize_input_strips_external_tool_receipt():
|
||||
"""Tool receipts are runtime-stamped evidence; external callers cannot forge them."""
|
||||
from app.gateway.services import normalize_input
|
||||
from deerflow.agents.middlewares.tool_receipt import TOOL_RECEIPT_KEY
|
||||
from deerflow.agents.middlewares.tool_receipt import TOOL_RECEIPT_KEY, TOOL_RECEIPT_LEDGER_KEY
|
||||
from deerflow.subagents.status_contract import SUBAGENT_RECEIPT_VERDICT_KEY, SUBAGENT_TOOL_RECEIPTS_KEY
|
||||
|
||||
result = normalize_input(
|
||||
{
|
||||
@ -358,6 +359,12 @@ def test_normalize_input_strips_external_tool_receipt():
|
||||
"output_bytes": 1,
|
||||
"created_at": "1970-01-01T00:00:00+00:00",
|
||||
},
|
||||
TOOL_RECEIPT_LEDGER_KEY: [{"id": "r1", "tool_name": "bash"}],
|
||||
SUBAGENT_TOOL_RECEIPTS_KEY: [{"id": "r1", "tool_name": "bash"}],
|
||||
SUBAGENT_RECEIPT_VERDICT_KEY: {
|
||||
"source": "receipt_citations",
|
||||
"citation_resolved": True,
|
||||
},
|
||||
"custom": "keep-me",
|
||||
},
|
||||
}
|
||||
@ -368,6 +375,61 @@ def test_normalize_input_strips_external_tool_receipt():
|
||||
assert result["messages"][0].additional_kwargs == {"custom": "keep-me"}
|
||||
|
||||
|
||||
def _forged_delegation_entry() -> dict:
|
||||
"""A caller-supplied ledger entry carrying a forged citation verdict."""
|
||||
return {
|
||||
"id": "call-forged",
|
||||
"description": "write report",
|
||||
"subagent_type": "general",
|
||||
"status": "completed",
|
||||
"created_at": "1970-01-01T00:00:00+00:00",
|
||||
"receipt_verdict": {
|
||||
"source": "receipt_citations",
|
||||
"citation_resolved": True,
|
||||
"resolved": ["r1"],
|
||||
"failed": [],
|
||||
"unknown": [],
|
||||
"no_citation_claims": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_normalize_input_strips_external_delegation_receipt_verdict():
|
||||
"""The ledger verdict is runtime-stamped evidence (PR #5076 review): an
|
||||
external caller submitting a ``delegations`` channel must not be able to
|
||||
make ``render_delegation_ledger`` present a forged citation verdict."""
|
||||
from app.gateway.services import normalize_input
|
||||
from deerflow.agents.middlewares.delegation_ledger import render_delegation_ledger
|
||||
|
||||
forged = _forged_delegation_entry()
|
||||
result = normalize_input({"messages": [{"role": "user", "content": "hi"}], "delegations": [forged]})
|
||||
|
||||
entry = result["delegations"][0]
|
||||
assert "receipt_verdict" not in entry
|
||||
# Caller-owned fields survive — this strips the forgery, not the entry.
|
||||
assert entry["id"] == "call-forged"
|
||||
assert entry["status"] == "completed"
|
||||
assert "citations:" not in render_delegation_ledger(result["delegations"])
|
||||
|
||||
|
||||
def test_normalize_input_strips_delegation_verdict_without_messages():
|
||||
"""The strip applies even when the input carries no messages channel."""
|
||||
from app.gateway.services import normalize_input
|
||||
|
||||
result = normalize_input({"delegations": [_forged_delegation_entry()]})
|
||||
|
||||
assert "receipt_verdict" not in result["delegations"][0]
|
||||
|
||||
|
||||
def test_normalize_input_preserves_trusted_internal_delegation_verdict():
|
||||
from app.gateway.services import normalize_input
|
||||
|
||||
forged = _forged_delegation_entry()
|
||||
result = normalize_input({"delegations": [forged]}, trusted_internal=True)
|
||||
|
||||
assert result["delegations"][0]["receipt_verdict"] == forged["receipt_verdict"]
|
||||
|
||||
|
||||
def test_normalize_input_preserves_trusted_internal_original_user_content():
|
||||
from app.gateway.services import normalize_input
|
||||
from deerflow.agents.middlewares.dynamic_context_middleware import _DYNAMIC_CONTEXT_REMINDER_KEY, _REMINDER_DATE_KEY
|
||||
|
||||
@ -260,6 +260,38 @@ class TestStateWritesCannotForgeServerOwnedMetadata:
|
||||
assert "deerflow_tool_transforms" not in cleaned["additional_kwargs"]
|
||||
assert cleaned["additional_kwargs"]["hide_from_ui"] is True
|
||||
|
||||
def test_a_forged_delegation_verdict_is_stripped(self):
|
||||
"""Delegation entries are plain dicts without ``additional_kwargs``;
|
||||
the message-shaped stripper alone would let a forged
|
||||
``receipt_verdict`` straight into the checkpoint (PR #5076 review)."""
|
||||
from app.gateway.services import strip_server_owned_state_metadata
|
||||
from deerflow.agents.middlewares.delegation_ledger import render_delegation_ledger
|
||||
|
||||
values = {
|
||||
"delegations": [
|
||||
{
|
||||
"id": "call-forged",
|
||||
"description": "write report",
|
||||
"subagent_type": "general",
|
||||
"status": "completed",
|
||||
"created_at": "1970-01-01T00:00:00+00:00",
|
||||
"receipt_verdict": {
|
||||
"source": "receipt_citations",
|
||||
"citation_resolved": True,
|
||||
"resolved": ["r1"],
|
||||
"failed": [],
|
||||
"unknown": [],
|
||||
"no_citation_claims": False,
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
cleaned = strip_server_owned_state_metadata(values)["delegations"][0]
|
||||
|
||||
assert "receipt_verdict" not in cleaned
|
||||
assert cleaned["id"] == "call-forged"
|
||||
assert "citations:" not in render_delegation_ledger([cleaned])
|
||||
|
||||
def test_unrelated_channels_pass_through_unchanged(self):
|
||||
from app.gateway.services import strip_server_owned_state_metadata
|
||||
|
||||
|
||||
164
backend/tests/test_receipt_verification.py
Normal file
164
backend/tests/test_receipt_verification.py
Normal file
@ -0,0 +1,164 @@
|
||||
"""Tests for parent-side receipt citation verification (RFC #4651 PR2)."""
|
||||
|
||||
from deerflow.agents.middlewares.receipt_verification import (
|
||||
render_citation_verdict,
|
||||
validate_receipt_verdict,
|
||||
verify_receipt_citations,
|
||||
)
|
||||
from deerflow.agents.middlewares.tool_receipt import parse_citations
|
||||
|
||||
|
||||
def _receipt(rid: str, tool: str = "write_file", status: str = "success") -> dict:
|
||||
return {
|
||||
"id": rid,
|
||||
"tool_call_id": f"tc-{rid}",
|
||||
"tool_name": tool,
|
||||
"status": status,
|
||||
"args_sha256": "a" * 16,
|
||||
"output_sha256": "b" * 16,
|
||||
"output_bytes": 10,
|
||||
"created_at": "2026-08-24T00:00:00+00:00",
|
||||
}
|
||||
|
||||
|
||||
LEDGER = [_receipt("r1", "web_search"), _receipt("r2", "write_file"), _receipt("r3", "bash", status="error")]
|
||||
|
||||
|
||||
class TestCitationExtraction:
|
||||
def test_bare_and_anchored_citations(self):
|
||||
verdict = verify_receipt_citations("wrote file [r2 write_file], searched [r1]", LEDGER)
|
||||
assert verdict["cited"] == ["r2", "r1"]
|
||||
assert verdict["resolved"] == ["r2", "r1"]
|
||||
assert verdict["citation_resolved"] is True
|
||||
|
||||
def test_duplicate_citations_deduped_first_seen(self):
|
||||
verdict = verify_receipt_citations("[r1] then [r2] then [r1] again", LEDGER)
|
||||
assert verdict["cited"] == ["r1", "r2"]
|
||||
|
||||
def test_same_receipt_with_different_anchors_checks_every_pair(self):
|
||||
verdict = verify_receipt_citations("[r2 write_file] then [r2 bash]", LEDGER)
|
||||
|
||||
assert verdict["cited"] == ["r2", "r2"]
|
||||
assert verdict["resolved"] == ["r2"]
|
||||
assert verdict["failed"] == [
|
||||
{
|
||||
"id": "r2",
|
||||
"reason": "anchor mismatch: cited as bash, receipt r2 is write_file",
|
||||
}
|
||||
]
|
||||
assert verdict["citation_resolved"] is False
|
||||
|
||||
def test_non_citation_brackets_ignored(self):
|
||||
verdict = verify_receipt_citations("see [1] and [rX] and [note]", LEDGER)
|
||||
assert verdict["cited"] == []
|
||||
|
||||
def test_parse_accepts_dotted_tool_names(self):
|
||||
assert parse_citations("[r4 mcp.server-tool]") == [("r4", "mcp.server-tool")]
|
||||
|
||||
|
||||
class TestClassification:
|
||||
def test_error_status_receipt_is_failed(self):
|
||||
verdict = verify_receipt_citations("tests passed [r3]", LEDGER)
|
||||
assert verdict["resolved"] == []
|
||||
assert verdict["failed"] == [{"id": "r3", "reason": "receipt status=error"}]
|
||||
assert verdict["citation_resolved"] is False
|
||||
|
||||
def test_anchor_mismatch_is_failed(self):
|
||||
verdict = verify_receipt_citations("saved the report [r1 write_file]", LEDGER)
|
||||
assert verdict["failed"] == [{"id": "r1", "reason": "anchor mismatch: cited as write_file, receipt r1 is web_search"}]
|
||||
assert verdict["citation_resolved"] is False
|
||||
|
||||
def test_anchor_match_resolves(self):
|
||||
verdict = verify_receipt_citations("saved the report [r2 write_file]", LEDGER)
|
||||
assert verdict["resolved"] == ["r2"]
|
||||
assert verdict["citation_resolved"] is True
|
||||
|
||||
def test_unknown_id(self):
|
||||
verdict = verify_receipt_citations("uploaded results [r9]", LEDGER)
|
||||
assert verdict["unknown"] == ["r9"]
|
||||
assert verdict["citation_resolved"] is False
|
||||
|
||||
|
||||
class TestZeroCitationHeuristic:
|
||||
def test_action_verb_without_citation_flagged(self):
|
||||
verdict = verify_receipt_citations("I wrote the analysis and ran the tests.", LEDGER)
|
||||
assert verdict["cited"] == []
|
||||
assert verdict["no_citation_claims"] is True
|
||||
assert verdict["citation_resolved"] is False
|
||||
|
||||
def test_file_path_without_citation_flagged(self):
|
||||
verdict = verify_receipt_citations("Done, see /outputs/report.md for details", LEDGER)
|
||||
assert verdict["no_citation_claims"] is True
|
||||
|
||||
def test_benign_report_is_vacuous_pass(self):
|
||||
verdict = verify_receipt_citations("The answer is 42.", LEDGER)
|
||||
assert verdict["no_citation_claims"] is False
|
||||
assert verdict["citation_resolved"] is True
|
||||
|
||||
def test_common_completion_verbs_flagged(self):
|
||||
"""The original list missed the most common completion phrasings."""
|
||||
for text in ["I fixed the bug and added tests.", "I tested the change successfully."]:
|
||||
verdict = verify_receipt_citations(text, LEDGER)
|
||||
assert verdict["no_citation_claims"] is True, text
|
||||
assert verdict["citation_resolved"] is False, text
|
||||
|
||||
def test_chinese_action_claims_flagged(self):
|
||||
"""CJK reports have no word boundaries; common action verbs must match."""
|
||||
verdict = verify_receipt_citations("我已经创建了文件并运行了测试。", LEDGER)
|
||||
assert verdict["no_citation_claims"] is True
|
||||
assert verdict["citation_resolved"] is False
|
||||
|
||||
def test_nontrivial_uncited_report_with_receipts_flagged(self):
|
||||
"""Safety net in any language: the run demonstrably executed tools
|
||||
(receipts harvested), yet a paragraph-length report cites none of them."""
|
||||
report = "The analysis is complete. " * 12 # 300 chars, no verbs, no paths
|
||||
verdict = verify_receipt_citations(report, LEDGER)
|
||||
assert verdict["cited"] == []
|
||||
assert verdict["no_citation_claims"] is True
|
||||
|
||||
def test_short_claim_free_report_with_receipts_stays_vacuous_pass(self):
|
||||
verdict = verify_receipt_citations("The answer is 42.", LEDGER)
|
||||
assert verdict["no_citation_claims"] is False
|
||||
assert verdict["citation_resolved"] is True
|
||||
|
||||
def test_claim_free_report_without_receipts_stays_vacuous_pass(self):
|
||||
"""No tools were stamped: nothing exists to cite against."""
|
||||
report = "The analysis is complete. " * 12
|
||||
verdict = verify_receipt_citations(report, [])
|
||||
assert verdict["no_citation_claims"] is False
|
||||
assert verdict["citation_resolved"] is True
|
||||
|
||||
def test_verdict_shape_vocabulary(self):
|
||||
verdict = verify_receipt_citations("x [r1]", LEDGER)
|
||||
assert verdict["source"] == "receipt_citations"
|
||||
assert verdict["requirement"] == "cited_ids_in_execution_record"
|
||||
assert "satisfied" not in verdict
|
||||
|
||||
|
||||
class TestRender:
|
||||
def test_render_counts_with_limitation_line(self):
|
||||
verdict = verify_receipt_citations("[r1] [r3] [r9]", LEDGER)
|
||||
rendered = render_citation_verdict(verdict)
|
||||
assert rendered == ("citations: 1 resolved, 1 failed, 1 unknown — execution evidence only, does not validate claim correctness")
|
||||
|
||||
def test_render_unverified_for_no_citation_claims(self):
|
||||
verdict = verify_receipt_citations("I wrote the file.", LEDGER)
|
||||
assert render_citation_verdict(verdict) == ("citations: UNVERIFIED — action claims without receipt citations")
|
||||
|
||||
def test_render_empty_for_vacuous_pass(self):
|
||||
verdict = verify_receipt_citations("The answer is 42.", LEDGER)
|
||||
assert render_citation_verdict(verdict) == ""
|
||||
|
||||
|
||||
class TestValidate:
|
||||
def test_round_trip(self):
|
||||
verdict = verify_receipt_citations("[r1] [r9]", LEDGER)
|
||||
assert validate_receipt_verdict(dict(verdict)) == verdict
|
||||
|
||||
def test_rejects_malformed(self):
|
||||
assert validate_receipt_verdict(None) is None
|
||||
assert validate_receipt_verdict({"source": 1}) is None
|
||||
assert validate_receipt_verdict({"citation_resolved": "yes"}) is None
|
||||
bad = dict(verify_receipt_citations("[r3]", LEDGER))
|
||||
bad["failed"] = [{"id": "r3"}] # missing reason
|
||||
assert validate_receipt_verdict(bad) is None
|
||||
@ -23,7 +23,7 @@ from datetime import datetime
|
||||
from importlib.metadata import version as package_version
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from packaging.version import Version
|
||||
@ -472,11 +472,7 @@ class TestAgentConstruction:
|
||||
lambda user_id, *, app_config=None: SimpleNamespace(load_skills=lambda *, enabled_only: [SimpleNamespace(name="my-skill", skill_file=skill_file, allowed_tools=None)]),
|
||||
)
|
||||
|
||||
executor = SubagentExecutor(
|
||||
config=base_config,
|
||||
tools=[],
|
||||
thread_id="test-thread",
|
||||
)
|
||||
executor = SubagentExecutor(config=base_config, tools=[], thread_id="test-thread")
|
||||
|
||||
state, _final_tools, _deferred_setup = await executor._build_initial_state("Do the task")
|
||||
|
||||
@ -2036,6 +2032,7 @@ class TestThreadSafety:
|
||||
"total_tokens": 15,
|
||||
}
|
||||
]
|
||||
tool_receipts = [{"id": "r1", "tool_call_id": "tc-1"}]
|
||||
|
||||
def set_terminal():
|
||||
try:
|
||||
@ -2043,6 +2040,7 @@ class TestThreadSafety:
|
||||
SubagentStatus.COMPLETED,
|
||||
result="done",
|
||||
token_usage_records=token_usage_records,
|
||||
tool_receipts=tool_receipts,
|
||||
)
|
||||
except BaseException as exc:
|
||||
writer_errors.append(exc)
|
||||
@ -2054,6 +2052,7 @@ class TestThreadSafety:
|
||||
assert result.completed_at is None
|
||||
assert result.status == SubagentStatus.RUNNING
|
||||
assert result.token_usage_records == token_usage_records
|
||||
assert result.tool_receipts == tool_receipts
|
||||
|
||||
release_now.set()
|
||||
writer.join(timeout=3)
|
||||
@ -2064,6 +2063,7 @@ class TestThreadSafety:
|
||||
assert result.status == SubagentStatus.COMPLETED
|
||||
assert result.result == "done"
|
||||
assert result.token_usage_records == token_usage_records
|
||||
assert result.tool_receipts == tool_receipts
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
@ -3466,3 +3466,462 @@ class TestSubagentGuardrailAttribution:
|
||||
tools=[],
|
||||
authz_attributes=["not", "a", "mapping"],
|
||||
)
|
||||
|
||||
|
||||
class TestToolReceiptHarvest:
|
||||
"""RFC #4651 PR2: the executor harvests the child's receipts at terminal."""
|
||||
|
||||
class _ImmediateSlot:
|
||||
async def __aenter__(self):
|
||||
return None
|
||||
|
||||
async def __aexit__(self, exc_type, exc, traceback):
|
||||
return False
|
||||
|
||||
class _ImmediateCapacity:
|
||||
def slot(self):
|
||||
return TestToolReceiptHarvest._ImmediateSlot()
|
||||
|
||||
class _ThreadSubmitter:
|
||||
"""Run submitted coroutines on test-owned threads, not global loops."""
|
||||
|
||||
def __init__(self):
|
||||
self.threads = []
|
||||
|
||||
def submit(self, context, coroutine_factory):
|
||||
from concurrent.futures import Future
|
||||
|
||||
future = Future()
|
||||
|
||||
def run():
|
||||
try:
|
||||
value = context.run(lambda: asyncio.run(coroutine_factory()))
|
||||
except BaseException as exc:
|
||||
if not future.cancelled():
|
||||
future.set_exception(exc)
|
||||
else:
|
||||
if not future.cancelled():
|
||||
future.set_result(value)
|
||||
|
||||
thread = threading.Thread(target=run, daemon=True)
|
||||
self.threads.append(thread)
|
||||
thread.start()
|
||||
return future
|
||||
|
||||
def close(self):
|
||||
for thread in self.threads:
|
||||
thread.join(timeout=3)
|
||||
assert not thread.is_alive()
|
||||
|
||||
def test_harvest_uses_current_scan_when_latest_chunk_ends_in_tool_result(self, classes, msg, monkeypatch):
|
||||
executor_module = importlib.import_module("deerflow.subagents.executor")
|
||||
latest = [{"id": "r1", "tool_call_id": "tc-latest"}]
|
||||
fake_tool_receipt = _module(
|
||||
"deerflow.agents.middlewares.tool_receipt",
|
||||
extract_citing_turn_receipts=lambda messages: [],
|
||||
extract_tool_receipts=lambda messages: latest,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "deerflow.agents.middlewares.tool_receipt", fake_tool_receipt)
|
||||
state = {
|
||||
"messages": [
|
||||
msg.ai("Earlier report", "msg-1"),
|
||||
msg.tool("latest output", "tc-latest", name="write_file"),
|
||||
]
|
||||
}
|
||||
|
||||
assert executor_module._harvest_tool_receipts(state) == latest
|
||||
|
||||
def test_completed_tool_ended_partial_prefers_bounded_citing_snapshot(self, classes, msg, monkeypatch):
|
||||
executor_module = importlib.import_module("deerflow.subagents.executor")
|
||||
bounded = [{"id": "r24", "tool_call_id": "tc-cited"}]
|
||||
latest = [
|
||||
{"id": "r1", "tool_call_id": "tc-omitted"},
|
||||
{"id": "r31", "tool_call_id": "tc-latest"},
|
||||
]
|
||||
fake_tool_receipt = _module(
|
||||
"deerflow.agents.middlewares.tool_receipt",
|
||||
extract_citing_turn_receipts=lambda messages: bounded,
|
||||
extract_tool_receipts=lambda messages: latest,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "deerflow.agents.middlewares.tool_receipt", fake_tool_receipt)
|
||||
state = {
|
||||
"messages": [
|
||||
msg.ai("Partial report [r24]", "msg-1"),
|
||||
msg.tool("latest output", "tc-latest", name="write_file"),
|
||||
]
|
||||
}
|
||||
|
||||
assert executor_module._harvest_tool_receipts(state) == latest
|
||||
assert executor_module._harvest_tool_receipts(state, prefer_citing_turn=True) == bounded
|
||||
|
||||
def test_completed_result_does_not_fallback_when_citing_snapshot_is_invalid(self, classes, msg, monkeypatch):
|
||||
executor_module = importlib.import_module("deerflow.subagents.executor")
|
||||
current = [{"id": "r1", "tool_call_id": "tc-renumbered"}]
|
||||
fake_tool_receipt = _module(
|
||||
"deerflow.agents.middlewares.tool_receipt",
|
||||
extract_citing_turn_receipts=lambda messages: None,
|
||||
extract_tool_receipts=lambda messages: current,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "deerflow.agents.middlewares.tool_receipt", fake_tool_receipt)
|
||||
state = {"messages": [msg.ai("Completed report [r1]", "msg-1")]}
|
||||
|
||||
assert executor_module._harvest_tool_receipts(state) == current
|
||||
assert executor_module._harvest_tool_receipts(state, prefer_citing_turn=True) is None
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_recursion_capped_tool_ended_partial_persists_bounded_citing_snapshot(
|
||||
self,
|
||||
classes,
|
||||
base_config,
|
||||
msg,
|
||||
monkeypatch,
|
||||
):
|
||||
from langgraph.errors import GraphRecursionError
|
||||
|
||||
SubagentExecutor = classes["SubagentExecutor"]
|
||||
SubagentStatus = classes["SubagentStatus"]
|
||||
bounded = [{"id": "r24", "tool_call_id": "tc-cited"}]
|
||||
latest = [{"id": "r1", "tool_call_id": "tc-omitted"}]
|
||||
fake_tool_receipt = _module(
|
||||
"deerflow.agents.middlewares.tool_receipt",
|
||||
extract_citing_turn_receipts=lambda messages: bounded,
|
||||
extract_tool_receipts=lambda messages: latest,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "deerflow.agents.middlewares.tool_receipt", fake_tool_receipt)
|
||||
final_state = {
|
||||
"messages": [
|
||||
msg.ai("Partial report [r24]", "msg-1"),
|
||||
msg.tool("latest output", "tc-latest", name="write_file"),
|
||||
]
|
||||
}
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield final_state
|
||||
raise GraphRecursionError("turn limit")
|
||||
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.astream = mock_astream
|
||||
executor = SubagentExecutor(
|
||||
config=base_config,
|
||||
tools=[],
|
||||
thread_id="test-thread",
|
||||
)
|
||||
with (
|
||||
patch.object(executor, "_build_initial_state", new=AsyncMock(return_value=({}, [], None))),
|
||||
patch.object(executor, "_create_agent", return_value=mock_agent),
|
||||
):
|
||||
result = await executor._aexecute_admitted("Do something")
|
||||
|
||||
assert result.status == SubagentStatus.COMPLETED
|
||||
assert result.result == "Partial report [r24]"
|
||||
assert result.tool_receipts == bounded
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_aexecute_harvests_tool_receipts_on_completion(self, classes, base_config, mock_agent, msg, monkeypatch):
|
||||
SubagentExecutor = classes["SubagentExecutor"]
|
||||
SubagentStatus = classes["SubagentStatus"]
|
||||
|
||||
harvested = [
|
||||
{
|
||||
"id": "r1",
|
||||
"tool_call_id": "tc-1",
|
||||
"tool_name": "write_file",
|
||||
"status": "success",
|
||||
"args_sha256": "a" * 16,
|
||||
"output_sha256": "b" * 16,
|
||||
"output_bytes": 3,
|
||||
"created_at": "2026-08-24T00:00:00+00:00",
|
||||
}
|
||||
]
|
||||
fake_tool_receipt = _module(
|
||||
"deerflow.agents.middlewares.tool_receipt",
|
||||
extract_citing_turn_receipts=lambda messages: harvested,
|
||||
extract_tool_receipts=lambda messages: harvested,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "deerflow.agents.middlewares.tool_receipt", fake_tool_receipt)
|
||||
|
||||
final_state = {
|
||||
"messages": [
|
||||
msg.human("Do something"),
|
||||
msg.tool("out", "tc-1", name="write_file"),
|
||||
msg.ai("Done [r1]", "msg-1"),
|
||||
]
|
||||
}
|
||||
mock_agent.astream = lambda *args, **kwargs: async_iterator([final_state])
|
||||
executor = SubagentExecutor(config=base_config, tools=[], thread_id="test-thread")
|
||||
with (
|
||||
patch.object(executor, "_build_initial_state", new=AsyncMock(return_value=({}, [], None))),
|
||||
patch.object(executor, "_create_agent", return_value=mock_agent),
|
||||
):
|
||||
result = await executor._aexecute_admitted("Do something")
|
||||
|
||||
assert result.status == SubagentStatus.COMPLETED
|
||||
assert result.tool_receipts == harvested
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_aexecute_cancelled_before_stream_has_no_receipts(self, classes, base_config, mock_agent, msg):
|
||||
SubagentExecutor = classes["SubagentExecutor"]
|
||||
SubagentStatus = classes["SubagentStatus"]
|
||||
|
||||
holder = classes["SubagentResult"](
|
||||
task_id="t1",
|
||||
trace_id="tr",
|
||||
status=SubagentStatus.RUNNING,
|
||||
started_at=datetime.now(),
|
||||
)
|
||||
holder.cancel_event.set()
|
||||
executor = SubagentExecutor(config=base_config, tools=[], thread_id="test-thread")
|
||||
with patch.object(executor, "_create_agent", return_value=mock_agent):
|
||||
result = await executor._aexecute_admitted("Do something", result_holder=holder)
|
||||
|
||||
assert result.status == SubagentStatus.CANCELLED
|
||||
assert result.tool_receipts is None
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_aexecute_harvests_receipt_from_chunk_yielded_after_cancellation(self, classes, base_config, msg, monkeypatch):
|
||||
SubagentExecutor = classes["SubagentExecutor"]
|
||||
SubagentStatus = classes["SubagentStatus"]
|
||||
harvested = [
|
||||
{
|
||||
"id": "r1",
|
||||
"tool_call_id": "tc-1",
|
||||
"tool_name": "write_file",
|
||||
"status": "success",
|
||||
"args_sha256": "a" * 16,
|
||||
"output_sha256": "b" * 16,
|
||||
"output_bytes": 3,
|
||||
"created_at": "2026-08-24T00:00:00+00:00",
|
||||
}
|
||||
]
|
||||
fake_tool_receipt = _module(
|
||||
"deerflow.agents.middlewares.tool_receipt",
|
||||
extract_citing_turn_receipts=lambda messages: harvested if messages[-1].id == "msg-2" else None,
|
||||
extract_tool_receipts=lambda messages: harvested,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "deerflow.agents.middlewares.tool_receipt", fake_tool_receipt)
|
||||
|
||||
holder = classes["SubagentResult"](
|
||||
task_id="cancel-after-tool",
|
||||
trace_id="tr",
|
||||
status=SubagentStatus.RUNNING,
|
||||
started_at=datetime.now(),
|
||||
)
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield {"messages": [msg.human("Task"), msg.ai("Working", "msg-1")]}
|
||||
holder.cancel_event.set()
|
||||
yield {"messages": [msg.human("Task"), msg.tool("out", "tc-1", name="write_file"), msg.ai("Done [r1]", "msg-2")]}
|
||||
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.astream = mock_astream
|
||||
executor = SubagentExecutor(config=base_config, tools=[], thread_id="test-thread")
|
||||
with (
|
||||
patch.object(executor, "_build_initial_state", new=AsyncMock(return_value=({}, [], None))),
|
||||
patch.object(executor, "_create_agent", return_value=mock_agent),
|
||||
):
|
||||
result = await executor._aexecute_admitted("Do something", result_holder=holder)
|
||||
|
||||
assert result.status == SubagentStatus.CANCELLED
|
||||
assert result.tool_receipts == harvested
|
||||
|
||||
def test_execute_async_preserves_published_receipts_on_forced_cancellation(self, classes, base_config, msg, monkeypatch):
|
||||
executor_module = importlib.import_module("deerflow.subagents.executor")
|
||||
SubagentExecutor = classes["SubagentExecutor"]
|
||||
SubagentStatus = classes["SubagentStatus"]
|
||||
harvested = [{"id": "r1", "tool_call_id": "tc-1"}]
|
||||
fake_tool_receipt = _module(
|
||||
"deerflow.agents.middlewares.tool_receipt",
|
||||
extract_citing_turn_receipts=lambda messages: harvested,
|
||||
extract_tool_receipts=lambda messages: harvested,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "deerflow.agents.middlewares.tool_receipt", fake_tool_receipt)
|
||||
chunk_seen = threading.Event()
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield {"messages": [msg.human("Task"), msg.tool("out", "tc-1", name="write_file")]}
|
||||
chunk_seen.set()
|
||||
while True:
|
||||
await asyncio.sleep(0.001)
|
||||
yield {"messages": [msg.human("Task"), msg.tool("out", "tc-1", name="write_file")]}
|
||||
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.astream = mock_astream
|
||||
executor = SubagentExecutor(
|
||||
config=base_config,
|
||||
tools=[],
|
||||
thread_id="test-thread",
|
||||
extensions=SimpleNamespace(needs_task_store=False, has_task_lifecycle=False),
|
||||
execution_capacity=self._ImmediateCapacity(),
|
||||
)
|
||||
submitter = self._ThreadSubmitter()
|
||||
try:
|
||||
with (
|
||||
patch.object(executor_module, "_submit_to_isolated_loop_in_context", side_effect=submitter.submit),
|
||||
patch.object(executor_module, "build_tracing_callbacks", return_value=[]),
|
||||
patch.object(executor_module, "inject_langfuse_metadata"),
|
||||
patch.object(executor, "_build_initial_state", new=AsyncMock(return_value=({}, [], None))),
|
||||
patch.object(executor, "_create_agent", return_value=mock_agent),
|
||||
):
|
||||
task_id = executor.execute_async("Do something")
|
||||
assert chunk_seen.wait(timeout=3)
|
||||
executor_module.request_cancel_background_task(task_id)
|
||||
deadline = time.monotonic() + 3
|
||||
while not executor_module._background_tasks[task_id].status.is_terminal and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
finally:
|
||||
submitter.close()
|
||||
|
||||
result = executor_module._background_tasks[task_id]
|
||||
assert result.status == SubagentStatus.CANCELLED
|
||||
assert result.tool_receipts == harvested
|
||||
|
||||
def test_execute_async_preserves_published_receipts_on_timeout(self, classes, msg, monkeypatch):
|
||||
executor_module = importlib.import_module("deerflow.subagents.executor")
|
||||
SubagentExecutor = classes["SubagentExecutor"]
|
||||
SubagentStatus = classes["SubagentStatus"]
|
||||
short_config = classes["SubagentConfig"](
|
||||
name="test-agent",
|
||||
description="Test agent",
|
||||
system_prompt="You are a test agent.",
|
||||
max_turns=10,
|
||||
timeout_seconds=0.05,
|
||||
)
|
||||
harvested = [{"id": "r1", "tool_call_id": "tc-1"}]
|
||||
fake_tool_receipt = _module(
|
||||
"deerflow.agents.middlewares.tool_receipt",
|
||||
extract_citing_turn_receipts=lambda messages: harvested,
|
||||
extract_tool_receipts=lambda messages: harvested,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "deerflow.agents.middlewares.tool_receipt", fake_tool_receipt)
|
||||
chunk_seen = threading.Event()
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield {"messages": [msg.human("Task"), msg.tool("out", "tc-1", name="write_file")]}
|
||||
chunk_seen.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.astream = mock_astream
|
||||
executor = SubagentExecutor(
|
||||
config=short_config,
|
||||
tools=[],
|
||||
thread_id="test-thread",
|
||||
extensions=SimpleNamespace(needs_task_store=False, has_task_lifecycle=False),
|
||||
execution_capacity=self._ImmediateCapacity(),
|
||||
)
|
||||
submitter = self._ThreadSubmitter()
|
||||
try:
|
||||
with (
|
||||
patch.object(executor_module, "_submit_to_isolated_loop_in_context", side_effect=submitter.submit),
|
||||
patch.object(executor_module, "build_tracing_callbacks", return_value=[]),
|
||||
patch.object(executor_module, "inject_langfuse_metadata"),
|
||||
patch.object(executor, "_build_initial_state", new=AsyncMock(return_value=({}, [], None))),
|
||||
patch.object(executor, "_create_agent", return_value=mock_agent),
|
||||
):
|
||||
task_id = executor.execute_async("Do something")
|
||||
assert chunk_seen.wait(timeout=3)
|
||||
deadline = time.monotonic() + 3
|
||||
while task_id in executor_module._background_futures and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
finally:
|
||||
submitter.close()
|
||||
|
||||
result = executor_module._background_tasks[task_id]
|
||||
assert result.status == SubagentStatus.TIMED_OUT
|
||||
assert result.tool_receipts == harvested
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_harvest_skipped_when_receipts_disabled(self, classes, base_config, mock_agent, msg, monkeypatch):
|
||||
SubagentExecutor = classes["SubagentExecutor"]
|
||||
SubagentStatus = classes["SubagentStatus"]
|
||||
|
||||
harvested = [
|
||||
{
|
||||
"id": "r1",
|
||||
"tool_call_id": "tc-1",
|
||||
"tool_name": "write_file",
|
||||
"status": "success",
|
||||
"args_sha256": "a",
|
||||
"output_sha256": "b",
|
||||
"output_bytes": 3,
|
||||
"created_at": "t",
|
||||
}
|
||||
]
|
||||
fake_tool_receipt = _module(
|
||||
"deerflow.agents.middlewares.tool_receipt",
|
||||
extract_citing_turn_receipts=lambda messages: harvested,
|
||||
extract_tool_receipts=lambda messages: harvested,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "deerflow.agents.middlewares.tool_receipt", fake_tool_receipt)
|
||||
|
||||
app_config = _default_app_config()
|
||||
app_config.models = [SimpleNamespace(name="default-model")]
|
||||
app_config.verification = SimpleNamespace(receipts_enabled=False)
|
||||
final_state = {"messages": [msg.human("Do something"), msg.ai("Done", "msg-1")]}
|
||||
mock_agent.astream = lambda *args, **kwargs: async_iterator([final_state])
|
||||
executor = SubagentExecutor(config=base_config, tools=[], thread_id="test-thread", app_config=app_config)
|
||||
with (
|
||||
patch.object(executor, "_build_initial_state", new=AsyncMock(return_value=({}, [], None))),
|
||||
patch.object(executor, "_create_agent", return_value=mock_agent),
|
||||
):
|
||||
result = await executor._aexecute_admitted("Do something")
|
||||
|
||||
assert result.status == SubagentStatus.COMPLETED
|
||||
assert result.tool_receipts is None
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_harvest_honors_globally_resolved_disabled_receipts(self, classes, base_config, mock_agent, msg, monkeypatch):
|
||||
executor_module = importlib.import_module("deerflow.subagents.executor")
|
||||
SubagentExecutor = classes["SubagentExecutor"]
|
||||
SubagentStatus = classes["SubagentStatus"]
|
||||
harvested = [{"id": "r1", "tool_call_id": "tc-1"}]
|
||||
fake_tool_receipt = _module(
|
||||
"deerflow.agents.middlewares.tool_receipt",
|
||||
extract_citing_turn_receipts=lambda messages: harvested,
|
||||
extract_tool_receipts=lambda messages: harvested,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "deerflow.agents.middlewares.tool_receipt", fake_tool_receipt)
|
||||
resolved_config = _default_app_config()
|
||||
resolved_config.verification = SimpleNamespace(receipts_enabled=False)
|
||||
monkeypatch.setattr(executor_module, "get_app_config", lambda: resolved_config)
|
||||
|
||||
final_state = {"messages": [msg.human("Do something"), msg.ai("Done", "msg-1")]}
|
||||
mock_agent.astream = lambda *args, **kwargs: async_iterator([final_state])
|
||||
executor = SubagentExecutor(config=base_config, tools=[], thread_id="test-thread")
|
||||
with (
|
||||
patch.object(executor, "_build_initial_state", new=AsyncMock(return_value=({}, [], None))),
|
||||
patch.object(executor, "_create_agent", return_value=mock_agent),
|
||||
):
|
||||
result = await executor._aexecute_admitted("Do something")
|
||||
|
||||
assert executor.app_config is None
|
||||
assert executor._get_resolved_app_config() is resolved_config
|
||||
assert result.status == SubagentStatus.COMPLETED
|
||||
assert result.tool_receipts is None
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_harvest_failure_never_breaks_execution(self, classes, base_config, mock_agent, msg, monkeypatch):
|
||||
SubagentExecutor = classes["SubagentExecutor"]
|
||||
SubagentStatus = classes["SubagentStatus"]
|
||||
|
||||
def _explode(messages):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
fake_tool_receipt = _module(
|
||||
"deerflow.agents.middlewares.tool_receipt",
|
||||
extract_citing_turn_receipts=_explode,
|
||||
extract_tool_receipts=_explode,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "deerflow.agents.middlewares.tool_receipt", fake_tool_receipt)
|
||||
|
||||
final_state = {"messages": [msg.human("Do something"), msg.ai("Done", "msg-1")]}
|
||||
mock_agent.astream = lambda *args, **kwargs: async_iterator([final_state])
|
||||
executor = SubagentExecutor(config=base_config, tools=[], thread_id="test-thread")
|
||||
with (
|
||||
patch.object(executor, "_build_initial_state", new=AsyncMock(return_value=({}, [], None))),
|
||||
patch.object(executor, "_create_agent", return_value=mock_agent),
|
||||
):
|
||||
result = await executor._aexecute_admitted("Do something")
|
||||
|
||||
assert result.status == SubagentStatus.COMPLETED
|
||||
assert result.tool_receipts is None
|
||||
|
||||
@ -240,3 +240,77 @@ def test_make_subagent_additional_kwargs_rejects_unknown_stop_reason():
|
||||
|
||||
with pytest.raises(ValueError, match="invalid subagent stop_reason"):
|
||||
make_subagent_additional_kwargs("completed", stop_reason="garbage") # type: ignore[arg-type]
|
||||
|
||||
|
||||
class TestToolReceiptTransport:
|
||||
"""RFC #4651 PR2: receipts + verdict ride additional_kwargs additively."""
|
||||
|
||||
def _receipt(self, rid: str = "r1") -> dict:
|
||||
return {
|
||||
"id": rid,
|
||||
"tool_call_id": f"tc-{rid}",
|
||||
"tool_name": "write_file",
|
||||
"status": "success",
|
||||
"args_sha256": "a" * 16,
|
||||
"output_sha256": "b" * 16,
|
||||
"output_bytes": 10,
|
||||
"created_at": "2026-08-24T00:00:00+00:00",
|
||||
}
|
||||
|
||||
def _verdict(self) -> dict:
|
||||
return {
|
||||
"source": "receipt_citations",
|
||||
"requirement": "cited_ids_in_execution_record",
|
||||
"citation_resolved": True,
|
||||
"cited": ["r1"],
|
||||
"resolved": ["r1"],
|
||||
"failed": [],
|
||||
"unknown": [],
|
||||
"no_citation_claims": False,
|
||||
}
|
||||
|
||||
def test_round_trip_receipts_and_verdict(self):
|
||||
kwargs = make_subagent_additional_kwargs(
|
||||
"completed",
|
||||
result="done [r1]",
|
||||
tool_receipts=[self._receipt()],
|
||||
receipt_verdict=self._verdict(),
|
||||
)
|
||||
assert kwargs["subagent_tool_receipts"] == [self._receipt()]
|
||||
assert kwargs["subagent_receipt_verdict"] == self._verdict()
|
||||
|
||||
structured = read_subagent_result_metadata(kwargs)
|
||||
assert structured is not None
|
||||
assert structured["tool_receipts"] == [self._receipt()]
|
||||
assert structured["receipt_verdict"] == self._verdict()
|
||||
|
||||
def test_malformed_receipts_dropped(self):
|
||||
kwargs = make_subagent_additional_kwargs(
|
||||
"completed",
|
||||
result="done",
|
||||
tool_receipts=[self._receipt(), {"broken": True}],
|
||||
)
|
||||
assert kwargs["subagent_tool_receipts"] == [self._receipt()]
|
||||
|
||||
def test_all_malformed_receipts_omit_key(self):
|
||||
kwargs = make_subagent_additional_kwargs(
|
||||
"completed",
|
||||
result="done",
|
||||
tool_receipts=[{"broken": True}],
|
||||
)
|
||||
assert "subagent_tool_receipts" not in kwargs
|
||||
|
||||
def test_malformed_verdict_dropped(self):
|
||||
kwargs = make_subagent_additional_kwargs(
|
||||
"completed",
|
||||
result="done",
|
||||
receipt_verdict={"citation_resolved": "yes"},
|
||||
)
|
||||
assert "subagent_receipt_verdict" not in kwargs
|
||||
|
||||
def test_old_payloads_read_clean(self):
|
||||
kwargs = make_subagent_additional_kwargs("completed", result="done")
|
||||
structured = read_subagent_result_metadata(kwargs)
|
||||
assert structured is not None
|
||||
assert "tool_receipts" not in structured
|
||||
assert "receipt_verdict" not in structured
|
||||
|
||||
@ -25,6 +25,7 @@ from deerflow.subagents.status_contract import (
|
||||
SUBAGENT_STATUS_KEY,
|
||||
SUBAGENT_STOP_REASON_KEY,
|
||||
SUBAGENT_TOKEN_USAGE_KEY,
|
||||
SUBAGENT_TOOL_RECEIPTS_KEY,
|
||||
)
|
||||
|
||||
# Use module import so tests can patch the exact symbols referenced inside task_tool().
|
||||
@ -78,6 +79,7 @@ def _make_result(
|
||||
error: str | None = None,
|
||||
stop_reason: str | None = None,
|
||||
token_usage_records: list[dict] | None = None,
|
||||
tool_receipts: list[dict] | None = None,
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
status=status,
|
||||
@ -87,6 +89,7 @@ def _make_result(
|
||||
stop_reason=stop_reason,
|
||||
token_usage_records=token_usage_records or [],
|
||||
usage_reported=False,
|
||||
tool_receipts=tool_receipts,
|
||||
)
|
||||
|
||||
|
||||
@ -1189,10 +1192,22 @@ def test_task_tool_polling_safety_timeout(monkeypatch):
|
||||
)
|
||||
monkeypatch.setattr(task_tool_module, "get_subagent_config", lambda _: config)
|
||||
|
||||
receipts = [
|
||||
{
|
||||
"id": "r1",
|
||||
"tool_call_id": "tc-before-poll-timeout",
|
||||
"tool_name": "write_file",
|
||||
"status": "success",
|
||||
"args_sha256": "a" * 16,
|
||||
"output_sha256": "b" * 16,
|
||||
"output_bytes": 3,
|
||||
"created_at": "2026-08-28T00:00:00+00:00",
|
||||
}
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
task_tool_module,
|
||||
"get_background_task_result",
|
||||
lambda _: _make_result(FakeSubagentStatus.RUNNING, ai_messages=[]),
|
||||
lambda _: _make_result(FakeSubagentStatus.RUNNING, ai_messages=[], tool_receipts=receipts),
|
||||
)
|
||||
monkeypatch.setattr(task_tool_module, "get_stream_writer", lambda: events.append)
|
||||
monkeypatch.setattr(task_tool_module.asyncio, "sleep", _no_sleep)
|
||||
@ -1211,6 +1226,7 @@ def test_task_tool_polling_safety_timeout(monkeypatch):
|
||||
assert message.content.startswith("Task polling timed out after 0 minutes")
|
||||
assert message.additional_kwargs[SUBAGENT_STATUS_KEY] == "polling_timed_out"
|
||||
assert message.additional_kwargs[SUBAGENT_ERROR_KEY] == message.content
|
||||
assert message.additional_kwargs[SUBAGENT_TOOL_RECEIPTS_KEY] == receipts
|
||||
assert events[0]["type"] == "task_started"
|
||||
assert events[-1]["type"] == "task_timed_out"
|
||||
|
||||
@ -1974,3 +1990,118 @@ async def test_deferred_cleanup_task_retained_and_survives_gc(monkeypatch):
|
||||
|
||||
assert cleaned == ["exec-gc"]
|
||||
assert weak_task() not in task_tool_module._deferred_cleanup_tasks
|
||||
|
||||
|
||||
def _receipt_fixture(rid: str = "r1", tool: str = "write_file", status: str = "success") -> dict:
|
||||
return {
|
||||
"id": rid,
|
||||
"tool_call_id": f"tc-{rid}",
|
||||
"tool_name": tool,
|
||||
"status": status,
|
||||
"args_sha256": "a" * 16,
|
||||
"output_sha256": "b" * 16,
|
||||
"output_bytes": 10,
|
||||
"created_at": "2026-08-24T00:00:00+00:00",
|
||||
}
|
||||
|
||||
|
||||
def _run_completed_task_tool(monkeypatch, *, result_text: str, tool_receipts: list[dict] | None) -> ToolMessage:
|
||||
"""Drive the completed branch and return the terminal ToolMessage."""
|
||||
|
||||
class DummyExecutor:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def execute_async(self, prompt, task_id=None):
|
||||
return task_id or "generated-task-id"
|
||||
|
||||
monkeypatch.setattr(task_tool_module, "SubagentStatus", FakeSubagentStatus)
|
||||
monkeypatch.setattr(task_tool_module, "SubagentExecutor", DummyExecutor)
|
||||
monkeypatch.setattr(task_tool_module, "get_subagent_config", lambda _: _make_subagent_config())
|
||||
monkeypatch.setattr(
|
||||
task_tool_module,
|
||||
"get_background_task_result",
|
||||
lambda _: _make_result(FakeSubagentStatus.COMPLETED, result=result_text, tool_receipts=tool_receipts),
|
||||
)
|
||||
monkeypatch.setattr(task_tool_module, "get_stream_writer", lambda: lambda _event: None)
|
||||
monkeypatch.setattr(task_tool_module.asyncio, "sleep", _no_sleep)
|
||||
monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: [])
|
||||
|
||||
command = _run_task_tool(
|
||||
runtime=_make_runtime(),
|
||||
description="test",
|
||||
prompt="p",
|
||||
subagent_type="general-purpose",
|
||||
tool_call_id="tc-verify",
|
||||
)
|
||||
return _task_tool_message(command)
|
||||
|
||||
|
||||
def test_task_tool_completed_attaches_receipts_and_verdict(monkeypatch):
|
||||
receipts = [_receipt_fixture("r1", "write_file")]
|
||||
message = _run_completed_task_tool(monkeypatch, result_text="saved the report [r1]", tool_receipts=receipts)
|
||||
|
||||
assert message.additional_kwargs["subagent_tool_receipts"] == receipts
|
||||
verdict = message.additional_kwargs["subagent_receipt_verdict"]
|
||||
assert verdict["citation_resolved"] is True
|
||||
assert verdict["resolved"] == ["r1"]
|
||||
|
||||
|
||||
def test_task_tool_completed_flags_unknown_citation(monkeypatch):
|
||||
receipts = [_receipt_fixture("r1", "write_file")]
|
||||
message = _run_completed_task_tool(monkeypatch, result_text="uploaded [r9]", tool_receipts=receipts)
|
||||
|
||||
verdict = message.additional_kwargs["subagent_receipt_verdict"]
|
||||
assert verdict["citation_resolved"] is False
|
||||
assert verdict["unknown"] == ["r9"]
|
||||
|
||||
|
||||
def test_task_tool_completed_flags_uncited_action_claims(monkeypatch):
|
||||
message = _run_completed_task_tool(monkeypatch, result_text="I wrote the report.", tool_receipts=[])
|
||||
|
||||
verdict = message.additional_kwargs["subagent_receipt_verdict"]
|
||||
assert verdict["no_citation_claims"] is True
|
||||
assert verdict["citation_resolved"] is False
|
||||
|
||||
|
||||
def test_task_tool_completed_without_receipts_produces_no_verdict(monkeypatch):
|
||||
# receipts=None means no harvest happened (disabled or pre-stream end):
|
||||
# skip the verdict entirely, keeping disabled deployments pre-PR2.
|
||||
message = _run_completed_task_tool(monkeypatch, result_text="done [r1]", tool_receipts=None)
|
||||
|
||||
assert "subagent_receipt_verdict" not in message.additional_kwargs
|
||||
assert "subagent_tool_receipts" not in message.additional_kwargs
|
||||
|
||||
|
||||
def test_task_tool_failed_carries_receipts_without_verdict(monkeypatch):
|
||||
receipts = [_receipt_fixture("r1", "bash", status="error")]
|
||||
|
||||
class DummyExecutor:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def execute_async(self, prompt, task_id=None):
|
||||
return task_id or "generated-task-id"
|
||||
|
||||
monkeypatch.setattr(task_tool_module, "SubagentStatus", FakeSubagentStatus)
|
||||
monkeypatch.setattr(task_tool_module, "SubagentExecutor", DummyExecutor)
|
||||
monkeypatch.setattr(task_tool_module, "get_subagent_config", lambda _: _make_subagent_config())
|
||||
monkeypatch.setattr(
|
||||
task_tool_module,
|
||||
"get_background_task_result",
|
||||
lambda _: _make_result(FakeSubagentStatus.FAILED, error="boom", tool_receipts=receipts),
|
||||
)
|
||||
monkeypatch.setattr(task_tool_module, "get_stream_writer", lambda: lambda _event: None)
|
||||
monkeypatch.setattr(task_tool_module.asyncio, "sleep", _no_sleep)
|
||||
monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: [])
|
||||
|
||||
command = _run_task_tool(
|
||||
runtime=_make_runtime(),
|
||||
description="test",
|
||||
prompt="p",
|
||||
subagent_type="general-purpose",
|
||||
tool_call_id="tc-failed",
|
||||
)
|
||||
message = _task_tool_message(command)
|
||||
assert message.additional_kwargs["subagent_tool_receipts"] == receipts
|
||||
assert "subagent_receipt_verdict" not in message.additional_kwargs
|
||||
|
||||
@ -6,8 +6,13 @@ from langchain_core.messages import AIMessage, ToolMessage
|
||||
|
||||
from deerflow.agents.middlewares.tool_receipt import (
|
||||
TOOL_RECEIPT_KEY,
|
||||
TOOL_RECEIPT_LEDGER_KEY,
|
||||
extract_citing_turn_receipts,
|
||||
extract_tool_receipts,
|
||||
format_citation,
|
||||
make_tool_receipt,
|
||||
parse_citations,
|
||||
receipt_id,
|
||||
render_tool_receipts,
|
||||
)
|
||||
from deerflow.agents.middlewares.tool_result_meta import TOOL_META_KEY
|
||||
@ -29,6 +34,29 @@ def _stamped_msg(content: str, *, tool_call_id: str, name: str, args: dict | Non
|
||||
return message
|
||||
|
||||
|
||||
def test_citation_format_round_trip():
|
||||
"""format_citation and parse_citations are inverse over the wire format."""
|
||||
assert parse_citations(format_citation(receipt_id(2), "write_file")) == [("r2", "write_file")]
|
||||
assert parse_citations(format_citation(receipt_id(2))) == [("r2", None)]
|
||||
|
||||
|
||||
def test_parse_citations_ignores_oversized_model_generated_id():
|
||||
oversized = "9" * 5000
|
||||
|
||||
assert parse_citations(f"bad [r{oversized}] then valid [r2]") == [("r2", None)]
|
||||
|
||||
|
||||
def test_render_prompt_example_matches_verifier_format():
|
||||
"""The instruction example is generated from the shared format, so the
|
||||
parent-side verifier must be able to parse it back."""
|
||||
receipts = extract_tool_receipts([_stamped_msg("ok", tool_call_id="tc-1", name="write_file")])
|
||||
text = render_tool_receipts(receipts)
|
||||
|
||||
example = format_citation(receipt_id(1), "write_file")
|
||||
assert f"e.g. {example})" in text
|
||||
assert parse_citations(example) == [("r1", "write_file")]
|
||||
|
||||
|
||||
def test_make_tool_receipt_hashes_args_and_output():
|
||||
receipt = make_tool_receipt(
|
||||
{"name": "write_file", "id": "tc-1", "args": {"path": "/tmp/a.txt", "content": "hello"}},
|
||||
@ -69,6 +97,52 @@ def test_extract_assigns_sequential_ids_and_skips_unstamped():
|
||||
assert receipts[1]["tool_name"] == "bash"
|
||||
|
||||
|
||||
def test_extract_citing_turn_receipts_preserves_pre_compaction_ids():
|
||||
original = extract_tool_receipts(
|
||||
[
|
||||
_stamped_msg("first", tool_call_id="tc-original", name="write_file"),
|
||||
_stamped_msg("second", tool_call_id="tc-survivor", name="bash"),
|
||||
]
|
||||
)
|
||||
final_report = AIMessage(
|
||||
content="saved it [r1]",
|
||||
additional_kwargs={TOOL_RECEIPT_LEDGER_KEY: [dict(receipt) for receipt in original]},
|
||||
)
|
||||
# Summarization dropped the original r1 ToolMessage. A terminal re-scan
|
||||
# would rename tc-survivor to r1, but the citing-turn snapshot must not.
|
||||
compacted_messages = [
|
||||
_stamped_msg("second", tool_call_id="tc-survivor", name="bash"),
|
||||
final_report,
|
||||
]
|
||||
|
||||
receipts = extract_citing_turn_receipts(compacted_messages)
|
||||
|
||||
assert receipts == original
|
||||
assert extract_tool_receipts(compacted_messages)[0]["tool_call_id"] == "tc-survivor"
|
||||
|
||||
|
||||
def test_extract_citing_turn_receipts_accepts_truncated_original_id_range():
|
||||
original = extract_tool_receipts([_stamped_msg(f"result-{index}", tool_call_id=f"tc-{index}", name="bash") for index in range(1, 31)])
|
||||
retained = original[23:]
|
||||
final_report = AIMessage(
|
||||
content="latest result [r30]",
|
||||
additional_kwargs={TOOL_RECEIPT_LEDGER_KEY: [dict(receipt) for receipt in retained]},
|
||||
)
|
||||
|
||||
receipts = extract_citing_turn_receipts([final_report])
|
||||
|
||||
assert receipts == retained
|
||||
assert [receipt["id"] for receipt in receipts] == [f"r{index}" for index in range(24, 31)]
|
||||
|
||||
|
||||
def test_extract_citing_turn_receipts_rejects_non_contiguous_original_ids():
|
||||
original = extract_tool_receipts([_stamped_msg(f"result-{index}", tool_call_id=f"tc-{index}", name="bash") for index in range(1, 4)])
|
||||
malformed = [dict(original[0]), dict(original[2])]
|
||||
final_report = AIMessage(content="done", additional_kwargs={TOOL_RECEIPT_LEDGER_KEY: malformed})
|
||||
|
||||
assert extract_citing_turn_receipts([final_report]) is None
|
||||
|
||||
|
||||
def test_render_empty_and_budget():
|
||||
assert render_tool_receipts([]) == ""
|
||||
receipts = extract_tool_receipts([_stamped_msg("ok", tool_call_id="tc-1", name="write_file", args={"path": "/tmp/a"})])
|
||||
|
||||
@ -5,10 +5,11 @@ from __future__ import annotations
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from langchain.agents.middleware.types import ExtendedModelResponse, ModelResponse
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
from langgraph.types import Command
|
||||
|
||||
from deerflow.agents.middlewares.tool_receipt import TOOL_RECEIPT_KEY
|
||||
from deerflow.agents.middlewares.tool_receipt import TOOL_RECEIPT_KEY, TOOL_RECEIPT_LEDGER_KEY
|
||||
from deerflow.agents.middlewares.tool_receipt_middleware import ToolReceiptMiddleware
|
||||
from deerflow.agents.middlewares.tool_result_meta import TOOL_META_KEY
|
||||
|
||||
@ -97,14 +98,41 @@ def test_wrap_model_call_injects_hidden_ledger():
|
||||
request.override = lambda messages: SimpleNamespace(messages=messages)
|
||||
captured = {}
|
||||
|
||||
response_message = AIMessage(content="done [r1]")
|
||||
|
||||
def handler(req):
|
||||
captured["messages"] = req.messages
|
||||
return MagicMock()
|
||||
return ModelResponse(result=[response_message])
|
||||
|
||||
middleware.wrap_model_call(request, handler)
|
||||
ledger_messages = [m for m in captured["messages"] if isinstance(m, HumanMessage) and m.additional_kwargs.get("hide_from_ui")]
|
||||
assert len(ledger_messages) == 1
|
||||
assert "r1" in ledger_messages[0].content and "bash" in ledger_messages[0].content
|
||||
assert response_message.additional_kwargs[TOOL_RECEIPT_LEDGER_KEY][0]["tool_call_id"] == "tc-1"
|
||||
|
||||
|
||||
def test_wrap_model_call_snapshots_only_receipts_rendered_within_budget():
|
||||
middleware = ToolReceiptMiddleware()
|
||||
request = MagicMock()
|
||||
request.messages = [HumanMessage(content="go"), *[_stamped_message() for _ in range(30)]]
|
||||
request.override = lambda messages: SimpleNamespace(messages=messages)
|
||||
captured = {}
|
||||
response_message = AIMessage(content="done")
|
||||
|
||||
def handler(req):
|
||||
captured["messages"] = req.messages
|
||||
return ModelResponse(result=[response_message])
|
||||
|
||||
middleware.wrap_model_call(request, handler)
|
||||
|
||||
ledger_message = next(m for m in captured["messages"] if isinstance(m, HumanMessage) and m.additional_kwargs.get("hide_from_ui"))
|
||||
snapshot = response_message.additional_kwargs[TOOL_RECEIPT_LEDGER_KEY]
|
||||
assert snapshot
|
||||
assert snapshot[0]["id"] != "r1"
|
||||
assert snapshot[-1]["id"] == "r30"
|
||||
assert all(f"[{receipt['id']}]" in ledger_message.content for receipt in snapshot)
|
||||
assert "[r1]" not in ledger_message.content
|
||||
assert "older receipts omitted" in ledger_message.content
|
||||
|
||||
|
||||
def test_wrap_model_call_no_receipts_no_injection():
|
||||
@ -121,6 +149,18 @@ def test_wrap_model_call_no_receipts_no_injection():
|
||||
assert seen["request"] is request # untouched passthrough
|
||||
|
||||
|
||||
def test_wrap_model_call_stamps_extended_response_ledger():
|
||||
middleware = ToolReceiptMiddleware()
|
||||
request = MagicMock()
|
||||
request.messages = [HumanMessage(content="go"), _stamped_message()]
|
||||
request.override = lambda messages: SimpleNamespace(messages=messages)
|
||||
response_message = AIMessage(content="done [r1]")
|
||||
response = ExtendedModelResponse(model_response=ModelResponse(result=[response_message]))
|
||||
|
||||
assert middleware.wrap_model_call(request, lambda req: response) is response
|
||||
assert response_message.additional_kwargs[TOOL_RECEIPT_LEDGER_KEY][0]["id"] == "r1"
|
||||
|
||||
|
||||
def _delegation_only_request(messages: list) -> MagicMock:
|
||||
request = MagicMock()
|
||||
request.messages = messages
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user