From fd61c48fa2782b34ee080cf33454250c366212a2 Mon Sep 17 00:00:00 2001 From: Dragon ze <52615460+Aoisop@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:10:35 +0800 Subject: [PATCH] fix(acceptance): reject all-skipped go test records as evidence (#6957) A `go test -v` run whose every case is skipped exits 0 and prints the same `ok` package banner a passing run does, so the banner alone supplied positive evidence for `tests_passed:` while no test executed. unittest (#6916) and Cargo (#6891) already enforce the non-skipped bar; Go's per-case `--- SKIP` lines are the only record content that separates the two here. Non-verbose `go test` output is byte-identical for an all-skipped and a passing package, so those records stay judged by the banner alone. --- README.md | 2 +- .../harness/deerflow/subagents/AGENTS.md | 2 +- .../deerflow/subagents/acceptance_checks.py | 24 +++- backend/tests/test_go_acceptance.py | 108 ++++++++++++++++++ 4 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 backend/tests/test_go_acceptance.py diff --git a/README.md b/README.md index a8b382792c..72ea9f0f26 100644 --- a/README.md +++ b/README.md @@ -2173,7 +2173,7 @@ result text does not turn an unknown status into success or failure. Cancelled or timed-out background sub-agent executions retain provider-reported token usage from completed model calls, including responses received before their next progress update. Final usage delivery to the parent run does not count earlier progress snapshots twice. -For `tests_passed:go test ./...`, packages marked `[no test files]` or `[no tests to run]` do not veto a passing summary from another package. Runs with only zero-test package summaries remain `UNVERIFIED`; failures still take precedence. This checks recorded execution evidence, not claim correctness. +For `tests_passed:go test ./...`, packages marked `[no test files]` or `[no tests to run]` do not veto a passing summary from another package. Runs with only zero-test package summaries remain `UNVERIFIED`; failures still take precedence. Under `-v`, a record whose per-case lines are all `--- SKIP` also remains `UNVERIFIED`: Go exits 0 and prints the same `ok` banner, so nothing ran to pass. Non-verbose output cannot separate an all-skipped package from a passing one and stays trusted as recorded. This checks recorded execution evidence, not claim correctness. For `tests_passed:cargo test`, a complete passing target summary can coexist with empty unit, integration or documentation targets. Complete zero-pass successful diff --git a/backend/packages/harness/deerflow/subagents/AGENTS.md b/backend/packages/harness/deerflow/subagents/AGENTS.md index 5499325eb4..4bc4ed8f6b 100644 --- a/backend/packages/harness/deerflow/subagents/AGENTS.md +++ b/backend/packages/harness/deerflow/subagents/AGENTS.md @@ -80,7 +80,7 @@ threaded initialization, peer cancellation and renewal drain. **Concurrency and total delegation cap**: Ordinary `task` concurrency is resolved once as the minimum of the per-run request, the startup-frozen `subagent_runtime.max_running`, and the schema safety ceiling (1-64), then shared by the lead prompt and `SubagentLimitMiddleware`. Hot reloads must not make either layer advertise more capacity than the already-created process controller; a changed startup-only value takes effect only after restart. The same middleware separately enforces `subagents.max_total_per_run` (default 6, config schema 1-50, runtime override `max_total_subagents` clamped to the same range) against current-run entries in the durable delegation ledger, so a long lead-agent run cannot bypass concurrency limits by launching repeated legal-sized batches at each planning checkpoint, but historical delegations from previous runs in the same thread do not consume the new run's budget. Explicit `batch_task` work does not consume or relax that ordinary-run ledger: its persisted total/live/running limits live under `subagent_batches`. Gateway `run_agent()` and embedded `DeerFlowClient.stream()` both provide a per-invocation `run_id` in runtime context; `DeerFlowClient.stream()` also tags its input `HumanMessage` with that same id so durable-context capture can identify the current request boundary. Gateway resume paths may not append a new `HumanMessage`, so the worker also exposes the pre-run checkpoint's message ids in runtime context; durable-context capture uses that as the current-run boundary and never re-tags older task calls as the resumed run. When no delegation slots remain, task calls are stripped, provider raw tool-call metadata is synced, `finish_reason` is forced to `stop`, and a visible "subagent delegation limit" note is appended so the agent can synthesize already-collected results. Default subagent timeout `subagents.timeout_seconds=1800` (30 min) and built-in `general-purpose` `max_turns=150`. **Flow**: `task()` → `SubagentExecutor` → shared slot → polling/SSE. Explicit `batch_task()` → durable rows → lease-based batch service (`subagents/batch_service.py`, Gateway or direct runtime) → same executor/slots → bounded result and owner-scoped API/JSONL. Batch mode is explicit, never inferred from prompt size. Queue rejection/timeout releases the lease without an item attempt; execution failure/expired leases consume one. User cancellation terminalizes nonterminal items and clears leases, fencing stale workers. Resolve background cancellation under `_background_tasks_lock`, then call `Future.cancel()` outside it because callbacks may reacquire the lock. `task_started` includes the effective model. `SubagentTokenCollector` publishes cumulative usage after each LLM response; `task_running` updates cards without double-counting. Terminal ToolMessage metadata (`subagent_model_name`, `subagent_token_usage`) and `subagent.end` preserve model/usage after reload; absent usage stays absent. Resolve one `AppConfig` snapshot (explicit or `get_app_config()`) for assembly, deferred setup, and receipt harvesting; `verification.receipts_enabled=false` holds on both construction paths. Harvest terminal receipts before `try_set_terminal` under the result lock so polling cannot see terminal status without metadata. Each values chunk updates the latest harvest state and publishes receipts before cooperative cancellation. Cancellation/failure scans current ToolMessages; completion uses the bounded ledger snapshot attached to returned assistant text, including max-turn partials. Missing/malformed completed snapshots yield no receipts, preventing omitted receipts from validating citations. Cancellation/timeouts retain the latest streamed evidence. **Report contract (RFC #4651 PR3)**: `report_contract.py` owns the prompt-layer text that makes Layer 1 receipt verification non-inert. `SubagentExecutor._build_initial_state` appends `build_report_contract_section(receipts_enabled=...)` to every subagent's consolidated `SystemMessage` — built-in and custom alike — requiring `[rN tool_name]` citations (from the Tool receipts ledger) for action claims, verifiable handles (absolute path, URL, ID, HTTP status) for deliverables, and explicit reporting of failures; the citation clause follows `verification.receipts_enabled`, and the citation example derives from the single-owner `format_citation`/`receipt_id` so prompt text cannot drift from the verifier. The `task` tool hands lead-supplied `acceptance_criteria` to the `SubagentExecutor` constructor, which appends them via `render_acceptance_criteria_block(...)` to the task `HumanMessage` (stripped, capped at 20 items × 500 chars, each entry neutralized) — the untrusted channel `InputSanitizationMiddleware` escapes and boundary-frames, matching their model-supplied provenance. The subagent's `SystemMessage` never carries criterion text; it gets only the framework-owned `build_acceptance_criteria_system_note(...)` pointer naming the list's location and authority, so natural-language injection inside a criterion cannot gain system-channel priority over framework instructions. Deterministic leaf checking is a separate layer. -**Acceptance checklist (RFC #4651 PR4)**: `acceptance_checks.py` checks lead-supplied `acceptance_criteria` in code on the `task` tool's completed branch (offloaded via `asyncio.to_thread`, failure-isolated). Decidable leaves: `file: exists|non-empty` and `file_written:` read through `read_current_file_content` scoped to the shared thread workspace (`workspace_path`/`outputs_path`; virtual `/mnt/user-data/...` prefixes and workspace-relative spellings normalized first) — the read itself uses the sandbox-native virtual form because the local read validator and provider mount tables resolve virtual paths, not host paths; a remote provider's `"Error: ..."` return string (remote providers return instead of raise for missing files) is normalized to a failed check, never evaluated as content — provider-typed via `is_local_sandbox`, so a genuine `Error:`-prefixed file on the local sandbox stays valid content; a UnicodeDecodeError on a text read marks a binary deliverable (PDF/image) as existing and non-empty instead of dropping the verdict; reads are byte-bounded — the size is established first (`os.stat` on the validated host path locally, so the host-bash-disabled configuration needs no shell; a metadata-only `stat`/`realpath` probe in a fresh `env -i` shell on remote providers — absolute-path utilities and a marker env that routes AIO off its persistent shell, so a completed subagent's poisoned session state cannot steer it; `stat` never opens content, so a FIFO cannot block the parent for the provider's idle timeout; the file must stat as a regular non-symlink file, and containment is canonicalized — the file's realpath must stay under the mount root's realpath, which is exactly what the provider's own read path resolves (e2b and Tenki realize `/mnt/user-data` as a symlink to the home dir by default), so a final-component symlink is rejected outright and an intermediate dir-link escape under a sane root still lands outside the canonical root), leaves above `_FILE_CONTENT_READ_CAP_BYTES` answer from the size alone — `file_written` only with an added bounded one-byte open probe (a mode-000 file stats fine while any open raises EACCES), and smaller files run the full read, and an unestablishable size degrades to UNVERIFIED rather than an unbounded fallback read; out-of-scope paths degrade to UNVERIFIED, never misjudge — on the local sandbox the scope decision canonicalizes with realpath, so a workspace symlink into uploads cannot satisfy a scoped leaf with upload content. `tests_passed:` anchors to a matching bash execution (newest match wins) with `status=success` and a test-summary shape in its bounded output tail — each harvested execution carries a `shell_persistent` provenance stamp — the producing sandbox's `persistent_shell_sessions` flag (AIO's legacy exec path) resolved from the state that carried the evidence, never the parent runtime, which has no `sandbox` key when the parent delegated before touching one — and a persistent stamp (or an unidentifiable or undeclared one — a custom provider that never declared its session semantics is unknown, not fresh-shell; unknown provenance fails closed) degrades the leaf to UNVERIFIED instead, because any earlier call in the shared session could have mutated the state the clean-looking run executed in and a fresh controlled session (RFC §6 verifier) would be needed to prove otherwise, harvested by the executor (`_harvest_bash_executions`, only when criteria were delegated) from the same stamped `ToolMessage`s the receipt layer reads, accumulated per streamed chunk (merged by `tool_call_id`, newest-capped) so subagent summarization compacting earlier messages cannot erase a recorded execution, with over-cap commands carrying `command_truncated` so the matcher degrades to UNVERIFIED instead of proving a match on a suffix-less prefix; the recorded status is the actual shell exit status parsed from the output's `Exit Code: N` / `Command exited with code N` marker (a nonzero bash exit returns ordinary text that `deerflow_tool_meta` still reports as success; every provider appends a marker on nonzero exit — local/e2b/opensandbox/tenki/boxlite textually, aio via the SDK's structured exit_code — and timeouts append `Exit Code: 124` (local process-group kill, AIO `hard_timeout`, e2b/tenki SDK deadlines, boxlite bridge deadline) while signal kills parse as signed markers (`Exit Code: -9` records error); `_truncate_bash_output` preserves a trailing exit marker inside its budget, with a 32-char floor for smaller configured limits and the remote `Command exited with code N` form accepted only as the whole trimmed output; the matched marker text travels on the entry as `status_marker`), falling back to the meta status only when no marker exists; matching is shell-structure aware (operator-separated segments — a physical newline separates with `;` semantics — or with the continuation operator the next line opens with (`cmd1\n&& cmd2` is `&&`; `cmd1\n|| cmd2` is `||`, which after a successful first command skips the rest while exiting 0, so flattening it to `;` would record a run that never happened) —, so a multi-line script's trailing `echo`/`seq` lines are never merged into the matched segment's arguments, status, or output attribution —, executable identity — directional: a bare criterion executable accepts any path spelling of the name, while an explicitly path-spelled criterion requires a path-spelled execution of the same normalized executable path (spelling judged on the raw token: `./pytest` names the project-local file and normpath collapsing `./` must not demote it to a PATH lookup; a `..` component on either side is unprovable outright), so `/tmp/fake/pytest` cannot certify `/opt/project/.venv/bin/pytest` —, ordered argument subsequence whose env-assignment prefix must equal the criterion's exactly as an effective name → final-value mapping — extra, missing, or differently-valued assignments degrade the match, as does any reordering of a repeated name (`CI=0 CI=1` vs `CI=1 CI=0` are last-wins opposite environments), since no variable is provably inert across repositories, and any assignment or argumented `export`/`unset` in a preceding segment is state pollution, and any span token carrying a runtime expansion (`$VAR`/`$( )`/backticks) or an extra token carrying glob metacharacters (crafted option-looking filenames narrow invisibly) is likewise unprovable), so a command that merely mentions the criterion string (`echo '12 passed'; # pytest x.py`) cannot anchor the leaf, and control flow is preserved — the matching span must end at the command's last segment with provable execution (`&&` needs recorded success, `||` needs recorded failure, pipelines inside the span and backgrounding are never provable), and the criterion's own connectors are preserved — an expected `&&` executed as `;` (`cd missing; pytest x` for `cd missing && pytest x`) lets a failed preceding step be bypassed, so it degrades; only the stricter direction (criterion `;` executed as `&&` with recorded success) survives), so a short-circuited segment (`false && pytest x; echo '3 passed'`) degrades to UNVERIFIED instead of a false hold; the summary shape is evaluated only when the output is attributable to the matched segment (every preceding segment provably silent by invocation form — only shape-free `cd dir` and pure assignments qualify; `pushd`/`umask`/`ulimit`, any `export`/`unset` (including an invalid identifier, whose bash error text is subagent-chosen), and any `source`/`.`, whose `*/bin/activate` path shape says nothing about what a crafted script prints, are not; the `cd` print channel is closed too — CDPATH makes `cd` print the subagent-chosen resolved path and the pass shapes match as substrings, so a `cd` argument carrying a summary shape or runtime expansion is non-silent, and any `CDPATH=` assignment degrades the match as state pollution), Go `[no test files]` / `[no tests to run]` lines are package-local: other packages' passes survive; all-zero stays UNVERIFIED; `0 passed` / `Ran 0 tests` veto, argument matching treats tokens consumed by negating options (`--ignore`/`--deselect`/`--exclude`/`--skip` families) as ineligible evidence (`pytest --ignore tests/security tests` cannot affirm the excluded tests) and a target negated anywhere in the command degrades the match (`pytest tests/security tests/unit --ignore tests/security` cannot certify the excluded target), and a bare criterion with no positional selection target (bare `pytest`, `make test`) treats ANY negating option as a different selection, and accepts extra executed flags only when provably selection-preserving (verbosity/output/xdist/coverage forms; `-k`/`-m`/`--lf`/`--collect-only`/`-c`/`-p` and friends degrade to UNVERIFIED — `pytest -k smoke tests/security` cannot certify the full selection; extra positionals widen only after a path-scoped criterion — scoping reads positional targets by option arity, so a path embedded in an option (`--basetemp=/tmp/p`) never counts as a target, and an option of unknown arity (absent from the value-taking table, no glued `=`) followed by a path-like token (`--rootdir /tmp/project`) fails closed, so `python -m unittest pkg.OneTest` cannot certify bare `python -m unittest`), and criterion text is neutralized with `neutralize_untrusted_tags` before storage/rendering so a blocked tag in a criterion can never reach the lead-visible checklist section. Other criteria stay UNVERIFIED. The verdict (`subagent_acceptance_verdict` in `additional_kwargs`, validated on read) feeds the delegation ledger's `acceptance:` segment and the model-visible checklist section appended to the result text; the gateway strips caller-forged verdicts from both ledger entries and message metadata like the citation verdict. Leaf read errors catch only `OSError`/`SandboxError` — unexpected errors propagate to the task-tool-level isolation so the blocking-IO gate keeps teeth (`tests/blocking_io/test_task_tool_acceptance_checklist.py`). Known accepted boundaries (pinned by `TestKnownBoundaries` — conscious, do not re-raise): a bare criterion executable trusts PATH/filesystem spelling (a subagent-crafted same-name script is textually indistinguishable; spelling an absolute criterion path restricts evidence to that normalized path); relative criterion targets resolve in the wrapper's cwd, so `cd` prefixes must stay in scope (relative without `..`, or absolute under the thread data roots / virtual prefix) while a symlink inside an allowed root pointing out is a filesystem-layer concern; runner semantics are trusted (a Makefile swallowing failures, a runner exiting 0 on failure) — Layer 2 is execution evidence only, claim correctness belongs to the PR5 judge / RFC §6 re-execution; evidence is bounded (500-char command, 1000-char tail) and truncation degrades to UNVERIFIED rather than proving. +**Acceptance checklist (RFC #4651 PR4)**: `acceptance_checks.py` checks lead-supplied `acceptance_criteria` in code on the `task` tool's completed branch (offloaded via `asyncio.to_thread`, failure-isolated). Decidable leaves: `file: exists|non-empty` and `file_written:` read through `read_current_file_content` scoped to the shared thread workspace (`workspace_path`/`outputs_path`; virtual `/mnt/user-data/...` prefixes and workspace-relative spellings normalized first) — the read itself uses the sandbox-native virtual form because the local read validator and provider mount tables resolve virtual paths, not host paths; a remote provider's `"Error: ..."` return string (remote providers return instead of raise for missing files) is normalized to a failed check, never evaluated as content — provider-typed via `is_local_sandbox`, so a genuine `Error:`-prefixed file on the local sandbox stays valid content; a UnicodeDecodeError on a text read marks a binary deliverable (PDF/image) as existing and non-empty instead of dropping the verdict; reads are byte-bounded — the size is established first (`os.stat` on the validated host path locally, so the host-bash-disabled configuration needs no shell; a metadata-only `stat`/`realpath` probe in a fresh `env -i` shell on remote providers — absolute-path utilities and a marker env that routes AIO off its persistent shell, so a completed subagent's poisoned session state cannot steer it; `stat` never opens content, so a FIFO cannot block the parent for the provider's idle timeout; the file must stat as a regular non-symlink file, and containment is canonicalized — the file's realpath must stay under the mount root's realpath, which is exactly what the provider's own read path resolves (e2b and Tenki realize `/mnt/user-data` as a symlink to the home dir by default), so a final-component symlink is rejected outright and an intermediate dir-link escape under a sane root still lands outside the canonical root), leaves above `_FILE_CONTENT_READ_CAP_BYTES` answer from the size alone — `file_written` only with an added bounded one-byte open probe (a mode-000 file stats fine while any open raises EACCES), and smaller files run the full read, and an unestablishable size degrades to UNVERIFIED rather than an unbounded fallback read; out-of-scope paths degrade to UNVERIFIED, never misjudge — on the local sandbox the scope decision canonicalizes with realpath, so a workspace symlink into uploads cannot satisfy a scoped leaf with upload content. `tests_passed:` anchors to a matching bash execution (newest match wins) with `status=success` and a test-summary shape in its bounded output tail — each harvested execution carries a `shell_persistent` provenance stamp — the producing sandbox's `persistent_shell_sessions` flag (AIO's legacy exec path) resolved from the state that carried the evidence, never the parent runtime, which has no `sandbox` key when the parent delegated before touching one — and a persistent stamp (or an unidentifiable or undeclared one — a custom provider that never declared its session semantics is unknown, not fresh-shell; unknown provenance fails closed) degrades the leaf to UNVERIFIED instead, because any earlier call in the shared session could have mutated the state the clean-looking run executed in and a fresh controlled session (RFC §6 verifier) would be needed to prove otherwise, harvested by the executor (`_harvest_bash_executions`, only when criteria were delegated) from the same stamped `ToolMessage`s the receipt layer reads, accumulated per streamed chunk (merged by `tool_call_id`, newest-capped) so subagent summarization compacting earlier messages cannot erase a recorded execution, with over-cap commands carrying `command_truncated` so the matcher degrades to UNVERIFIED instead of proving a match on a suffix-less prefix; the recorded status is the actual shell exit status parsed from the output's `Exit Code: N` / `Command exited with code N` marker (a nonzero bash exit returns ordinary text that `deerflow_tool_meta` still reports as success; every provider appends a marker on nonzero exit — local/e2b/opensandbox/tenki/boxlite textually, aio via the SDK's structured exit_code — and timeouts append `Exit Code: 124` (local process-group kill, AIO `hard_timeout`, e2b/tenki SDK deadlines, boxlite bridge deadline) while signal kills parse as signed markers (`Exit Code: -9` records error); `_truncate_bash_output` preserves a trailing exit marker inside its budget, with a 32-char floor for smaller configured limits and the remote `Command exited with code N` form accepted only as the whole trimmed output; the matched marker text travels on the entry as `status_marker`), falling back to the meta status only when no marker exists; matching is shell-structure aware (operator-separated segments — a physical newline separates with `;` semantics — or with the continuation operator the next line opens with (`cmd1\n&& cmd2` is `&&`; `cmd1\n|| cmd2` is `||`, which after a successful first command skips the rest while exiting 0, so flattening it to `;` would record a run that never happened) —, so a multi-line script's trailing `echo`/`seq` lines are never merged into the matched segment's arguments, status, or output attribution —, executable identity — directional: a bare criterion executable accepts any path spelling of the name, while an explicitly path-spelled criterion requires a path-spelled execution of the same normalized executable path (spelling judged on the raw token: `./pytest` names the project-local file and normpath collapsing `./` must not demote it to a PATH lookup; a `..` component on either side is unprovable outright), so `/tmp/fake/pytest` cannot certify `/opt/project/.venv/bin/pytest` —, ordered argument subsequence whose env-assignment prefix must equal the criterion's exactly as an effective name → final-value mapping — extra, missing, or differently-valued assignments degrade the match, as does any reordering of a repeated name (`CI=0 CI=1` vs `CI=1 CI=0` are last-wins opposite environments), since no variable is provably inert across repositories, and any assignment or argumented `export`/`unset` in a preceding segment is state pollution, and any span token carrying a runtime expansion (`$VAR`/`$( )`/backticks) or an extra token carrying glob metacharacters (crafted option-looking filenames narrow invisibly) is likewise unprovable), so a command that merely mentions the criterion string (`echo '12 passed'; # pytest x.py`) cannot anchor the leaf, and control flow is preserved — the matching span must end at the command's last segment with provable execution (`&&` needs recorded success, `||` needs recorded failure, pipelines inside the span and backgrounding are never provable), and the criterion's own connectors are preserved — an expected `&&` executed as `;` (`cd missing; pytest x` for `cd missing && pytest x`) lets a failed preceding step be bypassed, so it degrades; only the stricter direction (criterion `;` executed as `&&` with recorded success) survives), so a short-circuited segment (`false && pytest x; echo '3 passed'`) degrades to UNVERIFIED instead of a false hold; the summary shape is evaluated only when the output is attributable to the matched segment (every preceding segment provably silent by invocation form — only shape-free `cd dir` and pure assignments qualify; `pushd`/`umask`/`ulimit`, any `export`/`unset` (including an invalid identifier, whose bash error text is subagent-chosen), and any `source`/`.`, whose `*/bin/activate` path shape says nothing about what a crafted script prints, are not; the `cd` print channel is closed too — CDPATH makes `cd` print the subagent-chosen resolved path and the pass shapes match as substrings, so a `cd` argument carrying a summary shape or runtime expansion is non-silent, and any `CDPATH=` assignment degrades the match as state pollution), Go `[no test files]` / `[no tests to run]` lines are package-local: other packages' passes survive; all-zero stays UNVERIFIED, as does a `-v` record whose per-case lines are only `--- SKIP`; `0 passed` / `Ran 0 tests` veto, argument matching treats tokens consumed by negating options (`--ignore`/`--deselect`/`--exclude`/`--skip` families) as ineligible evidence (`pytest --ignore tests/security tests` cannot affirm the excluded tests) and a target negated anywhere in the command degrades the match (`pytest tests/security tests/unit --ignore tests/security` cannot certify the excluded target), and a bare criterion with no positional selection target (bare `pytest`, `make test`) treats ANY negating option as a different selection, and accepts extra executed flags only when provably selection-preserving (verbosity/output/xdist/coverage forms; `-k`/`-m`/`--lf`/`--collect-only`/`-c`/`-p` and friends degrade to UNVERIFIED — `pytest -k smoke tests/security` cannot certify the full selection; extra positionals widen only after a path-scoped criterion — scoping reads positional targets by option arity, so a path embedded in an option (`--basetemp=/tmp/p`) never counts as a target, and an option of unknown arity (absent from the value-taking table, no glued `=`) followed by a path-like token (`--rootdir /tmp/project`) fails closed, so `python -m unittest pkg.OneTest` cannot certify bare `python -m unittest`), and criterion text is neutralized with `neutralize_untrusted_tags` before storage/rendering so a blocked tag in a criterion can never reach the lead-visible checklist section. Other criteria stay UNVERIFIED. The verdict (`subagent_acceptance_verdict` in `additional_kwargs`, validated on read) feeds the delegation ledger's `acceptance:` segment and the model-visible checklist section appended to the result text; the gateway strips caller-forged verdicts from both ledger entries and message metadata like the citation verdict. Leaf read errors catch only `OSError`/`SandboxError` — unexpected errors propagate to the task-tool-level isolation so the blocking-IO gate keeps teeth (`tests/blocking_io/test_task_tool_acceptance_checklist.py`). Known accepted boundaries (pinned by `TestKnownBoundaries` — conscious, do not re-raise): a bare criterion executable trusts PATH/filesystem spelling (a subagent-crafted same-name script is textually indistinguishable; spelling an absolute criterion path restricts evidence to that normalized path); relative criterion targets resolve in the wrapper's cwd, so `cd` prefixes must stay in scope (relative without `..`, or absolute under the thread data roots / virtual prefix) while a symlink inside an allowed root pointing out is a filesystem-layer concern; runner semantics are trusted (a Makefile swallowing failures, a runner exiting 0 on failure) — Layer 2 is execution evidence only, claim correctness belongs to the PR5 judge / RFC §6 re-execution; evidence is bounded (500-char command, 1000-char tail) and truncation degrades to UNVERIFIED rather than proving. **Acceptance checklist path portability**: Paths are host-independent and raw `..` fails closed. Drive/UNC absolutes retain their class across `ntpath` normalization and cannot cross the root; drive-relative, shell-dependent provider/PSDrive, and POSIX-rooted `cd` on Windows are unprovable. Selection overlap separates pytest node IDs, normalizes safe `.`/duplicate separators, rejects `..`, and applies Windows casing to drive, UNC, PSDrive, or Windows-context paths; provider-qualified drive/UNC roots remain paths until a later node-ID split. POSIX paths and all node IDs keep their required case sensitivity. Ambiguous trailing-dot/space or 8.3 components, different volume IDs, cross-family pairs, and absolute/relative PSDrive pairs fail closed without execution/filesystem provenance; only same-volume, same-root-form paths compare lexically. Without shell provenance, raw commands reject backslashes; cmd `%VAR%`/`!VAR!`, `^`, `#`; Bash braces/tilde; and PowerShell splatting, typographic quotes, or unquoted parentheses. A lone `%` remains eligible. POSIX-only markers must short-circuit on Windows because pytest evaluates them at import. **JVM test evidence**: Build-success banners do not establish `tests_passed`. diff --git a/backend/packages/harness/deerflow/subagents/acceptance_checks.py b/backend/packages/harness/deerflow/subagents/acceptance_checks.py index 1be395afba..927e703780 100644 --- a/backend/packages/harness/deerflow/subagents/acceptance_checks.py +++ b/backend/packages/harness/deerflow/subagents/acceptance_checks.py @@ -152,6 +152,27 @@ _TEST_ZERO_SHAPE_RE = re.compile(r"\b0\s+passed\b|\[no test files\]|\[no tests t _GO_ZERO_TEST_LINE_RE = re.compile(r"^(?:ok|\?)[ \t]+\S+[^\r\n]*\[(?:no test files|no tests to run)\][ \t]*\r?$", re.MULTILINE) +# ``go test -v`` is the only shape that records what actually happened inside a +# package: one ``--- PASS|FAIL|SKIP:`` line per case. +_GO_TEST_RESULT_LINE_RE = re.compile(r"^[ \t]*--- (PASS|FAIL|SKIP):", re.MULTILINE) + + +def _go_package_summary_is_evidence(text: str) -> bool: + """Whether a Go ``ok`` package line can carry positive test evidence. + + A package whose cases are all skipped still exits 0 and prints the same + ``ok`` banner a passing package does, so under ``-v`` the per-case lines are + the only evidence separating an executed test from a declined one — the bar + unittest's ``OK (skipped=N)`` (#6916) and Cargo's ignored-only targets (#6891) + already enforce. Records without per-case lines (plain ``go test``) stay + judged by the banner alone: Go's non-verbose output for an all-skipped + package is byte-identical to a passing one, so nothing in the record can + separate them and inventing a veto there would fail real passes. + """ + results = _GO_TEST_RESULT_LINE_RE.findall(text) + return not results or "PASS" in results + + # Cargo emits separate summaries for unit, integration and documentation # targets. Only complete successful records may qualify an empty sibling; # a bare success banner or clipped counters cannot erase zero-test evidence. @@ -2043,7 +2064,8 @@ def _check_tests_passed_leaf(command: str, bash_executions: list[dict[str, Any]] return base # Empty packages/targets cannot veto complete positive sibling evidence. summary_tail = _without_empty_cargo_targets(_GO_ZERO_TEST_LINE_RE.sub("", output_tail)) - passing = _TEST_PASS_SHAPE_RE.search(summary_tail) or _GO_TEST_PASS_SHAPE_RE.search(summary_tail) or _unittest_test_summary_holds(summary_tail) or build_summary is True + go_ok = _GO_TEST_PASS_SHAPE_RE.search(summary_tail) and _go_package_summary_is_evidence(summary_tail) + passing = _TEST_PASS_SHAPE_RE.search(summary_tail) or go_ok or _unittest_test_summary_holds(summary_tail) or build_summary is True if passing and not _TEST_ZERO_SHAPE_RE.search(summary_tail): base["checked"] = True base["holds"] = True diff --git a/backend/tests/test_go_acceptance.py b/backend/tests/test_go_acceptance.py new file mode 100644 index 0000000000..c4845dbf00 --- /dev/null +++ b/backend/tests/test_go_acceptance.py @@ -0,0 +1,108 @@ +"""Go's all-skipped verbose package cannot stand in for an executed test. + +Outputs below are verbatim captures from ``go test`` (go1.27.1, darwin/arm64) +against a module whose tests call ``t.Skip``. +""" + +import pytest + +from deerflow.subagents.acceptance_checks import check_acceptance_criteria, render_acceptance_section + +ALL_SKIPPED = """\ +=== RUN TestAlpha + skipper_test.go:5: needs network +--- SKIP: TestAlpha (0.00s) +=== RUN TestBeta + skipper_test.go:6: needs credentials +--- SKIP: TestBeta (0.00s) +PASS +ok \texample.com/acceptancerepro\t1.051s +""" + +MIXED_PASS_AND_SKIP = """\ +=== RUN TestGamma +--- PASS: TestGamma (0.00s) +=== RUN TestDelta +=== RUN TestDelta/inner + mixed_test.go:8: nope +--- PASS: TestDelta (0.00s) + --- SKIP: TestDelta/inner (0.00s) +PASS +ok \texample.com/acceptancerepro\t0.589s +""" + +NON_VERBOSE_OK = "ok \texample.com/acceptancerepro\t0.124s\n" + + +def _verdict(output, **overrides): + execution = {"command": "go test -v ./...", "output_tail": output, "status": "success", "shell_persistent": False, **overrides} + return check_acceptance_criteria(["tests_passed:go test -v ./..."], bash_executions=[execution]) + + +@pytest.mark.parametrize("output", [ALL_SKIPPED, ALL_SKIPPED.replace("\n", "\r\n")]) +def test_all_skipped_verbose_package_remains_unverified(output): + verdict = _verdict(output) + assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False) + assert "[UNVERIFIED]" in render_acceptance_section(verdict) + + +@pytest.mark.parametrize("output", [MIXED_PASS_AND_SKIP, NON_VERBOSE_OK]) +def test_executed_or_indistinguishable_package_still_supplies_evidence(output): + verdict = _verdict(output) + assert verdict["all_hold"] is True + assert verdict["unchecked"] == [] + + +def test_skipped_package_does_not_veto_a_passing_sibling_package(): + output = ALL_SKIPPED + MIXED_PASS_AND_SKIP + verdict = _verdict(output) + assert verdict["all_hold"] is True + + +def test_passing_package_followed_by_an_all_skipped_one_still_holds(): + verdict = _verdict(MIXED_PASS_AND_SKIP + ALL_SKIPPED) + assert verdict["all_hold"] is True + + +@pytest.mark.parametrize( + "output", + [ + "ok \texample.com/a\t0.001s [no test files]\n", + "ok \texample.com/a\t0.001s [no tests to run]\n", + "? \texample.com/a\t[no test files]\n", + ALL_SKIPPED + "ok \texample.com/a\t0.001s [no test files]\n", + ], +) +def test_zero_test_packages_stay_unverified(output): + verdict = _verdict(output) + assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False) + + +@pytest.mark.parametrize( + "output", + [ + "--- FAIL: TestAlpha (0.00s)\nFAIL\nFAIL\texample.com/acceptancerepro\t0.100s\n", + ALL_SKIPPED + "--- FAIL: TestGamma (0.00s)\nFAIL\nFAIL\texample.com/other\t0.100s\n", + ], +) +def test_failure_beats_the_skip_rule(output): + verdict = _verdict(output) + assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (True, False) + + +def test_skipped_package_cannot_bypass_output_attribution(): + verdict = _verdict(ALL_SKIPPED, command="echo 'go test -v ./...'; true", output_tail="") + assert verdict["leaves"][0]["checked"] is False + + +@pytest.mark.parametrize("banner", ["ok \texample.com/a\t(cached)\n", "ok \texample.com/a\t0.01s\n"]) +def test_cached_and_plain_banners_are_unchanged(banner): + verdict = _verdict(banner) + assert verdict["all_hold"] is True + + +def test_evidence_cap_truncation_keeps_indented_sub_test_results(): + # The harness stores an output *tail*, so a record can begin inside a + # sub-test block; the indented line is then all the record says. + verdict = _verdict(" --- SKIP: TestDelta/inner (0.00s)\nPASS\nok \texample.com/acceptancerepro\t0.589s\n") + assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)