fix(acceptance): reject all-skipped go test records as evidence (#6957)

A `go test -v` run whose every case is skipped exits 0 and prints the same
`ok` package banner a passing run does, so the banner alone supplied positive
evidence for `tests_passed:` while no test executed. unittest (#6916) and Cargo
(#6891) already enforce the non-skipped bar; Go's per-case `--- SKIP` lines are
the only record content that separates the two here.

Non-verbose `go test` output is byte-identical for an all-skipped and a passing
package, so those records stay judged by the banner alone.
This commit is contained in:
Dragon ze 2026-10-16 09:10:35 +08:00 committed by GitHub
parent 5633320a56
commit fd61c48fa2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 133 additions and 3 deletions

View File

@ -2173,7 +2173,7 @@ result text does not turn an unknown status into success or failure.
Cancelled or timed-out background sub-agent executions retain provider-reported token usage from completed model calls, including responses received before their next progress update. Final usage delivery to the parent run does not count earlier progress snapshots twice.
For `tests_passed:go test ./...`, packages marked `[no test files]` or `[no tests to run]` do not veto a passing summary from another package. Runs with only zero-test package summaries remain `UNVERIFIED`; failures still take precedence. This checks recorded execution evidence, not claim correctness.
For `tests_passed:go test ./...`, packages marked `[no test files]` or `[no tests to run]` do not veto a passing summary from another package. Runs with only zero-test package summaries remain `UNVERIFIED`; failures still take precedence. Under `-v`, a record whose per-case lines are all `--- SKIP` also remains `UNVERIFIED`: Go exits 0 and prints the same `ok` banner, so nothing ran to pass. Non-verbose output cannot separate an all-skipped package from a passing one and stays trusted as recorded. This checks recorded execution evidence, not claim correctness.
For `tests_passed:cargo test`, a complete passing target summary can coexist with
empty unit, integration or documentation targets. Complete zero-pass successful

File diff suppressed because one or more lines are too long

View File

@ -152,6 +152,27 @@ _TEST_ZERO_SHAPE_RE = re.compile(r"\b0\s+passed\b|\[no test files\]|\[no tests t
_GO_ZERO_TEST_LINE_RE = re.compile(r"^(?:ok|\?)[ \t]+\S+[^\r\n]*\[(?:no test files|no tests to run)\][ \t]*\r?$", re.MULTILINE)
# ``go test -v`` is the only shape that records what actually happened inside a
# package: one ``--- PASS|FAIL|SKIP:`` line per case.
_GO_TEST_RESULT_LINE_RE = re.compile(r"^[ \t]*--- (PASS|FAIL|SKIP):", re.MULTILINE)
def _go_package_summary_is_evidence(text: str) -> bool:
"""Whether a Go ``ok`` package line can carry positive test evidence.
A package whose cases are all skipped still exits 0 and prints the same
``ok`` banner a passing package does, so under ``-v`` the per-case lines are
the only evidence separating an executed test from a declined one — the bar
unittest's ``OK (skipped=N)`` (#6916) and Cargo's ignored-only targets (#6891)
already enforce. Records without per-case lines (plain ``go test``) stay
judged by the banner alone: Go's non-verbose output for an all-skipped
package is byte-identical to a passing one, so nothing in the record can
separate them and inventing a veto there would fail real passes.
"""
results = _GO_TEST_RESULT_LINE_RE.findall(text)
return not results or "PASS" in results
# Cargo emits separate summaries for unit, integration and documentation
# targets. Only complete successful records may qualify an empty sibling;
# a bare success banner or clipped counters cannot erase zero-test evidence.
@ -2043,7 +2064,8 @@ def _check_tests_passed_leaf(command: str, bash_executions: list[dict[str, Any]]
return base
# Empty packages/targets cannot veto complete positive sibling evidence.
summary_tail = _without_empty_cargo_targets(_GO_ZERO_TEST_LINE_RE.sub("", output_tail))
passing = _TEST_PASS_SHAPE_RE.search(summary_tail) or _GO_TEST_PASS_SHAPE_RE.search(summary_tail) or _unittest_test_summary_holds(summary_tail) or build_summary is True
go_ok = _GO_TEST_PASS_SHAPE_RE.search(summary_tail) and _go_package_summary_is_evidence(summary_tail)
passing = _TEST_PASS_SHAPE_RE.search(summary_tail) or go_ok or _unittest_test_summary_holds(summary_tail) or build_summary is True
if passing and not _TEST_ZERO_SHAPE_RE.search(summary_tail):
base["checked"] = True
base["holds"] = True

View File

@ -0,0 +1,108 @@
"""Go's all-skipped verbose package cannot stand in for an executed test.
Outputs below are verbatim captures from ``go test`` (go1.27.1, darwin/arm64)
against a module whose tests call ``t.Skip``.
"""
import pytest
from deerflow.subagents.acceptance_checks import check_acceptance_criteria, render_acceptance_section
ALL_SKIPPED = """\
=== RUN TestAlpha
skipper_test.go:5: needs network
--- SKIP: TestAlpha (0.00s)
=== RUN TestBeta
skipper_test.go:6: needs credentials
--- SKIP: TestBeta (0.00s)
PASS
ok \texample.com/acceptancerepro\t1.051s
"""
MIXED_PASS_AND_SKIP = """\
=== RUN TestGamma
--- PASS: TestGamma (0.00s)
=== RUN TestDelta
=== RUN TestDelta/inner
mixed_test.go:8: nope
--- PASS: TestDelta (0.00s)
--- SKIP: TestDelta/inner (0.00s)
PASS
ok \texample.com/acceptancerepro\t0.589s
"""
NON_VERBOSE_OK = "ok \texample.com/acceptancerepro\t0.124s\n"
def _verdict(output, **overrides):
execution = {"command": "go test -v ./...", "output_tail": output, "status": "success", "shell_persistent": False, **overrides}
return check_acceptance_criteria(["tests_passed:go test -v ./..."], bash_executions=[execution])
@pytest.mark.parametrize("output", [ALL_SKIPPED, ALL_SKIPPED.replace("\n", "\r\n")])
def test_all_skipped_verbose_package_remains_unverified(output):
verdict = _verdict(output)
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)
assert "[UNVERIFIED]" in render_acceptance_section(verdict)
@pytest.mark.parametrize("output", [MIXED_PASS_AND_SKIP, NON_VERBOSE_OK])
def test_executed_or_indistinguishable_package_still_supplies_evidence(output):
verdict = _verdict(output)
assert verdict["all_hold"] is True
assert verdict["unchecked"] == []
def test_skipped_package_does_not_veto_a_passing_sibling_package():
output = ALL_SKIPPED + MIXED_PASS_AND_SKIP
verdict = _verdict(output)
assert verdict["all_hold"] is True
def test_passing_package_followed_by_an_all_skipped_one_still_holds():
verdict = _verdict(MIXED_PASS_AND_SKIP + ALL_SKIPPED)
assert verdict["all_hold"] is True
@pytest.mark.parametrize(
"output",
[
"ok \texample.com/a\t0.001s [no test files]\n",
"ok \texample.com/a\t0.001s [no tests to run]\n",
"? \texample.com/a\t[no test files]\n",
ALL_SKIPPED + "ok \texample.com/a\t0.001s [no test files]\n",
],
)
def test_zero_test_packages_stay_unverified(output):
verdict = _verdict(output)
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)
@pytest.mark.parametrize(
"output",
[
"--- FAIL: TestAlpha (0.00s)\nFAIL\nFAIL\texample.com/acceptancerepro\t0.100s\n",
ALL_SKIPPED + "--- FAIL: TestGamma (0.00s)\nFAIL\nFAIL\texample.com/other\t0.100s\n",
],
)
def test_failure_beats_the_skip_rule(output):
verdict = _verdict(output)
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (True, False)
def test_skipped_package_cannot_bypass_output_attribution():
verdict = _verdict(ALL_SKIPPED, command="echo 'go test -v ./...'; true", output_tail="")
assert verdict["leaves"][0]["checked"] is False
@pytest.mark.parametrize("banner", ["ok \texample.com/a\t(cached)\n", "ok \texample.com/a\t0.01s\n"])
def test_cached_and_plain_banners_are_unchanged(banner):
verdict = _verdict(banner)
assert verdict["all_hold"] is True
def test_evidence_cap_truncation_keeps_indented_sub_test_results():
# The harness stores an output *tail*, so a record can begin inside a
# sub-test block; the indented line is then all the record says.
verdict = _verdict(" --- SKIP: TestDelta/inner (0.00s)\nPASS\nok \texample.com/acceptancerepro\t0.589s\n")
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)