mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 21:26:39 +00:00
8f0ca58653
* fix(subagents): require test evidence beyond build success * test(subagents): isolate test evidence from sandbox file readers
114 lines
4.7 KiB
Python
114 lines
4.7 KiB
Python
"""Build completion is not proof that delegated tests ran successfully."""
|
|
|
|
import pytest
|
|
|
|
from deerflow.subagents.acceptance_checks import check_acceptance_criteria, render_acceptance_section
|
|
|
|
|
|
def _check(output: str, command: str = "mvn test", **overrides):
|
|
execution = {
|
|
"tool_call_id": "test-build",
|
|
"tool_name": "bash",
|
|
"command": command,
|
|
"output_tail": output,
|
|
"status": "success",
|
|
"shell_persistent": False,
|
|
**overrides,
|
|
}
|
|
return check_acceptance_criteria([f"tests_passed:{command}"], bash_executions=[execution])
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"output,command",
|
|
[
|
|
("[INFO] BUILD SUCCESS", "mvn test"),
|
|
("[INFO] Tests are skipped.\n[INFO] BUILD SUCCESS", "mvn test -DskipTests"),
|
|
("[INFO] Tests run: 0, Failures: 0, Errors: 0, Skipped: 0\n[INFO] BUILD SUCCESS", "mvn test"),
|
|
("[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 3\n[INFO] BUILD SUCCESS", "mvn test"),
|
|
("> Task :test NO-SOURCE\nBUILD SUCCESSFUL in 1s", "gradle test"),
|
|
("> Task :test SKIPPED\nBUILD SUCCESSFUL in 1s", "gradle test"),
|
|
("0 tests completed\nBUILD SUCCESSFUL", "gradle test"),
|
|
("3 tests completed, 3 skipped\nBUILD SUCCESSFUL", "gradle test"),
|
|
],
|
|
)
|
|
def test_build_without_executed_tests_remains_unverified(output, command):
|
|
verdict = _check(output, command)
|
|
leaf = verdict["leaves"][0]
|
|
assert (leaf["checked"], leaf["holds"]) == (False, False)
|
|
assert verdict["all_hold"] is False
|
|
assert verdict["unchecked"] == [f"tests_passed:{command}"]
|
|
assert "[UNVERIFIED]" in render_acceptance_section(verdict)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"output,command",
|
|
[
|
|
("[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 0", "mvn test"),
|
|
("[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 1, Time elapsed: 0.2 s -- in TestExample", "mvn test"),
|
|
("Tests run: 1, Failures: 0, Errors: 0, Skipped: 0\r\n[INFO] BUILD SUCCESS", "mvn test"),
|
|
("3 tests completed, 0 failed, 1 skipped\nBUILD SUCCESSFUL", "gradle test"),
|
|
("3 tests completed\nBUILD SUCCESSFUL", "gradle test"),
|
|
],
|
|
)
|
|
def test_nonzero_test_counts_are_positive_evidence(output, command):
|
|
verdict = _check(output, command)
|
|
assert verdict["all_hold"] is True
|
|
assert "[holds]" in render_acceptance_section(verdict)
|
|
|
|
|
|
@pytest.mark.parametrize("kind", ["Failures", "Errors"])
|
|
def test_maven_failure_counts_override_ignored_failure_build_success(kind):
|
|
counts = {"Failures": 0, "Errors": 0, kind: 1}
|
|
output = f"[INFO] Tests run: 3, Failures: {counts['Failures']}, Errors: {counts['Errors']}, Skipped: 0\n[INFO] BUILD SUCCESS"
|
|
verdict = _check(output, "mvn test -Dmaven.test.failure.ignore=true")
|
|
assert (verdict["leaves"][0]["checked"], verdict["leaves"][0]["holds"]) == (True, False)
|
|
assert "[does not hold]" in render_acceptance_section(verdict)
|
|
|
|
|
|
@pytest.mark.parametrize("reverse", [False, True])
|
|
def test_empty_maven_module_does_not_veto_a_tested_module(reverse):
|
|
summaries = [
|
|
"[INFO] Tests run: 0, Failures: 0, Errors: 0, Skipped: 0",
|
|
"[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 1",
|
|
]
|
|
if reverse:
|
|
summaries.reverse()
|
|
assert _check("\n".join(summaries))["all_hold"] is True
|
|
|
|
|
|
def test_failing_module_overrides_a_passing_module():
|
|
output = "[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 0\n[ERROR] Tests run: 2, Failures: 1, Errors: 0, Skipped: 0"
|
|
leaf = _check(output)["leaves"][0]
|
|
assert (leaf["checked"], leaf["holds"]) == (True, False)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"summary",
|
|
[
|
|
"Tests run: 1, Failures: 0, Errors: 0, Skipped: 2",
|
|
"Tests run: -1, Failures: 0, Errors: 0, Skipped: 0",
|
|
"Tests run: 3, Failures: 0, Errors:",
|
|
"Tests run: three, Failures: 0, Errors: 0, Skipped: 0",
|
|
"3 tests completed, 4 skipped",
|
|
],
|
|
)
|
|
def test_inconsistent_or_incomplete_count_summary_is_not_proof(summary):
|
|
leaf = _check(summary + "\nBUILD SUCCESS")["leaves"][0]
|
|
assert (leaf["checked"], leaf["holds"]) == (False, False)
|
|
|
|
|
|
@pytest.mark.parametrize("persistent", [None, True])
|
|
def test_count_summary_does_not_bypass_shell_provenance(persistent):
|
|
verdict = _check("Tests run: 3, Failures: 0, Errors: 0, Skipped: 0", shell_persistent=persistent)
|
|
assert verdict["leaves"][0]["checked"] is False
|
|
|
|
|
|
def test_count_summary_does_not_bypass_output_attribution():
|
|
verdict = _check("Tests run: 3, Failures: 0, Errors: 0, Skipped: 0", "echo 'unrelated'; mvn test")
|
|
assert verdict["leaves"][0]["checked"] is False
|
|
|
|
|
|
def test_recorded_failure_status_still_overrides_positive_counts():
|
|
leaf = _check("Tests run: 3, Failures: 0, Errors: 0, Skipped: 0", status="error")["leaves"][0]
|
|
assert (leaf["checked"], leaf["holds"]) == (True, False)
|