"""Cargo's empty test targets must not invalidate completed sibling tests.""" import asyncio import importlib.util import os import shutil import sys from pathlib import Path from types import SimpleNamespace import pytest from langchain_core.messages import AIMessage from langchain_core.tools import StructuredTool from langgraph.graph import END, START, MessagesState, StateGraph from langgraph.prebuilt import ToolNode from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping from deerflow.subagents.acceptance_checks import check_acceptance_criteria, render_acceptance_section, render_acceptance_segment # Each parametrization compiles the fixture from a cold target directory, and a # loaded CI runner can spend far longer than the compile itself on toolchain # startup. 60s occasionally expired mid-compile, which surfaces as "Exit Code: # 124" instead of a test summary; the run is still bounded by the harness. CARGO_BUILD_TIMEOUT = 180 def _summary(passed=0, failed=0, ignored=0): return f"test result: {'FAILED' if failed else 'ok'}. {passed} passed; {failed} failed; {ignored} ignored; 0 measured; 0 filtered out; finished in 0.00s" def _verdict(output, **overrides): execution = {"command": "cargo test", "output_tail": output, "status": "success", "shell_persistent": False, **overrides} return check_acceptance_criteria(["tests_passed:cargo test"], bash_executions=[execution]) @pytest.mark.parametrize("reverse", [False, True]) @pytest.mark.parametrize("ignored", [0, 2]) @pytest.mark.parametrize("newline", ["\n", "\r\n"]) def test_successful_tests_survive_complete_empty_target_summaries(reverse, ignored, newline): records = [_summary(passed=2), _summary(ignored=ignored)] if reverse: records.reverse() verdict = _verdict(newline.join(records)) assert verdict["all_hold"] is True assert verdict["unchecked"] == [] assert "[holds]" in render_acceptance_section(verdict) assert "execution evidence only" in render_acceptance_section(verdict) assert "acceptance:" in render_acceptance_segment(verdict) @pytest.mark.parametrize("output", [_summary(), _summary(ignored=3), _summary() + "\n" + _summary(ignored=1)]) def test_only_empty_or_ignored_targets_remain_unverified(output): verdict = _verdict(output) assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False) assert "[UNVERIFIED]" in render_acceptance_section(verdict) @pytest.mark.parametrize("output", ["test result: ok", "test result: ok. 0", _summary() + "\ntest result: ok"]) def test_bare_cargo_success_banner_cannot_supply_test_count_evidence(output): verdict = _verdict(output) assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False) @pytest.mark.parametrize("banner", ["OK", "all tests passed", "test result: ok. 2 passed; 0 failed"]) def test_zero_target_records_need_a_complete_positive_cargo_sibling(banner): verdict = _verdict(_summary() + "\n" + banner) assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False) def test_positive_counts_do_not_bypass_output_attribution(): verdict = _verdict(_summary(passed=2) + "\n" + _summary(), command="echo 'unrelated'; cargo test") assert verdict["leaves"][0]["checked"] is False @pytest.mark.parametrize( "zero", [ _summary().split("; 0 filtered out")[0], "warning: " + _summary(), _summary().replace("finished in 0.00s", "finished in"), _summary().replace("0 ignored", "9" * 100 + " ignored"), "0 passed in 0.1s", "Ran 0 tests\nOK", ], ) def test_incomplete_or_other_runner_zero_evidence_is_not_silenced(zero): verdict = _verdict(_summary(passed=2) + "\n" + zero) assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False) @pytest.mark.parametrize("failure", [_summary(failed=1), _summary(failed=1).replace("FAILED", "ok"), "error: doctest failed"]) @pytest.mark.parametrize("reverse", [False, True]) def test_failure_in_a_sibling_target_still_wins(failure, reverse): # Cargo normally exits nonzero; the terminal FAILED/count summary must # also prevent a positive verdict if a surrounding runner swallows it. records = [_summary(passed=2), _summary(), failure] output = "\n".join(reversed(records) if reverse else records) if failure.startswith("error:"): verdict = _verdict(output, status="error") else: verdict = _verdict(output) assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (True, False) assert "[does not hold]" in render_acceptance_section(verdict) @pytest.mark.parametrize("overrides", [{"shell_persistent": True}, {"shell_persistent": None}, {"command_truncated": True}]) def test_empty_target_handling_does_not_bypass_execution_provenance(overrides): verdict = _verdict(_summary(passed=2) + "\n" + _summary(), **overrides) assert verdict["leaves"][0]["checked"] is False @pytest.fixture def executor_module(monkeypatch): # conftest preloads an executor stub; harvest from the production module. path = Path(__file__).parents[1] / "packages/harness/deerflow/subagents/executor.py" spec = importlib.util.spec_from_file_location("_cargo_evidence_executor", path) module = importlib.util.module_from_spec(spec) monkeypatch.setitem(sys.modules, spec.name, module) spec.loader.exec_module(module) return module class SandboxMessages(MessagesState): sandbox: dict @pytest.mark.skipif(os.name == "nt" or shutil.which("cargo") is None, reason="POSIX Cargo integration requires an installed Rust toolchain") @pytest.mark.asyncio @pytest.mark.parametrize( "source,selection,expected", [ ("#[test] fn unit() { assert_eq!(2 + 2, 4); }", "", (True, True)), ("/// ```\n/// assert_eq!(2 + 2, 4);\n/// ```\npub fn documented() {}", "", (True, True)), ("#[test] #[ignore] fn ignored() {}", "", (False, False)), ('#[test] fn broken() { panic!("broken"); }', "", (True, False)), ("/// ```\n/// assert_eq!(1, 2);\n/// ```\npub fn documented() {}\n#[test] fn unit() {}", "", (True, False)), ("#[test] fn unit() {}", " no_test_matches", (False, False)), ], ids=["unit-with-empty-doc", "doc-with-empty-unit", "ignored", "failed-unit", "failed-doc", "filtered"], ) async def test_native_cargo_output_reaches_parent_acceptance_through_tool_node(tmp_path, monkeypatch, executor_module, source, selection, expected): # LocalSandbox starts an independent subprocess without inheriting make's # jobserver FDs. Stale flags produce long Cargo/rustdoc warnings that can # displace the test summaries from the intentionally bounded evidence tail. for variable in ("MAKEFLAGS", "MFLAGS", "CARGO_MAKEFLAGS"): monkeypatch.delenv(variable, raising=False) (tmp_path / "src").mkdir() (tmp_path / "Cargo.toml").write_text('[package]\nname = "acceptance_fixture"\nversion = "0.1.0"\nedition = "2021"\n', encoding="utf-8") (tmp_path / "src/lib.rs").write_text(source, encoding="utf-8") sandbox = LocalSandbox("cargo-evidence", path_mappings=[PathMapping("/mnt/user-data/workspace", str(tmp_path))], environment={"CARGO_TERM_COLOR": "never"}) monkeypatch.setattr("deerflow.sandbox.sandbox_provider.get_sandbox_provider", lambda: SimpleNamespace(get=lambda sandbox_id: sandbox if sandbox_id == sandbox.id else None)) outputs = [] async def bash(command: str) -> str: output = await asyncio.to_thread(sandbox.execute_command, command, timeout=CARGO_BUILD_TIMEOUT) outputs.append(output) return output tool = StructuredTool.from_function(coroutine=bash, name="bash", description="Execute the offline Rust fixture.") graph = StateGraph(SandboxMessages) graph.add_node("tools", ToolNode([tool], awrap_tool_call=ToolErrorHandlingMiddleware().awrap_tool_call)) graph.add_edge(START, "tools") graph.add_edge("tools", END) command = "cargo test --offline --manifest-path /mnt/user-data/workspace/Cargo.toml" + selection state = await graph.compile().ainvoke( { "sandbox": {"sandbox_id": sandbox.id}, "messages": [AIMessage(content="", tool_calls=[{"name": "bash", "id": "cargo-call", "type": "tool_call", "args": {"command": command}}])], } ) assert len(outputs) == 1 assert "test result:" in outputs[0], outputs[0] message = state["messages"][-1] assert (message.name, message.tool_call_id) == ("bash", "cargo-call") executions = executor_module._harvest_bash_executions(state) assert len(executions) == 1 assert executions[0]["shell_persistent"] is False assert executions[0]["tool_call_id"] == "cargo-call" verdict = check_acceptance_criteria([f"tests_passed:{command}"], bash_executions=executions) assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected, verdict if expected == (True, True): assert "1 passed" in outputs[0] and "0 passed" in outputs[0] assert "[holds]" in render_acceptance_section(verdict) elif expected == (True, False): assert "Exit Code: 101" in outputs[0] assert executions[0]["status"] == "error" else: assert "[UNVERIFIED]" in render_acceptance_section(verdict)