mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-12 14:00:35 +00:00
3e59e22b4c
* fix(subagents): keep tests_passed evidence per call when a provider reuses its id Tool-call ids are unique per response, not per conversation. Some providers number calls per response, so two turns can both issue call_0. The bash evidence for tests_passed acceptance assumed one call per id in two places: - _harvest_bash_executions built one id -> command map over the whole conversation, so every result under a reused id was labelled with the last call's command. A passing subset run (pytest tests/unit) became evidence for the full suite when a later call_0 ran pytest tests and had no result yet. - update_bash_executions merged values chunks by tool_call_id. A dict keeps a key's first position, so a later run under a reused id overwrote the earlier entry in place. With call_0 failing, call_1 passing and call_0 failing again, the merged list ended on the stale pass and the leaf held. Pair results through pair_tool_call_results, which answers a call only from its own response, and merge chunks by result message id, keeping the call id for entries without one. Each run stays a separate entry in execution order, so the latest run decides the leaf again. * fix(subagents): key the summary cache per run and document id-less calls - Cache full-output test summaries by result message id (call id only without one), so reused call ids keep one cache slot per run. - Update the bash_executions field doc and the subagents guide to the result-id merge, and state that calls without an id record no evidence (the leaf stays UNVERIFIED). - Tests pin both: one classification per run under a reused id, and no evidence for a call with a missing or empty id. * chore(subagents): drop the stray empty deerflow/executor.py The previous commit added a 0-byte module at the package root by mistake. Nothing imports deerflow.executor; the real module is deerflow/subagents/executor.py.
408 lines
22 KiB
Python
408 lines
22 KiB
Python
"""Node's native reporters must prove executed tests, not TAP point labels."""
|
||
|
||
import asyncio
|
||
import importlib.util
|
||
import os
|
||
import shlex
|
||
import shutil
|
||
import sys
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
import pytest
|
||
from _pytest.terminal import format_session_duration
|
||
from langchain_core.messages import AIMessage, ToolMessage
|
||
from langchain_core.tools import StructuredTool
|
||
from langgraph.graph import END, START, MessagesState, StateGraph
|
||
from langgraph.prebuilt import ToolNode
|
||
|
||
from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware
|
||
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping
|
||
from deerflow.subagents.acceptance_checks import _carries_summary_shape, check_acceptance_criteria, render_acceptance_section
|
||
|
||
|
||
def _summary(prefix="#", *, passed=1, failed=0, cancelled=0, skipped=0, todo=0, total=None):
|
||
counters = {"tests": passed + failed + cancelled + skipped + todo if total is None else total, "suites": 0, "pass": passed, "fail": failed, "cancelled": cancelled, "skipped": skipped, "todo": todo}
|
||
return "\n".join(f"{prefix} {key} {value}" for key, value in counters.items()) + f"\n{prefix} duration_ms 12.5"
|
||
|
||
|
||
def _verdict(output, **overrides):
|
||
execution = {"command": "node --test", "status": "success", "shell_persistent": False, "output_tail": output, **overrides}
|
||
return check_acceptance_criteria(["tests_passed:node --test"], bash_executions=[execution])
|
||
|
||
|
||
@pytest.mark.parametrize("prefix", ["#", "ℹ"])
|
||
@pytest.mark.parametrize("newline", ["\n", "\r\n"])
|
||
def test_native_reporters_accept_nonempty_passes(prefix, newline):
|
||
verdict = _verdict(_summary(prefix, skipped=1, todo=1).replace("\n", newline))
|
||
assert verdict["all_hold"] is True
|
||
assert "[holds]" in render_acceptance_section(verdict)
|
||
|
||
|
||
@pytest.mark.parametrize("prefix", ["#", "ℹ"])
|
||
@pytest.mark.parametrize("counts", [{"failed": 1}, {"cancelled": 1}])
|
||
def test_native_failures_override_incidental_pass_text(prefix, counts):
|
||
verdict = _verdict("ok 1 - all tests passed\n" + _summary(prefix, **counts))
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (True, False)
|
||
|
||
|
||
@pytest.mark.parametrize("prefix", ["#", "ℹ"])
|
||
@pytest.mark.parametrize("counts", [{}, {"skipped": 2}, {"todo": 2}, {"skipped": 1, "todo": 1}])
|
||
def test_empty_skipped_and_todo_summaries_supply_no_proof(prefix, counts):
|
||
verdict = _verdict("ok 1 - all tests passed # SKIP\n" + _summary(prefix, passed=0, **counts))
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"output",
|
||
[
|
||
"ok 1 - sample",
|
||
"ok 12 # SKIP",
|
||
_summary(total=4),
|
||
_summary(passed="9" * 100, total=1),
|
||
_summary().replace("# fail 0\n", ""),
|
||
_summary().replace("# fail 0", "ℹ fail 0"),
|
||
_summary().replace("# pass 1", "# pass -1"),
|
||
_summary().split("# todo")[0],
|
||
],
|
||
)
|
||
def test_tap_points_and_incomplete_or_inconsistent_counts_remain_unverified(output):
|
||
assert _verdict(output)["leaves"][0]["checked"] is False
|
||
|
||
|
||
@pytest.mark.parametrize("reverse", [False, True])
|
||
def test_failing_summary_wins_over_successful_sibling(reverse):
|
||
summaries = [_summary(), _summary(failed=1)]
|
||
verdict = _verdict("\n".join(reversed(summaries) if reverse else summaries))
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (True, False)
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"overrides",
|
||
[{"shell_persistent": True}, {"shell_persistent": None}, {"command_truncated": True}, {"command": "node --test sample.test.cjs"}, {"command": "echo other; node --test"}],
|
||
)
|
||
def test_node_counts_do_not_bypass_execution_admission(overrides):
|
||
assert _verdict(_summary(), **overrides)["leaves"][0]["checked"] is False
|
||
|
||
|
||
def test_go_package_and_pytest_controls_still_pass():
|
||
assert _verdict("ok\texample.org/project\t0.005s")["all_hold"] is True
|
||
assert _verdict("ok\t123\t0.005s")["all_hold"] is True
|
||
assert _verdict("2 passed in 0.01s")["all_hold"] is True
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"point",
|
||
[
|
||
"ok 1 - all tests passed # SKIP",
|
||
"ok 12 # all tests passed",
|
||
"✔ all tests passed (1ms)",
|
||
"﹣ all tests passed (1ms)",
|
||
"▶ all tests passed",
|
||
"⚠ all tests passed",
|
||
" ok 1 - all tests passed # SKIP",
|
||
"# Subtest: all tests passed\nok 1 - all tests passed # SKIP",
|
||
],
|
||
)
|
||
@pytest.mark.parametrize("tail", ["", _summary(passed=0, skipped=1).split("# todo")[0]])
|
||
def test_point_labels_cannot_supply_pass_evidence(point, tail):
|
||
verdict = _verdict(point + "\n" + tail)
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)
|
||
|
||
|
||
def test_recorded_exit_error_overrides_node_positive_counts():
|
||
verdict = _verdict(_summary(), status="error")
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (True, False)
|
||
|
||
|
||
def test_newest_matching_execution_is_decisive():
|
||
first = {"command": "node --test", "status": "success", "shell_persistent": False, "output_tail": _summary(failed=1)}
|
||
second = {**first, "output_tail": _summary()}
|
||
assert check_acceptance_criteria(["tests_passed:node --test"], bash_executions=[first, second])["all_hold"] is True
|
||
assert check_acceptance_criteria(["tests_passed:node --test"], bash_executions=[second, first])["all_hold"] is False
|
||
|
||
|
||
def test_empty_node_sibling_does_not_veto_another_runner():
|
||
output = "2 passed in 0.01s\n" + _summary(passed=0)
|
||
execution = {"command": "make test", "status": "success", "shell_persistent": False, "output_tail": output}
|
||
assert check_acceptance_criteria(["tests_passed:make test"], bash_executions=[execution])["all_hold"] is True
|
||
|
||
|
||
@pytest.mark.parametrize("label", ["0 passed", "1 failed", "1 error"])
|
||
@pytest.mark.parametrize("point", ["# Subtest: {label}\n ok 1 - {label}", "⚠ {label} # TODO"])
|
||
def test_test_names_cannot_supply_failure_or_zero_evidence(label, point):
|
||
verdict = _verdict(point.format(label=label) + "\n" + _summary())
|
||
assert verdict["all_hold"] is True
|
||
|
||
|
||
@pytest.mark.parametrize("newline", ["\n", "\r\n"])
|
||
def test_todo_diagnostic_location_cannot_supply_error_count(newline):
|
||
output = "⚠ 1 failed # TODO\n" + _summary("ℹ", todo=1) + "\ntest at sample.test.cjs:1:57\n\n Error: fixture"
|
||
assert _verdict(output.replace("\n", newline))["all_hold"] is True
|
||
|
||
|
||
def test_color_codes_are_normalized_for_counts_and_output_admission():
|
||
output = "\x1b[90m﹣ all tests passed\x1b[39m\n" + "\n".join(f"\x1b[34m{line}\x1b[39m" for line in _summary("ℹ", passed=0, skipped=1).splitlines())
|
||
assert _verdict(output)["leaves"][0]["checked"] is False
|
||
assert _carries_summary_shape(output) is True
|
||
|
||
|
||
@pytest.fixture
|
||
def executor_module(monkeypatch):
|
||
path = Path(__file__).parents[1] / "packages/harness/deerflow/subagents/executor.py"
|
||
spec = importlib.util.spec_from_file_location("_node_evidence_executor", path)
|
||
module = importlib.util.module_from_spec(spec)
|
||
monkeypatch.setitem(sys.modules, spec.name, module)
|
||
spec.loader.exec_module(module)
|
||
return module
|
||
|
||
|
||
class SandboxMessages(MessagesState):
|
||
sandbox: dict
|
||
|
||
|
||
@pytest.mark.parametrize("label", ["all tests passed", "1 failed", "0 passed"])
|
||
def test_harvest_drops_a_partial_label_before_interpreting_counts(executor_module, monkeypatch, label):
|
||
output = "ok 1 - " + "x" * 1500 + " " + label + " # SKIP\n" + _summary(passed=0, skipped=1)
|
||
messages = [AIMessage(content="", tool_calls=[{"name": "bash", "id": "long-name", "args": {"command": "node --test"}}]), ToolMessage(content=output, tool_call_id="long-name")]
|
||
monkeypatch.setattr(executor_module, "_harvest_shell_persistence", lambda state: False)
|
||
executions = executor_module._harvest_bash_executions({"messages": messages})
|
||
assert executions[0]["output_tail"] == _summary(passed=0, skipped=1)
|
||
assert messages[-1].content == output
|
||
verdict = check_acceptance_criteria(["tests_passed:node --test"], bash_executions=executions)
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"output,expected",
|
||
[
|
||
(_summary("ℹ", todo=1) + "\n" + " at fixture\n" * 150, (True, True)),
|
||
(_summary("ℹ", failed=1) + "\n" + " at fixture\n" * 150, (True, False)),
|
||
(_summary("ℹ", passed=0, todo=1) + "\n" + " at fixture\n" * 150, (False, False)),
|
||
(_summary("ℹ", failed=1) + "\n" + "diagnostic\n" * 150 + _summary("ℹ"), (True, False)),
|
||
("0 passed\n" + "diagnostic\n" * 150 + "2 passed", (False, False)),
|
||
],
|
||
ids=["pass-before-stack", "failure-before-stack", "all-todo-before-stack", "failure-before-passing-tail", "zero-before-passing-tail"],
|
||
)
|
||
def test_full_summary_survives_bounded_harvest(executor_module, monkeypatch, output, expected):
|
||
messages = [AIMessage(content="", tool_calls=[{"name": "bash", "id": "node-call", "args": {"command": "node --test"}}]), ToolMessage(content=output, tool_call_id="node-call")]
|
||
monkeypatch.setattr(executor_module, "_harvest_shell_persistence", lambda state: False)
|
||
executions = executor_module._harvest_bash_executions({"messages": messages})
|
||
assert len(executions[0]["output_tail"]) <= 1000
|
||
verdict = check_acceptance_criteria(["tests_passed:node --test"], bash_executions=executions)
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected
|
||
|
||
|
||
@pytest.mark.parametrize("overrides", [{"shell_persistent": True}, {"shell_persistent": None}, {"command_truncated": True}, {"command": "echo other; node --test"}, {"command": "node --test > log"}])
|
||
def test_full_summary_does_not_bypass_admission(overrides):
|
||
assert _verdict("", test_summary="passed", **overrides)["leaves"][0]["checked"] is False
|
||
|
||
|
||
@pytest.mark.skipif(os.name == "nt" or shutil.which("node") is None, reason="Requires POSIX LocalSandbox and Node.js")
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.parametrize("reporter", ["tap", "spec", "color-spec"])
|
||
@pytest.mark.parametrize(
|
||
"body,expected",
|
||
[
|
||
("test('sample', () => {});", (True, True)),
|
||
("test('0 passed', () => {}); test('1 failed', () => {});", (True, True)),
|
||
("test('sample', () => {}); test.skip('optional', () => {});", (True, True)),
|
||
("test.skip('all tests passed', () => {});", (False, False)),
|
||
("test.skip('x'.repeat(1500) + ' all tests passed', () => {});", (False, False)),
|
||
("test('x'.repeat(1500) + ' 1 failed', () => {});", (True, True)),
|
||
("test('x'.repeat(1500) + ' 0 passed', () => {});", (True, True)),
|
||
("test.todo('pending');", (False, False)),
|
||
("test('all tests passed', { todo: true }, () => { throw Error('fixture'); });", (False, False)),
|
||
("test('sample', () => {}); test('1 failed', { todo: true }, () => { throw Error('fixture'); });", (True, True)),
|
||
("describe('empty suite', () => {});", (False, False)),
|
||
("describe('all tests passed', () => { test.skip('optional', () => {}); });", (False, False)),
|
||
("describe('all tests passed', () => { test.todo('pending'); });", (False, False)),
|
||
("describe('1 failed', () => { test('sample', () => {}); });", (True, True)),
|
||
("test('sample', () => { throw Error('fixture'); });", (True, False)),
|
||
("test('pending', { timeout: 20 }, () => new Promise(() => {}));", (True, False)),
|
||
("console.log('12 passed'); for (let i=0; i<250; i++) test.skip('t'+i,()=>{});", (False, False)),
|
||
("console.log('validation found 2 errors'); for (let i=0; i<250; i++) test('t'+i,()=>{});", (True, True)),
|
||
],
|
||
ids=[
|
||
"pass",
|
||
"summary-like-names",
|
||
"pass-and-skip",
|
||
"all-skipped",
|
||
"long-skipped",
|
||
"long-failure-name",
|
||
"long-zero-name",
|
||
"all-todo",
|
||
"failing-todo-pass-name",
|
||
"failing-todo-failure-name",
|
||
"empty-suite",
|
||
"skipped-suite",
|
||
"todo-suite",
|
||
"summary-like-suite",
|
||
"failure",
|
||
"cancelled",
|
||
"stray-pass-all-skipped",
|
||
"stray-error-all-passing",
|
||
],
|
||
)
|
||
async def test_real_node_results_reach_parent_acceptance(tmp_path, monkeypatch, executor_module, reporter, body, expected):
|
||
(tmp_path / "sample.test.cjs").write_text("const { test, describe } = require('node:test');\n" + body, encoding="utf-8")
|
||
monkeypatch.delenv("NO_COLOR", raising=False)
|
||
sandbox = LocalSandbox("node-evidence", path_mappings=[PathMapping("/mnt/user-data/workspace", str(tmp_path))])
|
||
monkeypatch.setattr("deerflow.sandbox.sandbox_provider.get_sandbox_provider", lambda: SimpleNamespace(get=lambda sandbox_id: sandbox if sandbox_id == sandbox.id else None))
|
||
outputs = []
|
||
|
||
async def bash(command: str) -> str:
|
||
output = await asyncio.to_thread(sandbox.execute_command, command, timeout=30)
|
||
outputs.append(output)
|
||
return output
|
||
|
||
tool = StructuredTool.from_function(coroutine=bash, name="bash", description="Run offline native Node tests.")
|
||
graph = StateGraph(SandboxMessages)
|
||
graph.add_node("tools", ToolNode([tool], awrap_tool_call=ToolErrorHandlingMiddleware().awrap_tool_call))
|
||
graph.add_edge(START, "tools")
|
||
graph.add_edge("tools", END)
|
||
color = "FORCE_COLOR=1 " if reporter == "color-spec" else ""
|
||
reporter = reporter.removeprefix("color-")
|
||
command = f"{color}{shlex.quote(shutil.which('node'))} --test --test-reporter={reporter} /mnt/user-data/workspace/sample.test.cjs"
|
||
state = await graph.compile().ainvoke({"sandbox": {"sandbox_id": sandbox.id}, "messages": [AIMessage(content="", tool_calls=[{"name": "bash", "id": "node-call", "args": {"command": command}}])]})
|
||
assert len(outputs) == 1 and "duration_ms" in outputs[0], outputs
|
||
executions = executor_module._harvest_bash_executions(state)
|
||
assert len(executions) == 1 and executions[0]["shell_persistent"] is False
|
||
verdict = check_acceptance_criteria([f"tests_passed:{command}"], bash_executions=executions)
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected, (outputs, verdict)
|
||
|
||
|
||
@pytest.mark.skipif(os.name == "nt" or shutil.which("node") is None, reason="Requires POSIX LocalSandbox and Node.js")
|
||
def test_real_node_todo_stack_after_summary_survives_harvest(tmp_path, monkeypatch, executor_module):
|
||
workspace = tmp_path / ("long-path-" * 15)
|
||
workspace.mkdir()
|
||
test_file = workspace / "sample.test.cjs"
|
||
test_file.write_text(
|
||
"const {test}=require('node:test');\ntest('sample',()=>{});\ntest('1 failed',{todo:true},()=>{throw Error('fixture');});\n",
|
||
encoding="utf-8",
|
||
)
|
||
command = f"NO_COLOR=1 FORCE_COLOR=1 {shlex.quote(shutil.which('node'))} --test --test-reporter=spec {shlex.quote(str(test_file))}"
|
||
sandbox = LocalSandbox("node-long-todo-evidence")
|
||
monkeypatch.setattr("deerflow.sandbox.sandbox_provider.get_sandbox_provider", lambda: SimpleNamespace(get=lambda sandbox_id: sandbox if sandbox_id == sandbox.id else None))
|
||
output = sandbox.execute_command(command, timeout=30)
|
||
state = {
|
||
"sandbox": {"sandbox_id": sandbox.id},
|
||
"messages": [AIMessage(content="", tool_calls=[{"name": "bash", "id": "node-call", "args": {"command": command}}]), ToolMessage(content=output, tool_call_id="node-call")],
|
||
}
|
||
executions = executor_module._harvest_bash_executions(state)
|
||
assert len(output) > 1000
|
||
assert len(executions[0]["output_tail"]) <= 1000
|
||
assert executions[0]["test_summary"] == "passed"
|
||
verdict = check_acceptance_criteria([f"tests_passed:{command}"], bash_executions=executions)
|
||
assert verdict["all_hold"] is True, (output, executions, verdict)
|
||
|
||
|
||
@pytest.mark.parametrize("prefix", ["#", "ℹ"])
|
||
@pytest.mark.parametrize("log", ["12 passed", "all tests passed", "validation found 2 errors", "1 failed"])
|
||
@pytest.mark.parametrize("passed,skipped,expected", [(0, 250, (False, False)), (250, 0, (True, True))])
|
||
def test_node_counts_outrank_incidental_logs(executor_module, monkeypatch, prefix, log, passed, skipped, expected):
|
||
output = (f"# {log}" if prefix == "#" else log) + "\n" + "diagnostic\n" * 250 + _summary(prefix, passed=passed, skipped=skipped)
|
||
messages = [AIMessage(content="", tool_calls=[{"name": "bash", "id": "node-call", "args": {"command": "node --test"}}]), ToolMessage(content=output, tool_call_id="node-call")]
|
||
monkeypatch.setattr(executor_module, "_harvest_shell_persistence", lambda state: False)
|
||
executions = executor_module._harvest_bash_executions({"messages": messages})
|
||
verdict = check_acceptance_criteria(["tests_passed:node --test"], bash_executions=executions)
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"sibling,expected", [("2 passed in 0.01s", (True, True)), ("1 failed in 0.01s", (True, False)), ("Tests run: 2, Failures: 0, Errors: 0, Skipped: 0", (True, True)), ("Tests run: 2, Failures: 1, Errors: 0, Skipped: 0", (True, False))]
|
||
)
|
||
def test_node_counts_preserve_other_runner_summaries(sibling, expected):
|
||
verdict = _verdict(_summary(passed=0, skipped=250) + "\n" + sibling)
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected
|
||
|
||
|
||
def test_harvest_reuses_unchanged_summaries_and_invalidates_changes(executor_module, monkeypatch):
|
||
from unittest.mock import Mock
|
||
|
||
from deerflow.subagents import acceptance_checks
|
||
|
||
classify = Mock(wraps=acceptance_checks.summarize_test_output)
|
||
monkeypatch.setattr(acceptance_checks, "summarize_test_output", classify)
|
||
monkeypatch.setattr(executor_module, "_harvest_shell_persistence", lambda state: False)
|
||
call = AIMessage(content="", tool_calls=[{"name": "bash", "id": "node-call", "args": {"command": "node --test"}}])
|
||
tool = ToolMessage(content=_summary(), tool_call_id="node-call")
|
||
cache = {}
|
||
for _ in range(5):
|
||
entries = executor_module._harvest_bash_executions({"messages": [call, tool]}, summary_cache=cache)
|
||
assert entries[0]["test_summary"] == "passed"
|
||
assert classify.call_count == 1
|
||
tool.content = _summary(failed=1)
|
||
entries = executor_module._harvest_bash_executions({"messages": [call, tool]}, summary_cache=cache)
|
||
assert entries[0]["test_summary"] == "failed"
|
||
assert classify.call_count == 2
|
||
|
||
|
||
def test_summary_cache_is_bounded_and_tracks_compaction(executor_module, monkeypatch):
|
||
from unittest.mock import Mock
|
||
|
||
from deerflow.subagents import acceptance_checks
|
||
|
||
classify = Mock(wraps=acceptance_checks.summarize_test_output)
|
||
monkeypatch.setattr(acceptance_checks, "summarize_test_output", classify)
|
||
monkeypatch.setattr(executor_module, "_harvest_shell_persistence", lambda state: False)
|
||
calls = [{"name": "bash", "id": f"call-{i}", "args": {"command": "node --test"}} for i in range(25)]
|
||
ai = AIMessage(content="", tool_calls=calls)
|
||
tools = [ToolMessage(content=_summary(), tool_call_id=call["id"]) for call in calls]
|
||
cache = {}
|
||
for _ in range(3):
|
||
entries = executor_module._harvest_bash_executions({"messages": [ai, *tools]}, summary_cache=cache)
|
||
assert len(entries) == len(cache) == 20
|
||
assert classify.call_count == 20
|
||
executor_module._harvest_bash_executions({"messages": [ai, tools[-1]]}, summary_cache=cache)
|
||
assert list(cache) == [("call", "call-24")]
|
||
assert classify.call_count == 20
|
||
executor_module._harvest_bash_executions({"messages": [AIMessage(content="compacted")]}, summary_cache=cache)
|
||
assert cache == {}
|
||
|
||
|
||
@pytest.mark.parametrize("node_passed", [0, 1])
|
||
@pytest.mark.parametrize(
|
||
"sibling,expected",
|
||
[
|
||
("1 failed, 1 passed, 1 warning in 0.05s", (True, False)),
|
||
("2 passed, 3 warnings in 0.05s", (True, True)),
|
||
("=== 1 failed, 1 passed, 2 warnings in 0.05s ===", (True, False)),
|
||
],
|
||
)
|
||
def test_node_sibling_pytest_warning_summaries(node_passed, sibling, expected):
|
||
verdict = _verdict(_summary(passed=node_passed, skipped=1) + "\n" + sibling)
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected
|
||
|
||
|
||
@pytest.mark.parametrize("duration", [59.0, 60.0, 65.0, 3605.0, 86405.0, 172805.0])
|
||
@pytest.mark.parametrize("prefix,newline,decoration", [("#", "\n", ""), ("ℹ", "\r\n", "===")])
|
||
@pytest.mark.parametrize("node_passed", [0, 1])
|
||
@pytest.mark.parametrize("summary,expected", [("1 failed, 1 passed, 1 warning", (True, False)), ("2 passed, 3 warnings", (True, True))])
|
||
def test_harvest_preserves_pytest_sibling_duration(executor_module, monkeypatch, duration, prefix, newline, decoration, node_passed, summary, expected):
|
||
sibling = f"{decoration} {summary} in {format_session_duration(duration)} {decoration}".strip()
|
||
output = (_summary(prefix, passed=node_passed, skipped=1) + "\n" + sibling + "\n" + "diagnostic\n" * 150).replace("\n", newline)
|
||
messages = [AIMessage(content="", tool_calls=[{"name": "bash", "id": "mixed-test-call", "args": {"command": "make test"}}]), ToolMessage(content=output, tool_call_id="mixed-test-call")]
|
||
monkeypatch.setattr(executor_module, "_harvest_shell_persistence", lambda state: False)
|
||
executions = executor_module._harvest_bash_executions({"messages": messages})
|
||
assert len(executions[0]["output_tail"]) <= 1000
|
||
assert summary not in executions[0]["output_tail"]
|
||
assert executions[0]["test_summary"] == ("passed" if expected[1] else "failed")
|
||
verdict = check_acceptance_criteria(["tests_passed:make test"], bash_executions=executions)
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected
|
||
|
||
|
||
@pytest.mark.parametrize("suffix", [" (extra log text)", " (0:1:05)", " (0:61:05)", " (0:01:05) trailing log text"])
|
||
def test_node_sibling_pytest_duration_does_not_admit_arbitrary_suffixes(suffix):
|
||
verdict = _verdict(_summary(passed=0, skipped=1) + "\n2 passed in 65.00s" + suffix)
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)
|
||
|
||
|
||
@pytest.mark.parametrize("node_passed", [0, 1])
|
||
@pytest.mark.parametrize("cases", ["--- SKIP: TestA (0.00s)", "--- PASS: TestA (0.00s)", "--- SKIP: TestA (0.00s)\n--- PASS: TestB (0.00s)"])
|
||
def test_node_sibling_go_preserves_case_evidence(node_passed, cases):
|
||
output = _summary(passed=node_passed, skipped=1) + "\n=== RUN TestA\n" + cases + "\nok \texample/pkg\t0.002s"
|
||
verdict = _verdict(output)
|
||
holds = node_passed > 0 or "--- PASS:" in cases
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (holds, holds)
|