mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
fix(subagents): preserve test summaries before evidence truncation (#7407)
* fix(subagents): preserve test summaries before evidence truncation * fix(subagents): honor Node counters and cache harvested summaries * fix(subagents): retain warning counts and Go case evidence * fix(subagents): retain long pytest sibling summaries --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
Xuehao Xu
GitHub
parent
3113c150cd
commit
8237bc6c9e
@ -2568,7 +2568,13 @@ For `tests_passed:node --test`, complete TAP or spec counter summaries must
|
||||
contain an actual pass and no failed or cancelled tests. Individual TAP `ok`
|
||||
points do not prove the run passed. Empty, all-skipped, all-todo, incomplete or
|
||||
inconsistent summaries remain `UNVERIFIED`; execution and shell-provenance
|
||||
checks still apply.
|
||||
checks still apply. Summary detection uses the complete recorded tool output
|
||||
before retaining a bounded diagnostic tail, so later TODO stacks or warnings
|
||||
do not erase the test evidence. Complete, consistent Node counters take
|
||||
precedence over incidental log text such as `12 passed` or `2 errors`;
|
||||
independently shaped summaries from other runners still contribute evidence.
|
||||
Pytest sibling summaries retain warning counts and the optional human-readable
|
||||
duration suffix printed for runs lasting at least a minute, such as `(0:01:05)`.
|
||||
|
||||
For Maven and Gradle test acceptance, a `BUILD SUCCESS` / `BUILD SUCCESSFUL`
|
||||
banner alone remains `UNVERIFIED`: builds can succeed without running tests.
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -41,7 +41,8 @@ Leaf families:
|
||||
anchor to a *specific recorded execution* — a matching bash execution
|
||||
(harvested by the executor from the same stamped ``ToolMessage``s the
|
||||
receipt layer reads) with ``status=success`` and a test-summary shape in
|
||||
its output tail — not merely to some successful call. Matching accounts
|
||||
its recorded output (classified before tail truncation) — not merely to
|
||||
some successful call. Matching accounts
|
||||
for shell command structure (operator-separated segments, executables,
|
||||
arguments), so an unrelated command that merely mentions the criterion
|
||||
string (e.g. inside an ``echo`` argument or a comment) cannot anchor the
|
||||
@ -75,7 +76,7 @@ import stat
|
||||
import time
|
||||
import unicodedata
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Any, TypedDict
|
||||
from typing import Any, Literal, TypedDict
|
||||
|
||||
from deerflow.config.paths import VIRTUAL_PATH_PREFIX
|
||||
from deerflow.subagents.report_contract import MAX_ACCEPTANCE_CRITERIA, normalize_acceptance_criteria
|
||||
@ -163,17 +164,19 @@ _NODE_TEST_SUMMARY_RE = re.compile(
|
||||
)
|
||||
|
||||
|
||||
def _node_test_summary_holds(text: str) -> bool | None:
|
||||
def _node_test_summary_outcome(text: str) -> Literal["passed", "failed", "unverified"] | None:
|
||||
"""Use complete, consistent counts; failures/cancellations win."""
|
||||
positive = False
|
||||
complete = False
|
||||
for match in _NODE_TEST_SUMMARY_RE.finditer(text):
|
||||
total, passed, failed, cancelled, skipped, todo = (int(match[key]) for key in ("total", "passed", "failed", "cancelled", "skipped", "todo"))
|
||||
if total != passed + failed + cancelled + skipped + todo:
|
||||
continue
|
||||
complete = True
|
||||
if failed or cancelled:
|
||||
return False
|
||||
return "failed"
|
||||
positive = positive or passed > 0
|
||||
return True if positive else None
|
||||
return "passed" if positive else "unverified" if complete else None
|
||||
|
||||
|
||||
#: Zero-test evidence vetoes the pass shapes the count-bearing alternatives
|
||||
@ -2026,6 +2029,49 @@ def _output_attribution(executed_command: str, thread_data: Mapping[str, Any] |
|
||||
return None
|
||||
|
||||
|
||||
# With complete Node counters present, only independently shaped sibling runner
|
||||
# summaries may contribute evidence. Bare "N passed/errors" in program logs
|
||||
# cannot override those counters. Keep the legacy loose fallback without Node.
|
||||
_SIBLING_TEST_SUMMARY_RE = re.compile(
|
||||
r"^[ \t=]*(?:[0-9]+ (?:passed|failed|errors?|warnings?|skipped|deselected|xfailed|xpassed)(?:, )?)+ in [0-9]+(?:\.[0-9]+)?s"
|
||||
r"(?: \((?:[0-9]+ days?, )?[0-9]+:[0-5][0-9]:[0-5][0-9]\))?[ \t=]*\r?$"
|
||||
r"|^test result: (?:ok|FAILED)\.[^\r\n]*$"
|
||||
r"|^(?:ok|FAIL)[ \t]+[^\r\n]+$"
|
||||
r"|^[ \t]*--- (?:PASS|FAIL|SKIP):[^\r\n]*$"
|
||||
r"|^Ran [0-9]+ tests? in [0-9]+(?:\.[0-9]+)?s\r?\n\r?\n(?:OK|FAILED)[^\r\n]*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def summarize_test_output(output: str) -> Literal["passed", "failed", "unverified"]:
|
||||
"""Classify the complete recorded tool output before evidence is truncated.
|
||||
|
||||
This bounded stamp carries summary semantics without retaining full logs.
|
||||
It is execution evidence, not an acceptance verdict: consumers must still
|
||||
enforce command matching, exit status, output attribution and shell provenance.
|
||||
"""
|
||||
# Test names are arbitrary data, not runner summaries. Exclude TAP/spec
|
||||
# point labels from negative/zero evidence too (`test('1 failed', ...)`).
|
||||
text = _NODE_TEST_POINT_RE.sub("", _ANSI_SGR_RE.sub("", output))
|
||||
build_summary = _build_test_summary_holds(text)
|
||||
node_summary = _node_test_summary_outcome(text)
|
||||
if node_summary is not None:
|
||||
# A complete Node report owns its family. Recognizable sibling reports
|
||||
# still participate (for make/test scripts running several frameworks).
|
||||
# Retain pytest warnings/durations and Go per-case markers too: these
|
||||
# are part of the sibling evidence, not disposable diagnostic text.
|
||||
text = "\n".join(match[0] for pattern in (_SIBLING_TEST_SUMMARY_RE, _MAVEN_TEST_SUMMARY_RE, _GRADLE_TEST_SUMMARY_RE) for match in pattern.finditer(text))
|
||||
if _TEST_FAIL_SHAPE_RE.search(text) or build_summary is False or node_summary == "failed":
|
||||
return "failed"
|
||||
# Empty packages/targets cannot veto complete positive sibling evidence.
|
||||
summary_text = _without_empty_cargo_targets(_GO_ZERO_TEST_LINE_RE.sub("", text))
|
||||
go_ok = _GO_TEST_PASS_SHAPE_RE.search(summary_text) and _go_package_summary_is_evidence(summary_text)
|
||||
passing = _TEST_PASS_SHAPE_RE.search(summary_text) or go_ok or _unittest_test_summary_holds(summary_text) or build_summary is True or node_summary == "passed"
|
||||
if passing and not _TEST_ZERO_SHAPE_RE.search(summary_text):
|
||||
return "passed"
|
||||
return "unverified"
|
||||
|
||||
|
||||
def _check_tests_passed_leaf(command: str, bash_executions: list[dict[str, Any]] | None, thread_data: Mapping[str, Any] | None = None) -> AcceptanceLeaf:
|
||||
base: AcceptanceLeaf = {"criterion": "", "family": "tests_passed", "checked": False, "holds": False, "detail": ""}
|
||||
matches: list[tuple[str, dict[str, Any]]] = []
|
||||
@ -2083,26 +2129,21 @@ def _check_tests_passed_leaf(command: str, bash_executions: list[dict[str, Any]]
|
||||
else:
|
||||
base["detail"] = f"latest matching run recorded status={status or 'unknown'}"
|
||||
return base
|
||||
output_tail = _ANSI_SGR_RE.sub("", str(latest.get("output_tail") or ""))
|
||||
unattributable = _output_attribution(str(latest.get("command") or ""), thread_data)
|
||||
if unattributable is not None:
|
||||
# Neither a pass nor a fail shape here can be trusted either way.
|
||||
base["detail"] = unattributable
|
||||
return base
|
||||
# Test names are arbitrary data, not runner summaries. Exclude TAP/spec
|
||||
# point labels from negative/zero evidence too (`test('1 failed', ...)`).
|
||||
output_tail = _NODE_TEST_POINT_RE.sub("", output_tail)
|
||||
build_summary = _build_test_summary_holds(output_tail)
|
||||
node_summary = _node_test_summary_holds(output_tail)
|
||||
if _TEST_FAIL_SHAPE_RE.search(output_tail) or build_summary is False or node_summary is False:
|
||||
# New harvests classify the full ToolMessage before truncating its tail.
|
||||
# Legacy entries have only the tail; retain their existing conservative check.
|
||||
summary = latest.get("test_summary")
|
||||
if summary not in ("passed", "failed", "unverified"):
|
||||
summary = summarize_test_output(str(latest.get("output_tail") or ""))
|
||||
if summary == "failed":
|
||||
base["checked"] = True
|
||||
base["detail"] = "recorded output carries a failing test summary"
|
||||
return base
|
||||
# Empty packages/targets cannot veto complete positive sibling evidence.
|
||||
summary_tail = _without_empty_cargo_targets(_GO_ZERO_TEST_LINE_RE.sub("", output_tail))
|
||||
go_ok = _GO_TEST_PASS_SHAPE_RE.search(summary_tail) and _go_package_summary_is_evidence(summary_tail)
|
||||
passing = _TEST_PASS_SHAPE_RE.search(summary_tail) or go_ok or _unittest_test_summary_holds(summary_tail) or build_summary is True or node_summary is True
|
||||
if passing and not _TEST_ZERO_SHAPE_RE.search(summary_tail):
|
||||
if summary == "passed":
|
||||
base["checked"] = True
|
||||
base["holds"] = True
|
||||
base["detail"] = "recorded output carries a passing test summary"
|
||||
|
||||
@ -473,7 +473,7 @@ def _harvest_tool_receipts(
|
||||
#: Bash-family tool names whose calls count as recorded command executions.
|
||||
_BASH_EVIDENCE_TOOL_NAMES = BASH_TOOL_NAMES
|
||||
#: Bounds for the harvested evidence: only the newest few executions travel,
|
||||
#: with command/output text capped (test summaries print at the tail).
|
||||
#: with command/output text capped and full-output summary semantics preserved.
|
||||
_BASH_EVIDENCE_MAX_ENTRIES = 20
|
||||
_BASH_EVIDENCE_COMMAND_CHARS = 500
|
||||
_BASH_EVIDENCE_OUTPUT_TAIL_CHARS = 1000
|
||||
@ -529,14 +529,16 @@ def _harvest_shell_persistence(final_state: Any) -> bool | None:
|
||||
|
||||
def _harvest_bash_executions(
|
||||
final_state: Any,
|
||||
*,
|
||||
summary_cache: dict[str, tuple[str, str]] | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
"""Harvest bounded bash command/output evidence from one streamed state.
|
||||
|
||||
RFC #4651 PR4: a ``tests_passed:<command>`` acceptance leaf must anchor to
|
||||
a specific recorded execution — the command text lets the parent match the
|
||||
criterion against the call that actually ran, and the bounded output tail
|
||||
carries the test-summary shape. The recorded status is the **actual shell
|
||||
exit status**: a nonzero bash exit comes back as ordinary output text
|
||||
criterion against the call that actually ran. A bounded summary stamp is
|
||||
computed from the full tool output before retaining the diagnostic tail.
|
||||
The recorded status is the **actual shell exit status**: a nonzero bash exit comes back as ordinary output text
|
||||
(local: a trailing ``Exit Code: N``; e2b/opensandbox with empty output:
|
||||
``Command exited with code N``), which ``deerflow_tool_meta`` still reports
|
||||
as success — so an explicit exit marker wins, and the meta status is only
|
||||
@ -549,12 +551,17 @@ def _harvest_bash_executions(
|
||||
UNVERIFIED.
|
||||
"""
|
||||
if not final_state:
|
||||
if summary_cache is not None:
|
||||
summary_cache.clear()
|
||||
return None
|
||||
messages = final_state.get("messages") if isinstance(final_state, dict) else None
|
||||
if not messages:
|
||||
if summary_cache is not None:
|
||||
summary_cache.clear()
|
||||
return None
|
||||
try:
|
||||
from deerflow.agents.middlewares.tool_result_meta import TOOL_META_KEY
|
||||
from deerflow.subagents.acceptance_checks import summarize_test_output
|
||||
|
||||
commands: dict[str, tuple[str, str, bool]] = {}
|
||||
for message in messages:
|
||||
@ -573,20 +580,28 @@ def _harvest_bash_executions(
|
||||
# selection-changing suffix like ``-k smoke`` could be cut).
|
||||
commands[tool_call_id] = (name, command[:_BASH_EVIDENCE_COMMAND_CHARS], len(command) > _BASH_EVIDENCE_COMMAND_CHARS)
|
||||
if not commands:
|
||||
if summary_cache is not None:
|
||||
summary_cache.clear()
|
||||
return []
|
||||
executions: list[dict[str, Any]] = []
|
||||
for message in messages:
|
||||
if not isinstance(message, ToolMessage):
|
||||
continue
|
||||
# Only retained executions need full-output parsing. The per-run cache
|
||||
# survives values chunks, but keeps at most the same newest 20 outputs.
|
||||
tool_messages = [message for message in messages if isinstance(message, ToolMessage) and str(message.tool_call_id or "") in commands][-_BASH_EVIDENCE_MAX_ENTRIES:]
|
||||
next_cache: dict[str, tuple[str, str]] = {}
|
||||
for message in tool_messages:
|
||||
tool_call_id = str(message.tool_call_id or "")
|
||||
entry = commands.get(tool_call_id)
|
||||
if entry is None:
|
||||
continue
|
||||
entry = commands[tool_call_id]
|
||||
name, command, command_truncated = entry
|
||||
meta = (message.additional_kwargs or {}).get(TOOL_META_KEY) or {}
|
||||
meta_status = str(meta.get("status") or getattr(message, "status", "success") or "success")
|
||||
content = message.content if isinstance(message.content, str) else json.dumps(message.content, sort_keys=True, default=str)
|
||||
status, status_marker = _bash_evidence_status(content, meta_status)
|
||||
cached = summary_cache.get(tool_call_id) if summary_cache is not None else None
|
||||
if cached is not None and (cached[0] is content or cached[0] == content):
|
||||
test_summary = cached[1]
|
||||
else:
|
||||
test_summary = summarize_test_output(content)
|
||||
next_cache[tool_call_id] = (content, test_summary)
|
||||
output_tail = content[-_BASH_EVIDENCE_OUTPUT_TAIL_CHARS:]
|
||||
if len(content) > _BASH_EVIDENCE_OUTPUT_TAIL_CHARS and content[-_BASH_EVIDENCE_OUTPUT_TAIL_CHARS - 1] != "\n":
|
||||
# A clipped test name/log line can look like a summary after
|
||||
@ -599,10 +614,14 @@ def _harvest_bash_executions(
|
||||
"command": command,
|
||||
"command_truncated": command_truncated,
|
||||
"output_tail": output_tail,
|
||||
"test_summary": test_summary,
|
||||
"status": status,
|
||||
"status_marker": status_marker,
|
||||
}
|
||||
)
|
||||
if summary_cache is not None:
|
||||
summary_cache.clear()
|
||||
summary_cache.update(next_cache)
|
||||
# Provenance stamp: whether the producing sandbox reuses one
|
||||
# persistent shell session. Captured here — while the state that
|
||||
# carried the evidence is at hand — because the parent-side checker
|
||||
@ -1716,6 +1735,8 @@ class SubagentExecutor:
|
||||
return None
|
||||
return _harvest_tool_receipts(final_state, prefer_citing_turn=prefer_citing_turn)
|
||||
|
||||
bash_summary_cache: dict[str, tuple[str, str]] = {}
|
||||
|
||||
def current_bash_executions() -> list[dict[str, Any]] | None:
|
||||
# RFC #4651 PR4: evidence for tests_passed acceptance leaves.
|
||||
# Accumulated from every chunk (not harvested once at terminal) so
|
||||
@ -1723,7 +1744,7 @@ class SubagentExecutor:
|
||||
# execution. Criteria-free runs pay nothing.
|
||||
if not self.acceptance_criteria:
|
||||
return None
|
||||
return _harvest_bash_executions(final_state)
|
||||
return _harvest_bash_executions(final_state, summary_cache=bash_summary_cache)
|
||||
|
||||
try:
|
||||
if task_info is not None and task_store is not None:
|
||||
|
||||
@ -10,6 +10,7 @@ from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from _pytest.terminal import format_session_duration
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
from langchain_core.tools import StructuredTool
|
||||
from langgraph.graph import END, START, MessagesState, StateGraph
|
||||
@ -172,6 +173,31 @@ def test_harvest_drops_a_partial_label_before_interpreting_counts(executor_modul
|
||||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"output,expected",
|
||||
[
|
||||
(_summary("ℹ", todo=1) + "\n" + " at fixture\n" * 150, (True, True)),
|
||||
(_summary("ℹ", failed=1) + "\n" + " at fixture\n" * 150, (True, False)),
|
||||
(_summary("ℹ", passed=0, todo=1) + "\n" + " at fixture\n" * 150, (False, False)),
|
||||
(_summary("ℹ", failed=1) + "\n" + "diagnostic\n" * 150 + _summary("ℹ"), (True, False)),
|
||||
("0 passed\n" + "diagnostic\n" * 150 + "2 passed", (False, False)),
|
||||
],
|
||||
ids=["pass-before-stack", "failure-before-stack", "all-todo-before-stack", "failure-before-passing-tail", "zero-before-passing-tail"],
|
||||
)
|
||||
def test_full_summary_survives_bounded_harvest(executor_module, monkeypatch, output, expected):
|
||||
messages = [AIMessage(content="", tool_calls=[{"name": "bash", "id": "node-call", "args": {"command": "node --test"}}]), ToolMessage(content=output, tool_call_id="node-call")]
|
||||
monkeypatch.setattr(executor_module, "_harvest_shell_persistence", lambda state: False)
|
||||
executions = executor_module._harvest_bash_executions({"messages": messages})
|
||||
assert len(executions[0]["output_tail"]) <= 1000
|
||||
verdict = check_acceptance_criteria(["tests_passed:node --test"], bash_executions=executions)
|
||||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("overrides", [{"shell_persistent": True}, {"shell_persistent": None}, {"command_truncated": True}, {"command": "echo other; node --test"}, {"command": "node --test > log"}])
|
||||
def test_full_summary_does_not_bypass_admission(overrides):
|
||||
assert _verdict("", test_summary="passed", **overrides)["leaves"][0]["checked"] is False
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt" or shutil.which("node") is None, reason="Requires POSIX LocalSandbox and Node.js")
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("reporter", ["tap", "spec", "color-spec"])
|
||||
@ -194,6 +220,8 @@ def test_harvest_drops_a_partial_label_before_interpreting_counts(executor_modul
|
||||
("describe('1 failed', () => { test('sample', () => {}); });", (True, True)),
|
||||
("test('sample', () => { throw Error('fixture'); });", (True, False)),
|
||||
("test('pending', { timeout: 20 }, () => new Promise(() => {}));", (True, False)),
|
||||
("console.log('12 passed'); for (let i=0; i<250; i++) test.skip('t'+i,()=>{});", (False, False)),
|
||||
("console.log('validation found 2 errors'); for (let i=0; i<250; i++) test('t'+i,()=>{});", (True, True)),
|
||||
],
|
||||
ids=[
|
||||
"pass",
|
||||
@ -212,12 +240,12 @@ def test_harvest_drops_a_partial_label_before_interpreting_counts(executor_modul
|
||||
"summary-like-suite",
|
||||
"failure",
|
||||
"cancelled",
|
||||
"stray-pass-all-skipped",
|
||||
"stray-error-all-passing",
|
||||
],
|
||||
)
|
||||
async def test_real_node_results_reach_parent_acceptance(tmp_path, monkeypatch, executor_module, reporter, body, expected):
|
||||
# Keep reporter counts inside the bounded evidence tail regardless of
|
||||
# temporary-path length; truncation itself has dedicated coverage above.
|
||||
(tmp_path / "sample.test.cjs").write_text("const { test, describe } = require('node:test');\nError.stackTraceLimit = 0;\n" + body, encoding="utf-8")
|
||||
(tmp_path / "sample.test.cjs").write_text("const { test, describe } = require('node:test');\n" + body, encoding="utf-8")
|
||||
monkeypatch.delenv("NO_COLOR", raising=False)
|
||||
sandbox = LocalSandbox("node-evidence", path_mappings=[PathMapping("/mnt/user-data/workspace", str(tmp_path))])
|
||||
monkeypatch.setattr("deerflow.sandbox.sandbox_provider.get_sandbox_provider", lambda: SimpleNamespace(get=lambda sandbox_id: sandbox if sandbox_id == sandbox.id else None))
|
||||
@ -242,3 +270,138 @@ async def test_real_node_results_reach_parent_acceptance(tmp_path, monkeypatch,
|
||||
assert len(executions) == 1 and executions[0]["shell_persistent"] is False
|
||||
verdict = check_acceptance_criteria([f"tests_passed:{command}"], bash_executions=executions)
|
||||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected, (outputs, verdict)
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt" or shutil.which("node") is None, reason="Requires POSIX LocalSandbox and Node.js")
|
||||
def test_real_node_todo_stack_after_summary_survives_harvest(tmp_path, monkeypatch, executor_module):
|
||||
workspace = tmp_path / ("long-path-" * 15)
|
||||
workspace.mkdir()
|
||||
test_file = workspace / "sample.test.cjs"
|
||||
test_file.write_text(
|
||||
"const {test}=require('node:test');\ntest('sample',()=>{});\ntest('1 failed',{todo:true},()=>{throw Error('fixture');});\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
command = f"NO_COLOR=1 FORCE_COLOR=1 {shlex.quote(shutil.which('node'))} --test --test-reporter=spec {shlex.quote(str(test_file))}"
|
||||
sandbox = LocalSandbox("node-long-todo-evidence")
|
||||
monkeypatch.setattr("deerflow.sandbox.sandbox_provider.get_sandbox_provider", lambda: SimpleNamespace(get=lambda sandbox_id: sandbox if sandbox_id == sandbox.id else None))
|
||||
output = sandbox.execute_command(command, timeout=30)
|
||||
state = {
|
||||
"sandbox": {"sandbox_id": sandbox.id},
|
||||
"messages": [AIMessage(content="", tool_calls=[{"name": "bash", "id": "node-call", "args": {"command": command}}]), ToolMessage(content=output, tool_call_id="node-call")],
|
||||
}
|
||||
executions = executor_module._harvest_bash_executions(state)
|
||||
assert len(output) > 1000
|
||||
assert len(executions[0]["output_tail"]) <= 1000
|
||||
assert executions[0]["test_summary"] == "passed"
|
||||
verdict = check_acceptance_criteria([f"tests_passed:{command}"], bash_executions=executions)
|
||||
assert verdict["all_hold"] is True, (output, executions, verdict)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prefix", ["#", "ℹ"])
|
||||
@pytest.mark.parametrize("log", ["12 passed", "all tests passed", "validation found 2 errors", "1 failed"])
|
||||
@pytest.mark.parametrize("passed,skipped,expected", [(0, 250, (False, False)), (250, 0, (True, True))])
|
||||
def test_node_counts_outrank_incidental_logs(executor_module, monkeypatch, prefix, log, passed, skipped, expected):
|
||||
output = (f"# {log}" if prefix == "#" else log) + "\n" + "diagnostic\n" * 250 + _summary(prefix, passed=passed, skipped=skipped)
|
||||
messages = [AIMessage(content="", tool_calls=[{"name": "bash", "id": "node-call", "args": {"command": "node --test"}}]), ToolMessage(content=output, tool_call_id="node-call")]
|
||||
monkeypatch.setattr(executor_module, "_harvest_shell_persistence", lambda state: False)
|
||||
executions = executor_module._harvest_bash_executions({"messages": messages})
|
||||
verdict = check_acceptance_criteria(["tests_passed:node --test"], bash_executions=executions)
|
||||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sibling,expected", [("2 passed in 0.01s", (True, True)), ("1 failed in 0.01s", (True, False)), ("Tests run: 2, Failures: 0, Errors: 0, Skipped: 0", (True, True)), ("Tests run: 2, Failures: 1, Errors: 0, Skipped: 0", (True, False))]
|
||||
)
|
||||
def test_node_counts_preserve_other_runner_summaries(sibling, expected):
|
||||
verdict = _verdict(_summary(passed=0, skipped=250) + "\n" + sibling)
|
||||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected
|
||||
|
||||
|
||||
def test_harvest_reuses_unchanged_summaries_and_invalidates_changes(executor_module, monkeypatch):
|
||||
from unittest.mock import Mock
|
||||
|
||||
from deerflow.subagents import acceptance_checks
|
||||
|
||||
classify = Mock(wraps=acceptance_checks.summarize_test_output)
|
||||
monkeypatch.setattr(acceptance_checks, "summarize_test_output", classify)
|
||||
monkeypatch.setattr(executor_module, "_harvest_shell_persistence", lambda state: False)
|
||||
call = AIMessage(content="", tool_calls=[{"name": "bash", "id": "node-call", "args": {"command": "node --test"}}])
|
||||
tool = ToolMessage(content=_summary(), tool_call_id="node-call")
|
||||
cache = {}
|
||||
for _ in range(5):
|
||||
entries = executor_module._harvest_bash_executions({"messages": [call, tool]}, summary_cache=cache)
|
||||
assert entries[0]["test_summary"] == "passed"
|
||||
assert classify.call_count == 1
|
||||
tool.content = _summary(failed=1)
|
||||
entries = executor_module._harvest_bash_executions({"messages": [call, tool]}, summary_cache=cache)
|
||||
assert entries[0]["test_summary"] == "failed"
|
||||
assert classify.call_count == 2
|
||||
|
||||
|
||||
def test_summary_cache_is_bounded_and_tracks_compaction(executor_module, monkeypatch):
|
||||
from unittest.mock import Mock
|
||||
|
||||
from deerflow.subagents import acceptance_checks
|
||||
|
||||
classify = Mock(wraps=acceptance_checks.summarize_test_output)
|
||||
monkeypatch.setattr(acceptance_checks, "summarize_test_output", classify)
|
||||
monkeypatch.setattr(executor_module, "_harvest_shell_persistence", lambda state: False)
|
||||
calls = [{"name": "bash", "id": f"call-{i}", "args": {"command": "node --test"}} for i in range(25)]
|
||||
ai = AIMessage(content="", tool_calls=calls)
|
||||
tools = [ToolMessage(content=_summary(), tool_call_id=call["id"]) for call in calls]
|
||||
cache = {}
|
||||
for _ in range(3):
|
||||
entries = executor_module._harvest_bash_executions({"messages": [ai, *tools]}, summary_cache=cache)
|
||||
assert len(entries) == len(cache) == 20
|
||||
assert classify.call_count == 20
|
||||
executor_module._harvest_bash_executions({"messages": [ai, tools[-1]]}, summary_cache=cache)
|
||||
assert list(cache) == ["call-24"]
|
||||
assert classify.call_count == 20
|
||||
executor_module._harvest_bash_executions({"messages": [AIMessage(content="compacted")]}, summary_cache=cache)
|
||||
assert cache == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("node_passed", [0, 1])
|
||||
@pytest.mark.parametrize(
|
||||
"sibling,expected",
|
||||
[
|
||||
("1 failed, 1 passed, 1 warning in 0.05s", (True, False)),
|
||||
("2 passed, 3 warnings in 0.05s", (True, True)),
|
||||
("=== 1 failed, 1 passed, 2 warnings in 0.05s ===", (True, False)),
|
||||
],
|
||||
)
|
||||
def test_node_sibling_pytest_warning_summaries(node_passed, sibling, expected):
|
||||
verdict = _verdict(_summary(passed=node_passed, skipped=1) + "\n" + sibling)
|
||||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("duration", [59.0, 60.0, 65.0, 3605.0, 86405.0, 172805.0])
|
||||
@pytest.mark.parametrize("prefix,newline,decoration", [("#", "\n", ""), ("ℹ", "\r\n", "===")])
|
||||
@pytest.mark.parametrize("node_passed", [0, 1])
|
||||
@pytest.mark.parametrize("summary,expected", [("1 failed, 1 passed, 1 warning", (True, False)), ("2 passed, 3 warnings", (True, True))])
|
||||
def test_harvest_preserves_pytest_sibling_duration(executor_module, monkeypatch, duration, prefix, newline, decoration, node_passed, summary, expected):
|
||||
sibling = f"{decoration} {summary} in {format_session_duration(duration)} {decoration}".strip()
|
||||
output = (_summary(prefix, passed=node_passed, skipped=1) + "\n" + sibling + "\n" + "diagnostic\n" * 150).replace("\n", newline)
|
||||
messages = [AIMessage(content="", tool_calls=[{"name": "bash", "id": "mixed-test-call", "args": {"command": "make test"}}]), ToolMessage(content=output, tool_call_id="mixed-test-call")]
|
||||
monkeypatch.setattr(executor_module, "_harvest_shell_persistence", lambda state: False)
|
||||
executions = executor_module._harvest_bash_executions({"messages": messages})
|
||||
assert len(executions[0]["output_tail"]) <= 1000
|
||||
assert summary not in executions[0]["output_tail"]
|
||||
assert executions[0]["test_summary"] == ("passed" if expected[1] else "failed")
|
||||
verdict = check_acceptance_criteria(["tests_passed:make test"], bash_executions=executions)
|
||||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", [" (extra log text)", " (0:1:05)", " (0:61:05)", " (0:01:05) trailing log text"])
|
||||
def test_node_sibling_pytest_duration_does_not_admit_arbitrary_suffixes(suffix):
|
||||
verdict = _verdict(_summary(passed=0, skipped=1) + "\n2 passed in 65.00s" + suffix)
|
||||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("node_passed", [0, 1])
|
||||
@pytest.mark.parametrize("cases", ["--- SKIP: TestA (0.00s)", "--- PASS: TestA (0.00s)", "--- SKIP: TestA (0.00s)\n--- PASS: TestB (0.00s)"])
|
||||
def test_node_sibling_go_preserves_case_evidence(node_passed, cases):
|
||||
output = _summary(passed=node_passed, skipped=1) + "\n=== RUN TestA\n" + cases + "\nok \texample/pkg\t0.002s"
|
||||
verdict = _verdict(output)
|
||||
holds = node_passed > 0 or "--- PASS:" in cases
|
||||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (holds, holds)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user