mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
34ccd79351
* fix(acceptance): recognize native Node test summaries * test(acceptance): isolate native Node suite labels * fix(acceptance): recognize native Node test summaries * fix(acceptance): filter Node warning points * test(acceptance): cover Node warning reporter points * test(acceptance): cover Node warning labels with native reporters * fix(acceptance): keep failure counts on one diagnostic line
242 lines
12 KiB
Python
242 lines
12 KiB
Python
"""Node's native reporters must prove executed tests, not TAP point labels."""
|
||
|
||
import asyncio
|
||
import importlib.util
|
||
import os
|
||
import shlex
|
||
import shutil
|
||
import sys
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
import pytest
|
||
from langchain_core.messages import AIMessage, ToolMessage
|
||
from langchain_core.tools import StructuredTool
|
||
from langgraph.graph import END, START, MessagesState, StateGraph
|
||
from langgraph.prebuilt import ToolNode
|
||
|
||
from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware
|
||
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping
|
||
from deerflow.subagents.acceptance_checks import _carries_summary_shape, check_acceptance_criteria, render_acceptance_section
|
||
|
||
|
||
def _summary(prefix="#", *, passed=1, failed=0, cancelled=0, skipped=0, todo=0, total=None):
|
||
counters = {"tests": passed + failed + cancelled + skipped + todo if total is None else total, "suites": 0, "pass": passed, "fail": failed, "cancelled": cancelled, "skipped": skipped, "todo": todo}
|
||
return "\n".join(f"{prefix} {key} {value}" for key, value in counters.items()) + f"\n{prefix} duration_ms 12.5"
|
||
|
||
|
||
def _verdict(output, **overrides):
|
||
execution = {"command": "node --test", "status": "success", "shell_persistent": False, "output_tail": output, **overrides}
|
||
return check_acceptance_criteria(["tests_passed:node --test"], bash_executions=[execution])
|
||
|
||
|
||
@pytest.mark.parametrize("prefix", ["#", "ℹ"])
|
||
@pytest.mark.parametrize("newline", ["\n", "\r\n"])
|
||
def test_native_reporters_accept_nonempty_passes(prefix, newline):
|
||
verdict = _verdict(_summary(prefix, skipped=1, todo=1).replace("\n", newline))
|
||
assert verdict["all_hold"] is True
|
||
assert "[holds]" in render_acceptance_section(verdict)
|
||
|
||
|
||
@pytest.mark.parametrize("prefix", ["#", "ℹ"])
|
||
@pytest.mark.parametrize("counts", [{"failed": 1}, {"cancelled": 1}])
|
||
def test_native_failures_override_incidental_pass_text(prefix, counts):
|
||
verdict = _verdict("ok 1 - all tests passed\n" + _summary(prefix, **counts))
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (True, False)
|
||
|
||
|
||
@pytest.mark.parametrize("prefix", ["#", "ℹ"])
|
||
@pytest.mark.parametrize("counts", [{}, {"skipped": 2}, {"todo": 2}, {"skipped": 1, "todo": 1}])
|
||
def test_empty_skipped_and_todo_summaries_supply_no_proof(prefix, counts):
|
||
verdict = _verdict("ok 1 - all tests passed # SKIP\n" + _summary(prefix, passed=0, **counts))
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"output",
|
||
[
|
||
"ok 1 - sample",
|
||
"ok 12 # SKIP",
|
||
_summary(total=4),
|
||
_summary(passed="9" * 100, total=1),
|
||
_summary().replace("# fail 0\n", ""),
|
||
_summary().replace("# fail 0", "ℹ fail 0"),
|
||
_summary().replace("# pass 1", "# pass -1"),
|
||
_summary().split("# todo")[0],
|
||
],
|
||
)
|
||
def test_tap_points_and_incomplete_or_inconsistent_counts_remain_unverified(output):
|
||
assert _verdict(output)["leaves"][0]["checked"] is False
|
||
|
||
|
||
@pytest.mark.parametrize("reverse", [False, True])
|
||
def test_failing_summary_wins_over_successful_sibling(reverse):
|
||
summaries = [_summary(), _summary(failed=1)]
|
||
verdict = _verdict("\n".join(reversed(summaries) if reverse else summaries))
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (True, False)
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"overrides",
|
||
[{"shell_persistent": True}, {"shell_persistent": None}, {"command_truncated": True}, {"command": "node --test sample.test.cjs"}, {"command": "echo other; node --test"}],
|
||
)
|
||
def test_node_counts_do_not_bypass_execution_admission(overrides):
|
||
assert _verdict(_summary(), **overrides)["leaves"][0]["checked"] is False
|
||
|
||
|
||
def test_go_package_and_pytest_controls_still_pass():
|
||
assert _verdict("ok\texample.org/project\t0.005s")["all_hold"] is True
|
||
assert _verdict("ok\t123\t0.005s")["all_hold"] is True
|
||
assert _verdict("2 passed in 0.01s")["all_hold"] is True
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"point",
|
||
[
|
||
"ok 1 - all tests passed # SKIP",
|
||
"ok 12 # all tests passed",
|
||
"✔ all tests passed (1ms)",
|
||
"﹣ all tests passed (1ms)",
|
||
"▶ all tests passed",
|
||
"⚠ all tests passed",
|
||
" ok 1 - all tests passed # SKIP",
|
||
"# Subtest: all tests passed\nok 1 - all tests passed # SKIP",
|
||
],
|
||
)
|
||
@pytest.mark.parametrize("tail", ["", _summary(passed=0, skipped=1).split("# todo")[0]])
|
||
def test_point_labels_cannot_supply_pass_evidence(point, tail):
|
||
verdict = _verdict(point + "\n" + tail)
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)
|
||
|
||
|
||
def test_recorded_exit_error_overrides_node_positive_counts():
|
||
verdict = _verdict(_summary(), status="error")
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (True, False)
|
||
|
||
|
||
def test_newest_matching_execution_is_decisive():
|
||
first = {"command": "node --test", "status": "success", "shell_persistent": False, "output_tail": _summary(failed=1)}
|
||
second = {**first, "output_tail": _summary()}
|
||
assert check_acceptance_criteria(["tests_passed:node --test"], bash_executions=[first, second])["all_hold"] is True
|
||
assert check_acceptance_criteria(["tests_passed:node --test"], bash_executions=[second, first])["all_hold"] is False
|
||
|
||
|
||
def test_empty_node_sibling_does_not_veto_another_runner():
|
||
output = "2 passed in 0.01s\n" + _summary(passed=0)
|
||
execution = {"command": "make test", "status": "success", "shell_persistent": False, "output_tail": output}
|
||
assert check_acceptance_criteria(["tests_passed:make test"], bash_executions=[execution])["all_hold"] is True
|
||
|
||
|
||
@pytest.mark.parametrize("label", ["0 passed", "1 failed", "1 error"])
|
||
@pytest.mark.parametrize("point", ["# Subtest: {label}\n ok 1 - {label}", "⚠ {label} # TODO"])
|
||
def test_test_names_cannot_supply_failure_or_zero_evidence(label, point):
|
||
verdict = _verdict(point.format(label=label) + "\n" + _summary())
|
||
assert verdict["all_hold"] is True
|
||
|
||
|
||
@pytest.mark.parametrize("newline", ["\n", "\r\n"])
|
||
def test_todo_diagnostic_location_cannot_supply_error_count(newline):
|
||
output = "⚠ 1 failed # TODO\n" + _summary("ℹ", todo=1) + "\ntest at sample.test.cjs:1:57\n\n Error: fixture"
|
||
assert _verdict(output.replace("\n", newline))["all_hold"] is True
|
||
|
||
|
||
def test_color_codes_are_normalized_for_counts_and_output_admission():
|
||
output = "\x1b[90m﹣ all tests passed\x1b[39m\n" + "\n".join(f"\x1b[34m{line}\x1b[39m" for line in _summary("ℹ", passed=0, skipped=1).splitlines())
|
||
assert _verdict(output)["leaves"][0]["checked"] is False
|
||
assert _carries_summary_shape(output) is True
|
||
|
||
|
||
@pytest.fixture
|
||
def executor_module(monkeypatch):
|
||
path = Path(__file__).parents[1] / "packages/harness/deerflow/subagents/executor.py"
|
||
spec = importlib.util.spec_from_file_location("_node_evidence_executor", path)
|
||
module = importlib.util.module_from_spec(spec)
|
||
monkeypatch.setitem(sys.modules, spec.name, module)
|
||
spec.loader.exec_module(module)
|
||
return module
|
||
|
||
|
||
class SandboxMessages(MessagesState):
|
||
sandbox: dict
|
||
|
||
|
||
@pytest.mark.parametrize("label", ["all tests passed", "1 failed", "0 passed"])
|
||
def test_harvest_drops_a_partial_label_before_interpreting_counts(executor_module, monkeypatch, label):
|
||
output = "ok 1 - " + "x" * 1500 + " " + label + " # SKIP\n" + _summary(passed=0, skipped=1)
|
||
messages = [AIMessage(content="", tool_calls=[{"name": "bash", "id": "long-name", "args": {"command": "node --test"}}]), ToolMessage(content=output, tool_call_id="long-name")]
|
||
monkeypatch.setattr(executor_module, "_harvest_shell_persistence", lambda state: False)
|
||
executions = executor_module._harvest_bash_executions({"messages": messages})
|
||
assert executions[0]["output_tail"] == _summary(passed=0, skipped=1)
|
||
assert messages[-1].content == output
|
||
verdict = check_acceptance_criteria(["tests_passed:node --test"], bash_executions=executions)
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == (False, False)
|
||
|
||
|
||
@pytest.mark.skipif(os.name == "nt" or shutil.which("node") is None, reason="Requires POSIX LocalSandbox and Node.js")
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.parametrize("reporter", ["tap", "spec", "color-spec"])
|
||
@pytest.mark.parametrize(
|
||
"body,expected",
|
||
[
|
||
("test('sample', () => {});", (True, True)),
|
||
("test('0 passed', () => {}); test('1 failed', () => {});", (True, True)),
|
||
("test('sample', () => {}); test.skip('optional', () => {});", (True, True)),
|
||
("test.skip('all tests passed', () => {});", (False, False)),
|
||
("test.skip('x'.repeat(1500) + ' all tests passed', () => {});", (False, False)),
|
||
("test('x'.repeat(1500) + ' 1 failed', () => {});", (True, True)),
|
||
("test('x'.repeat(1500) + ' 0 passed', () => {});", (True, True)),
|
||
("test.todo('pending');", (False, False)),
|
||
("test('all tests passed', { todo: true }, () => { throw Error('fixture'); });", (False, False)),
|
||
("test('sample', () => {}); test('1 failed', { todo: true }, () => { throw Error('fixture'); });", (True, True)),
|
||
("describe('empty suite', () => {});", (False, False)),
|
||
("describe('all tests passed', () => { test.skip('optional', () => {}); });", (False, False)),
|
||
("describe('all tests passed', () => { test.todo('pending'); });", (False, False)),
|
||
("describe('1 failed', () => { test('sample', () => {}); });", (True, True)),
|
||
("test('sample', () => { throw Error('fixture'); });", (True, False)),
|
||
("test('pending', { timeout: 20 }, () => new Promise(() => {}));", (True, False)),
|
||
],
|
||
ids=[
|
||
"pass",
|
||
"summary-like-names",
|
||
"pass-and-skip",
|
||
"all-skipped",
|
||
"long-skipped",
|
||
"long-failure-name",
|
||
"long-zero-name",
|
||
"all-todo",
|
||
"failing-todo-pass-name",
|
||
"failing-todo-failure-name",
|
||
"empty-suite",
|
||
"skipped-suite",
|
||
"todo-suite",
|
||
"summary-like-suite",
|
||
"failure",
|
||
"cancelled",
|
||
],
|
||
)
|
||
async def test_real_node_results_reach_parent_acceptance(tmp_path, monkeypatch, executor_module, reporter, body, expected):
|
||
(tmp_path / "sample.test.cjs").write_text("const { test, describe } = require('node:test');\n" + body, encoding="utf-8")
|
||
sandbox = LocalSandbox("node-evidence", path_mappings=[PathMapping("/mnt/user-data/workspace", str(tmp_path))])
|
||
monkeypatch.setattr("deerflow.sandbox.sandbox_provider.get_sandbox_provider", lambda: SimpleNamespace(get=lambda sandbox_id: sandbox if sandbox_id == sandbox.id else None))
|
||
outputs = []
|
||
|
||
async def bash(command: str) -> str:
|
||
output = await asyncio.to_thread(sandbox.execute_command, command, timeout=30)
|
||
outputs.append(output)
|
||
return output
|
||
|
||
tool = StructuredTool.from_function(coroutine=bash, name="bash", description="Run offline native Node tests.")
|
||
graph = StateGraph(SandboxMessages)
|
||
graph.add_node("tools", ToolNode([tool], awrap_tool_call=ToolErrorHandlingMiddleware().awrap_tool_call))
|
||
graph.add_edge(START, "tools")
|
||
graph.add_edge("tools", END)
|
||
color = "FORCE_COLOR=1 " if reporter == "color-spec" else ""
|
||
reporter = reporter.removeprefix("color-")
|
||
command = f"{color}{shlex.quote(shutil.which('node'))} --test --test-reporter={reporter} /mnt/user-data/workspace/sample.test.cjs"
|
||
state = await graph.compile().ainvoke({"sandbox": {"sandbox_id": sandbox.id}, "messages": [AIMessage(content="", tool_calls=[{"name": "bash", "id": "node-call", "args": {"command": command}}])]})
|
||
assert len(outputs) == 1 and "duration_ms" in outputs[0], outputs
|
||
executions = executor_module._harvest_bash_executions(state)
|
||
assert len(executions) == 1 and executions[0]["shell_persistent"] is False
|
||
verdict = check_acceptance_criteria([f"tests_passed:{command}"], bash_executions=executions)
|
||
assert (verdict["leaves"][0]["checked"], verdict["all_hold"]) == expected, (outputs, verdict)
|