mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-16 16:31:18 +00:00
* fix(runtime): publish the delivery failure on the run stream * docs(api): note the delivery-failure stream event * test(runtime): cover the deferred delivery-failure frame * docs(runtime): log the delivery fix and explain the fallback exemption * test(e2e): present the recorded output in the replay scenario The write/read-file fixture predates the delivery check, so the replayed run has been ending as an error without presenting note.txt. This PR makes that failure visible on the stream, which broke the replay golden and the render e2e. Add the present_files step the current scenario produces and regenerate the golden. * docs: drop the CHANGELOG entry CONTRIBUTING.md asks contributors to leave CHANGELOG.md and CHANGELOG_zh.md to the maintainers at merge time, so the Unreleased ordering and the English/Chinese pair stay in sync.
1453 lines
55 KiB
Python
1453 lines
55 KiB
Python
"""Worker-level regression tests for the terminal run.delivery event (#4272 slice 1)."""
|
|
|
|
import asyncio
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
|
from langgraph.graph import END, START, StateGraph
|
|
from langgraph.types import Command, Interrupt, interrupt
|
|
|
|
from deerflow.config.app_config import AppConfig
|
|
from deerflow.config.paths import Paths
|
|
from deerflow.config.run_ownership_config import RunOwnershipConfig
|
|
from deerflow.config.sandbox_config import SandboxConfig
|
|
from deerflow.config.tool_output_config import ToolOutputConfig
|
|
from deerflow.runtime.events.store.memory import MemoryRunEventStore
|
|
from deerflow.runtime.runs.manager import RunManager
|
|
from deerflow.runtime.runs.schemas import RunStatus
|
|
from deerflow.runtime.runs.store.memory import MemoryRunStore
|
|
from deerflow.runtime.runs.worker import RunContext, _delivery_content_with_outputs, run_agent
|
|
from deerflow.runtime.user_context import get_effective_user_id
|
|
from deerflow.workspace_changes import WorkspaceChangeLimits, capture_workspace_snapshot
|
|
from deerflow.workspace_changes.api import get_workspace_changes_response
|
|
|
|
|
|
def _make_bridge():
|
|
return SimpleNamespace(publish=AsyncMock(), publish_end=AsyncMock(), cleanup=AsyncMock())
|
|
|
|
|
|
class _RecordingBridge:
|
|
"""A stream bridge that keeps every published event in order."""
|
|
|
|
def __init__(self):
|
|
self.events: list[tuple[str, object]] = []
|
|
|
|
async def publish(self, run_id, event, payload=None, **kwargs):
|
|
self.events.append((event, payload))
|
|
|
|
async def publish_end(self, run_id, **kwargs):
|
|
self.events.append(("end", None))
|
|
|
|
async def cleanup(self, run_id, **kwargs):
|
|
return None
|
|
|
|
|
|
async def _delivery_events(store: MemoryRunEventStore, thread_id: str, run_id: str) -> list[dict]:
|
|
events = await store.list_events(thread_id, run_id)
|
|
return [e for e in events if e["event_type"] == "run.delivery"]
|
|
|
|
|
|
def test_delivery_verification_treats_presented_directory_as_covering_produced_files():
|
|
content = {
|
|
"presented": 1,
|
|
"paths": ["/mnt/user-data/outputs/site"],
|
|
"by_tool": {"present_files": ["/mnt/user-data/outputs/site"]},
|
|
}
|
|
|
|
delivery = _delivery_content_with_outputs(
|
|
content,
|
|
[
|
|
"/mnt/user-data/outputs/site/index.html",
|
|
"/mnt/user-data/outputs/site/assets/style.css",
|
|
],
|
|
)
|
|
|
|
assert delivery["matched_paths"] == [
|
|
"/mnt/user-data/outputs/site/index.html",
|
|
"/mnt/user-data/outputs/site/assets/style.css",
|
|
]
|
|
assert delivery["satisfied"] is True
|
|
|
|
|
|
@pytest.mark.anyio
|
|
@pytest.mark.parametrize("disappear_during", ["before-run", "after-run"])
|
|
async def test_vanished_scratch_file_does_not_discard_run_file_changes(tmp_path, monkeypatch, disappear_during):
|
|
import deerflow.workspace_changes.scanner as scanner
|
|
|
|
paths = Paths(base_dir=tmp_path)
|
|
monkeypatch.setattr("deerflow.workspace_changes.recorder.get_paths", lambda: paths)
|
|
user_id = get_effective_user_id()
|
|
workspace = paths.sandbox_work_dir("thread-1", user_id=user_id)
|
|
outputs = paths.sandbox_outputs_dir("thread-1", user_id=user_id)
|
|
workspace.mkdir(parents=True)
|
|
outputs.mkdir(parents=True)
|
|
scratch = workspace / "scratch.txt"
|
|
if disappear_during == "before-run":
|
|
scratch.write_text("temporary\n", encoding="utf-8")
|
|
removed = []
|
|
real_hash = scanner._sha256_file
|
|
|
|
def remove_before_hash(path):
|
|
if path == scratch:
|
|
path.unlink()
|
|
removed.append(path)
|
|
return real_hash(path)
|
|
|
|
monkeypatch.setattr(scanner, "_sha256_file", remove_before_hash)
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
|
|
class ProducingAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
if disappear_during == "after-run":
|
|
scratch.write_text("temporary\n", encoding="utf-8")
|
|
(outputs / "report.md").write_text("Report ready\n", encoding="utf-8")
|
|
yield {"messages": [AIMessage(content="Created report.md.")]}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: ProducingAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
assert removed == [scratch]
|
|
changes = await get_workspace_changes_response(store, "thread-1", record.run_id)
|
|
assert changes["available"] is True
|
|
assert [(change["path"], change["status"]) for change in changes["files"]] == [("/mnt/user-data/outputs/report.md", "created")]
|
|
assert "+Report ready" in changes["files"][0]["diff"]
|
|
delivery = await _delivery_events(store, "thread-1", record.run_id)
|
|
assert delivery[0]["content"]["produced_paths"] == ["/mnt/user-data/outputs/report.md"]
|
|
assert delivery[0]["content"]["satisfied"] is False
|
|
assert record.status == RunStatus.error
|
|
assert record.error == "Artifact delivery incomplete: no produced output artifact was presented"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_delivery_event_records_present_files_paths_on_success():
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
|
|
class DummyAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
journal = config["context"]["__run_journal"]
|
|
ai = AIMessage(content="", tool_calls=[{"id": "call_1", "name": "present_files", "args": {}}])
|
|
journal._remember_current_run_tool_calls(ai, caller="lead_agent")
|
|
journal.on_tool_end(
|
|
Command(
|
|
update={
|
|
"artifacts": ["/mnt/user-data/outputs/report.md"],
|
|
"messages": [ToolMessage("Successfully presented files", tool_call_id="call_1")],
|
|
}
|
|
),
|
|
run_id=uuid4(),
|
|
)
|
|
yield {"messages": []}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: DummyAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
await asyncio.sleep(0)
|
|
|
|
delivery = await _delivery_events(store, "thread-1", record.run_id)
|
|
assert len(delivery) == 1
|
|
assert delivery[0]["content"]["presented"] == 1
|
|
assert delivery[0]["content"]["paths"] == ["/mnt/user-data/outputs/report.md"]
|
|
assert delivery[0]["content"]["by_tool"] == {"present_files": ["/mnt/user-data/outputs/report.md"]}
|
|
fetched = await run_manager.get(record.run_id)
|
|
assert fetched.status == RunStatus.success
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_delivery_event_presented_zero_without_artifact_production():
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
|
|
class DummyAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
yield {"messages": []}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: DummyAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
await asyncio.sleep(0)
|
|
|
|
delivery = await _delivery_events(store, "thread-1", record.run_id)
|
|
assert len(delivery) == 1
|
|
assert delivery[0]["content"] == {"presented": 0, "paths": [], "by_tool": {}}
|
|
fetched = await run_manager.get(record.run_id)
|
|
assert fetched.status == RunStatus.success
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_changed_outputs_succeed_when_a_produced_output_is_presented(monkeypatch):
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
monkeypatch.setattr(
|
|
"deerflow.runtime.runs.worker._produced_output_paths",
|
|
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
|
|
)
|
|
|
|
class DummyAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
journal = config["context"]["__run_journal"]
|
|
ai = AIMessage(content="", tool_calls=[{"id": "call_1", "name": "present_files", "args": {}}])
|
|
journal._remember_current_run_tool_calls(ai, caller="lead_agent")
|
|
journal.on_tool_end(
|
|
Command(
|
|
update={
|
|
"artifacts": ["/mnt/user-data/outputs/report.md"],
|
|
"messages": [ToolMessage("Successfully presented files", tool_call_id="call_1")],
|
|
}
|
|
),
|
|
run_id=uuid4(),
|
|
)
|
|
yield {"messages": []}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: DummyAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
delivery = await _delivery_events(store, "thread-1", record.run_id)
|
|
assert delivery[0]["content"] == {
|
|
"presented": 1,
|
|
"paths": ["/mnt/user-data/outputs/report.md"],
|
|
"by_tool": {"present_files": ["/mnt/user-data/outputs/report.md"]},
|
|
"verification": {
|
|
"source": "outputs_changed",
|
|
"requirement": "verified_presentation_matches_produced_output",
|
|
},
|
|
"produced_paths": ["/mnt/user-data/outputs/report.md"],
|
|
"presented_paths": ["/mnt/user-data/outputs/report.md"],
|
|
"matched_paths": ["/mnt/user-data/outputs/report.md"],
|
|
"stage": "presented",
|
|
"satisfied": True,
|
|
}
|
|
assert record.status == RunStatus.success
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_changed_outputs_fail_closed_when_not_presented(monkeypatch):
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
monkeypatch.setattr(
|
|
"deerflow.runtime.runs.worker._produced_output_paths",
|
|
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
|
|
)
|
|
|
|
class ProseOnlyAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
yield {"messages": [AIMessage(content="SESSION SUMMARY")]}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: ProseOnlyAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
delivery = await _delivery_events(store, "thread-1", record.run_id)
|
|
assert delivery[0]["content"] == {
|
|
"presented": 0,
|
|
"paths": [],
|
|
"by_tool": {},
|
|
"verification": {
|
|
"source": "outputs_changed",
|
|
"requirement": "verified_presentation_matches_produced_output",
|
|
},
|
|
"produced_paths": ["/mnt/user-data/outputs/report.md"],
|
|
"presented_paths": [],
|
|
"matched_paths": [],
|
|
"stage": "not_started",
|
|
"satisfied": False,
|
|
}
|
|
assert record.status == RunStatus.error
|
|
assert record.error == "Artifact delivery incomplete: no produced output artifact was presented"
|
|
assert record.stop_reason is None
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_unpresented_outputs_publish_an_error_frame_before_end(monkeypatch):
|
|
"""A run failed by delivery verification must say so on the stream.
|
|
|
|
The record and ``/wait`` report the failure; the web UI raises its run-failure
|
|
toast and streaming IM channels mark a turn failed only on an ``error`` frame,
|
|
so without one they present the turn as a successful answer.
|
|
"""
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
monkeypatch.setattr(
|
|
"deerflow.runtime.runs.worker._produced_output_paths",
|
|
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
|
|
)
|
|
bridge = _RecordingBridge()
|
|
|
|
class ProseOnlyAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
yield {"messages": [AIMessage(content="SESSION SUMMARY")]}
|
|
|
|
await run_agent(
|
|
bridge,
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: ProseOnlyAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
assert record.status == RunStatus.error
|
|
assert [payload for event, payload in bridge.events if event == "error"] == [
|
|
{
|
|
"message": "Artifact delivery incomplete: no produced output artifact was presented",
|
|
"name": "RunError",
|
|
}
|
|
]
|
|
assert bridge.events[-1][0] == "end"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_unpersisted_delivery_receipt_publishes_an_error_frame_before_end(monkeypatch):
|
|
"""The receipt failure also flips the record to ``error``; the stream must follow."""
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
monkeypatch.setattr(
|
|
"deerflow.runtime.runs.worker._produced_output_paths",
|
|
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
|
|
)
|
|
monkeypatch.setattr(
|
|
"deerflow.runtime.runs.worker._persist_delivery_receipt",
|
|
AsyncMock(return_value=False),
|
|
)
|
|
bridge = _RecordingBridge()
|
|
|
|
class PresentingAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
journal = config["context"]["__run_journal"]
|
|
ai = AIMessage(content="", tool_calls=[{"id": "call_1", "name": "present_files", "args": {}}])
|
|
journal._remember_current_run_tool_calls(ai, caller="lead_agent")
|
|
journal.on_tool_end(
|
|
Command(
|
|
update={
|
|
"artifacts": ["/mnt/user-data/outputs/report.md"],
|
|
"messages": [ToolMessage("Successfully presented files", tool_call_id="call_1")],
|
|
}
|
|
),
|
|
run_id=uuid4(),
|
|
)
|
|
yield {"messages": []}
|
|
|
|
await run_agent(
|
|
bridge,
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: PresentingAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
assert record.status == RunStatus.error
|
|
assert record.error == "Artifact delivery verification failed: terminal delivery receipt could not be persisted"
|
|
assert [payload for event, payload in bridge.events if event == "error"] == [
|
|
{
|
|
"message": "Artifact delivery verification failed: terminal delivery receipt could not be persisted",
|
|
"name": "RunError",
|
|
}
|
|
]
|
|
assert bridge.events[-1][0] == "end"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_unconfirmed_delivery_terminal_status_defers_the_error_frame_until_end(monkeypatch):
|
|
"""A delivery failure defers the frame while its terminal status is not durable.
|
|
|
|
The worker publishes the ``error`` frame directly only once the record is
|
|
terminal *and* persisted, and hands it to the finalizer otherwise. A
|
|
store-backed run whose first terminal write fails (a transient store outage)
|
|
therefore publishes the frame after the finalizer's retry confirmed the row —
|
|
and only because that retry succeeded, so a stream that never reaches ``end``
|
|
never carries the frame.
|
|
"""
|
|
order: list[str] = []
|
|
|
|
class FlakyTerminalRunStore(MemoryRunStore):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.terminal_write_attempts = 0
|
|
|
|
async def update_status(self, run_id, status, *, error=None, stop_reason=None, **kwargs):
|
|
terminal = status not in {"pending", "running"}
|
|
if terminal:
|
|
self.terminal_write_attempts += 1
|
|
if self.terminal_write_attempts == 1:
|
|
raise RuntimeError("transient run store outage")
|
|
updated = await super().update_status(run_id, status, error=error, stop_reason=stop_reason, **kwargs)
|
|
if terminal and updated:
|
|
order.append("durable-error")
|
|
return updated
|
|
|
|
class OrderingBridge:
|
|
async def publish(self, _run_id, event, payload=None, **kwargs):
|
|
if event == "error":
|
|
order.append("stream-error")
|
|
|
|
async def publish_end(self, _run_id, **kwargs):
|
|
order.append("stream-end")
|
|
|
|
async def cleanup(self, _run_id, **kwargs):
|
|
return None
|
|
|
|
run_store = FlakyTerminalRunStore()
|
|
run_manager = RunManager(store=run_store)
|
|
record = await run_manager.create("thread-1")
|
|
monkeypatch.setattr(
|
|
"deerflow.runtime.runs.worker._produced_output_paths",
|
|
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
|
|
)
|
|
|
|
class ProseOnlyAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
yield {"messages": [AIMessage(content="SESSION SUMMARY")]}
|
|
|
|
await run_agent(
|
|
OrderingBridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None),
|
|
agent_factory=lambda *, config: ProseOnlyAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
assert run_store.terminal_write_attempts == 2
|
|
assert order == ["durable-error", "stream-error", "stream-end"]
|
|
assert record.status == RunStatus.error
|
|
assert (await run_store.get(record.run_id))["status"] == "error"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
@pytest.mark.parametrize("modify_existing", [False, True], ids=["unproven-new-output", "observed-modified-output"])
|
|
@pytest.mark.parametrize("terminal_outcome", ["success", "error", "cancelled"])
|
|
async def test_truncated_snapshots_limit_output_delivery_verification(tmp_path, monkeypatch, modify_existing, terminal_outcome):
|
|
paths = Paths(base_dir=tmp_path)
|
|
monkeypatch.setattr("deerflow.workspace_changes.recorder.get_paths", lambda: paths)
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
outputs = paths.sandbox_outputs_dir("thread-1", user_id=get_effective_user_id())
|
|
outputs.mkdir(parents=True)
|
|
for name in ["b.txt", "c.txt", "d.txt"]:
|
|
(outputs / name).write_text("before\n", encoding="utf-8")
|
|
|
|
snapshots = []
|
|
|
|
async def capture_bounded_snapshot(*args, **kwargs):
|
|
snapshot = await capture_workspace_snapshot(*args, limits=WorkspaceChangeLimits(max_scanned_files=2), **kwargs)
|
|
snapshots.append(snapshot)
|
|
return snapshot
|
|
|
|
monkeypatch.setattr("deerflow.runtime.runs.worker.capture_workspace_snapshot", capture_bounded_snapshot)
|
|
|
|
class ProducingAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
(outputs / "a.txt").write_text("new output\n", encoding="utf-8")
|
|
if modify_existing:
|
|
(outputs / "b.txt").write_text("modified\n", encoding="utf-8")
|
|
if terminal_outcome == "error":
|
|
raise RuntimeError("Agent failed after producing output")
|
|
if terminal_outcome == "cancelled":
|
|
raise asyncio.CancelledError
|
|
yield {"messages": [AIMessage(content="Created the output.")]}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: ProducingAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
assert snapshots[0].truncated is True
|
|
assert snapshots[-1].truncated is True
|
|
assert "/mnt/user-data/outputs/a.txt" not in snapshots[0].files
|
|
assert "/mnt/user-data/outputs/a.txt" in snapshots[-1].files
|
|
assert (outputs / "a.txt").is_file()
|
|
delivery = await _delivery_events(store, "thread-1", record.run_id)
|
|
assert len(delivery) == 1
|
|
content = delivery[0]["content"]
|
|
if modify_existing:
|
|
assert content["produced_paths"] == ["/mnt/user-data/outputs/b.txt"]
|
|
assert content["verification"]["source"] == "outputs_changed"
|
|
assert content["satisfied"] is False
|
|
assert "verification_skipped" not in content
|
|
else:
|
|
assert content == {"presented": 0, "paths": [], "by_tool": {}, "verification_skipped": "truncated_pre_run_snapshot"}
|
|
if terminal_outcome == "error":
|
|
assert record.status == RunStatus.error
|
|
assert record.error == "Agent failed after producing output"
|
|
elif terminal_outcome == "cancelled":
|
|
assert record.status == RunStatus.interrupted
|
|
elif modify_existing:
|
|
assert record.status == RunStatus.error
|
|
assert record.error == "Artifact delivery incomplete: no produced output artifact was presented"
|
|
else:
|
|
assert record.status == RunStatus.success
|
|
assert record.error is None
|
|
|
|
|
|
@pytest.mark.anyio
|
|
@pytest.mark.parametrize("failure_stage", ["before", "after"])
|
|
async def test_snapshot_capture_failure_does_not_claim_truncated_verification(tmp_path, monkeypatch, failure_stage):
|
|
paths = Paths(base_dir=tmp_path)
|
|
monkeypatch.setattr("deerflow.workspace_changes.recorder.get_paths", lambda: paths)
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
outputs = paths.sandbox_outputs_dir("thread-1", user_id=get_effective_user_id())
|
|
outputs.mkdir(parents=True)
|
|
captures = 0
|
|
|
|
async def capture_or_fail(*args, **kwargs):
|
|
nonlocal captures
|
|
captures += 1
|
|
if captures == (1 if failure_stage == "before" else 2):
|
|
raise OSError("Snapshot capture unavailable")
|
|
return await capture_workspace_snapshot(*args, **kwargs)
|
|
|
|
monkeypatch.setattr("deerflow.runtime.runs.worker.capture_workspace_snapshot", capture_or_fail)
|
|
|
|
class ProducingAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
(outputs / "report.md").write_text("Report ready\n", encoding="utf-8")
|
|
yield {"messages": [AIMessage(content="Created report.md.")]}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: ProducingAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
assert captures == (1 if failure_stage == "before" else 2)
|
|
assert (outputs / "report.md").is_file()
|
|
delivery = await _delivery_events(store, "thread-1", record.run_id)
|
|
assert len(delivery) == 1
|
|
assert delivery[0]["content"] == {"presented": 0, "paths": [], "by_tool": {}}
|
|
assert record.status == RunStatus.success
|
|
assert record.error is None
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_externalized_tool_results_do_not_trigger_delivery_verification(tmp_path, monkeypatch):
|
|
"""Oversized tool outputs externalized under outputs/.tool-results/ are
|
|
process feedback for the model, not deliverables: a run that only produced
|
|
those files must succeed without any present_files call."""
|
|
paths = Paths(base_dir=tmp_path)
|
|
monkeypatch.setattr("deerflow.workspace_changes.recorder.get_paths", lambda: paths)
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
|
|
class ExternalizingAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
# Simulates ToolOutputBudgetMiddleware persisting an oversized tool
|
|
# output mid-run (default storage_subdir is ".tool-results").
|
|
tool_results = paths.sandbox_outputs_dir("thread-1", user_id=get_effective_user_id()) / ".tool-results"
|
|
tool_results.mkdir(parents=True, exist_ok=True)
|
|
(tool_results / "bash-abcdef123456.log").write_text("x" * 20000, encoding="utf-8")
|
|
yield {"messages": [AIMessage(content="Here is the answer.")]}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: ExternalizingAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
delivery = await _delivery_events(store, "thread-1", record.run_id)
|
|
assert len(delivery) == 1
|
|
assert delivery[0]["content"] == {"presented": 0, "paths": [], "by_tool": {}}
|
|
fetched = await run_manager.get(record.run_id)
|
|
assert fetched.status == RunStatus.success
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_custom_tool_output_storage_subdir_does_not_trigger_delivery_verification(tmp_path, monkeypatch):
|
|
"""A custom tool_output.storage_subdir is honoured by the exclusion, not
|
|
only the default .tool-results name."""
|
|
paths = Paths(base_dir=tmp_path)
|
|
monkeypatch.setattr("deerflow.workspace_changes.recorder.get_paths", lambda: paths)
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
app_config = AppConfig(sandbox=SandboxConfig(use="test"), tool_output=ToolOutputConfig(storage_subdir="tool-output-cache"))
|
|
|
|
class ExternalizingAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
cache = paths.sandbox_outputs_dir("thread-1", user_id=get_effective_user_id()) / "tool-output-cache"
|
|
cache.mkdir(parents=True, exist_ok=True)
|
|
(cache / "web_fetch-abcdef123456.log").write_text("y" * 20000, encoding="utf-8")
|
|
yield {"messages": [AIMessage(content="Here is the answer.")]}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store, app_config=app_config),
|
|
agent_factory=lambda *, config: ExternalizingAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
fetched = await run_manager.get(record.run_id)
|
|
assert fetched.status == RunStatus.success
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_changed_outputs_succeed_when_one_of_multiple_outputs_is_presented(monkeypatch):
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
monkeypatch.setattr(
|
|
"deerflow.runtime.runs.worker._produced_output_paths",
|
|
AsyncMock(
|
|
return_value=[
|
|
"/mnt/user-data/outputs/report.md",
|
|
"/mnt/user-data/outputs/appendix.md",
|
|
]
|
|
),
|
|
)
|
|
|
|
class PartiallyPresentingAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
journal = config["context"]["__run_journal"]
|
|
journal._remember_current_run_tool_calls(
|
|
AIMessage(content="", tool_calls=[{"id": "call_1", "name": "present_files", "args": {}}]),
|
|
caller="lead_agent",
|
|
)
|
|
journal.on_tool_end(
|
|
Command(
|
|
update={
|
|
"artifacts": ["/mnt/user-data/outputs/report.md"],
|
|
"messages": [ToolMessage("Successfully presented files", tool_call_id="call_1")],
|
|
}
|
|
),
|
|
run_id=uuid4(),
|
|
)
|
|
yield {"messages": []}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: PartiallyPresentingAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
delivery = (await _delivery_events(store, "thread-1", record.run_id))[0]["content"]
|
|
assert delivery["stage"] == "presented"
|
|
assert delivery["presented_paths"] == ["/mnt/user-data/outputs/report.md"]
|
|
assert delivery["matched_paths"] == ["/mnt/user-data/outputs/report.md"]
|
|
assert delivery["satisfied"] is True
|
|
assert record.status == RunStatus.success
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_changed_outputs_fail_when_present_files_only_presents_an_unrelated_file(monkeypatch):
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
monkeypatch.setattr(
|
|
"deerflow.runtime.runs.worker._produced_output_paths",
|
|
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
|
|
)
|
|
|
|
class UnrelatedPresentingAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
journal = config["context"]["__run_journal"]
|
|
journal._remember_current_run_tool_calls(
|
|
AIMessage(content="", tool_calls=[{"id": "call_1", "name": "present_files", "args": {}}]),
|
|
caller="lead_agent",
|
|
)
|
|
journal.on_tool_end(
|
|
Command(
|
|
update={
|
|
"artifacts": ["/mnt/user-data/outputs/old-report.md"],
|
|
"messages": [ToolMessage("Successfully presented files", tool_call_id="call_1")],
|
|
}
|
|
),
|
|
run_id=uuid4(),
|
|
)
|
|
yield {"messages": []}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: UnrelatedPresentingAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
delivery = (await _delivery_events(store, "thread-1", record.run_id))[0]["content"]
|
|
assert delivery["stage"] == "mismatched"
|
|
assert delivery["presented_paths"] == ["/mnt/user-data/outputs/old-report.md"]
|
|
assert delivery["matched_paths"] == []
|
|
assert delivery["satisfied"] is False
|
|
assert record.status == RunStatus.error
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_fenced_worker_leaves_delivery_receipt_to_peer_recovery():
|
|
"""A stale worker must not finalize the singleton delivery receipt."""
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-lease-lost")
|
|
record.ownership_lost = True
|
|
record.abort_event.set()
|
|
record.status = RunStatus.error
|
|
event_store = MemoryRunEventStore()
|
|
thread_store = SimpleNamespace(
|
|
update_display_name=AsyncMock(),
|
|
update_status=AsyncMock(),
|
|
)
|
|
on_run_completed = AsyncMock()
|
|
agent_factory = MagicMock(side_effect=AssertionError("fenced worker started the agent"))
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(
|
|
checkpointer=None,
|
|
event_store=event_store,
|
|
thread_store=thread_store,
|
|
on_run_completed=on_run_completed,
|
|
),
|
|
agent_factory=agent_factory,
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
assert await _delivery_events(event_store, record.thread_id, record.run_id) == []
|
|
agent_factory.assert_not_called()
|
|
thread_store.update_display_name.assert_not_awaited()
|
|
thread_store.update_status.assert_not_awaited()
|
|
on_run_completed.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_delivery_event_is_singleton_across_goal_continuations(monkeypatch):
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
stream_calls = 0
|
|
continuation_calls = 0
|
|
|
|
class ContinuingAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
nonlocal stream_calls
|
|
stream_calls += 1
|
|
journal = config["context"]["__run_journal"]
|
|
tool_call_id = f"call_{stream_calls}"
|
|
journal._remember_current_run_tool_calls(
|
|
AIMessage(content="", tool_calls=[{"id": tool_call_id, "name": "present_files", "args": {}}]),
|
|
caller="lead_agent",
|
|
)
|
|
artifacts = ["/mnt/user-data/outputs/report.md"]
|
|
if stream_calls == 2:
|
|
artifacts.append("/mnt/user-data/outputs/appendix.md")
|
|
journal.on_tool_end(
|
|
Command(
|
|
update={
|
|
"artifacts": artifacts,
|
|
"messages": [ToolMessage("Successfully presented files", tool_call_id=tool_call_id)],
|
|
}
|
|
),
|
|
run_id=uuid4(),
|
|
)
|
|
yield {"messages": []}
|
|
|
|
async def prepare_continuation(**kwargs):
|
|
nonlocal continuation_calls
|
|
continuation_calls += 1
|
|
if continuation_calls == 1:
|
|
return {"messages": []}
|
|
return None
|
|
|
|
monkeypatch.setattr("deerflow.runtime.runs.worker._prepare_goal_continuation_input", prepare_continuation)
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: ContinuingAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
delivery = await _delivery_events(store, "thread-1", record.run_id)
|
|
assert stream_calls == 2
|
|
assert len(delivery) == 1
|
|
assert delivery[0]["content"] == {
|
|
"presented": 2,
|
|
"paths": [
|
|
"/mnt/user-data/outputs/report.md",
|
|
"/mnt/user-data/outputs/appendix.md",
|
|
],
|
|
"by_tool": {
|
|
"present_files": [
|
|
"/mnt/user-data/outputs/report.md",
|
|
"/mnt/user-data/outputs/appendix.md",
|
|
]
|
|
},
|
|
}
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_delivery_event_emitted_exactly_once_on_error_path():
|
|
run_manager = RunManager()
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
|
|
class FailingAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
raise RuntimeError("boom")
|
|
yield # pragma: no cover - make this an async generator
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=lambda *, config: FailingAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
await asyncio.sleep(0)
|
|
|
|
delivery = await _delivery_events(store, "thread-1", record.run_id)
|
|
assert len(delivery) == 1
|
|
assert delivery[0]["content"]["presented"] == 0
|
|
fetched = await run_manager.get(record.run_id)
|
|
assert fetched.status == RunStatus.error
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_delivery_is_durable_before_terminal_run_status():
|
|
events = MemoryRunEventStore()
|
|
|
|
class OrderingRunStore(MemoryRunStore):
|
|
async def update_status(self, run_id, status, *, error=None, stop_reason=None):
|
|
if status not in {"pending", "running"}:
|
|
receipt = await events.list_events("thread-1", run_id, event_types=["run.delivery"])
|
|
assert len(receipt) == 1
|
|
return await super().update_status(run_id, status, error=error, stop_reason=stop_reason)
|
|
|
|
run_store = OrderingRunStore()
|
|
run_manager = RunManager(store=run_store)
|
|
record = await run_manager.create("thread-1")
|
|
|
|
class DummyAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
yield {"messages": []}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=events),
|
|
agent_factory=lambda *, config: DummyAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
assert (await run_store.get(record.run_id))["status"] == "success"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_deferred_error_stream_frames_follow_durable_terminal_status():
|
|
events = MemoryRunEventStore()
|
|
order: list[str] = []
|
|
|
|
class OrderingRunStore(MemoryRunStore):
|
|
async def update_status(self, run_id, status, *, error=None, stop_reason=None):
|
|
result = await super().update_status(
|
|
run_id,
|
|
status,
|
|
error=error,
|
|
stop_reason=stop_reason,
|
|
)
|
|
if status == "error":
|
|
order.append("durable-error")
|
|
return result
|
|
|
|
class OrderingBridge:
|
|
async def publish(self, _run_id, event, _data):
|
|
if event == "error":
|
|
order.append("stream-error")
|
|
|
|
async def publish_end(self, _run_id):
|
|
order.append("stream-end")
|
|
|
|
async def cleanup(self, _run_id, *, delay=0):
|
|
del delay
|
|
|
|
class FailingAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
del graph_input, config, stream_mode, subgraphs
|
|
raise RuntimeError("boom")
|
|
yield # pragma: no cover - make this an async generator
|
|
|
|
run_store = OrderingRunStore()
|
|
run_manager = RunManager(store=run_store)
|
|
record = await run_manager.create("thread-1")
|
|
|
|
await run_agent(
|
|
OrderingBridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=events),
|
|
agent_factory=lambda *, config: FailingAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
assert order == ["durable-error", "stream-error", "stream-end"]
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_delivery_write_retries_before_persisting_success():
|
|
class FlakyReceiptStore(MemoryRunEventStore):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.attempts = 0
|
|
|
|
async def put_if_absent(self, **kwargs):
|
|
if kwargs["event_type"] == "run.delivery":
|
|
self.attempts += 1
|
|
if self.attempts == 1:
|
|
raise RuntimeError("transient event store outage")
|
|
return await super().put_if_absent(**kwargs)
|
|
|
|
event_store = FlakyReceiptStore()
|
|
run_store = MemoryRunStore()
|
|
run_manager = RunManager(store=run_store)
|
|
record = await run_manager.create("thread-1")
|
|
|
|
class DummyAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
yield {"messages": []}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=event_store),
|
|
agent_factory=lambda *, config: DummyAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
assert event_store.attempts == 2
|
|
assert len(await _delivery_events(event_store, "thread-1", record.run_id)) == 1
|
|
assert (await run_store.get(record.run_id))["status"] == "success"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_delivery_write_failure_preserves_real_durable_terminal_status():
|
|
class FailingReceiptStore(MemoryRunEventStore):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.attempts = 0
|
|
|
|
async def put_if_absent(self, **kwargs):
|
|
self.attempts += 1
|
|
raise RuntimeError("event store unavailable")
|
|
|
|
run_store = MemoryRunStore()
|
|
run_manager = RunManager(store=run_store)
|
|
record = await run_manager.create("thread-1")
|
|
event_store = FailingReceiptStore()
|
|
|
|
class DummyAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
yield {"messages": []}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=event_store),
|
|
agent_factory=lambda *, config: DummyAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
# A receipt outage must not let lease recovery rewrite a genuine success
|
|
# as an error. After bounded retries, preserve the worker's real outcome.
|
|
assert event_store.attempts > 1
|
|
assert record.status == RunStatus.success
|
|
assert (await run_store.get(record.run_id))["status"] == "success"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_produced_artifact_delivery_fails_closed_when_receipt_cannot_be_persisted(monkeypatch):
|
|
class FailingReceiptStore(MemoryRunEventStore):
|
|
async def put_if_absent(self, **kwargs):
|
|
raise RuntimeError("event store unavailable")
|
|
|
|
run_store = MemoryRunStore()
|
|
run_manager = RunManager(store=run_store)
|
|
record = await run_manager.create("thread-1")
|
|
monkeypatch.setattr(
|
|
"deerflow.runtime.runs.worker._produced_output_paths",
|
|
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
|
|
)
|
|
|
|
class PresentingAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
journal = config["context"]["__run_journal"]
|
|
journal._remember_current_run_tool_calls(
|
|
AIMessage(content="", tool_calls=[{"id": "call_1", "name": "present_files", "args": {}}]),
|
|
caller="lead_agent",
|
|
)
|
|
journal.on_tool_end(
|
|
Command(
|
|
update={
|
|
"artifacts": ["/mnt/user-data/outputs/report.md"],
|
|
"messages": [ToolMessage("Successfully presented files", tool_call_id="call_1")],
|
|
}
|
|
),
|
|
run_id=uuid4(),
|
|
)
|
|
yield {"messages": []}
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=FailingReceiptStore()),
|
|
agent_factory=lambda *, config: PresentingAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
assert record.status == RunStatus.error
|
|
assert record.error == "Artifact delivery verification failed: terminal delivery receipt could not be persisted"
|
|
assert (await run_store.get(record.run_id))["status"] == "error"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_delivery_event_emitted_when_checkpoint_preflight_fails(monkeypatch):
|
|
run_manager = RunManager()
|
|
run_manager.update_run_completion = AsyncMock(wraps=run_manager.update_run_completion)
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
compatibility_check = AsyncMock(side_effect=RuntimeError("incompatible checkpoint"))
|
|
monkeypatch.setattr("deerflow.runtime.runs.worker.aensure_checkpoint_mode_compatible", compatibility_check)
|
|
|
|
def unexpected_agent_factory(**kwargs):
|
|
raise AssertionError("agent must not be built after preflight failure")
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=object(), event_store=store),
|
|
agent_factory=unexpected_agent_factory,
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
delivery = await _delivery_events(store, "thread-1", record.run_id)
|
|
assert len(delivery) == 1
|
|
assert delivery[0]["content"] == {"presented": 0, "paths": [], "by_tool": {}}
|
|
fetched = await run_manager.get(record.run_id)
|
|
assert fetched.status == RunStatus.error
|
|
run_manager.update_run_completion.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_delivery_event_emitted_when_cancelled_waiting_for_prior_finalization(monkeypatch):
|
|
run_manager = RunManager()
|
|
run_manager.update_run_completion = AsyncMock(wraps=run_manager.update_run_completion)
|
|
record = await run_manager.create("thread-1")
|
|
store = MemoryRunEventStore()
|
|
monkeypatch.setattr(
|
|
run_manager,
|
|
"wait_for_prior_finalizing",
|
|
AsyncMock(side_effect=asyncio.CancelledError()),
|
|
)
|
|
|
|
def unexpected_agent_factory(**kwargs):
|
|
raise AssertionError("agent must not be built after preflight cancellation")
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=None, event_store=store),
|
|
agent_factory=unexpected_agent_factory,
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
delivery = await _delivery_events(store, "thread-1", record.run_id)
|
|
assert len(delivery) == 1
|
|
assert delivery[0]["content"] == {"presented": 0, "paths": [], "by_tool": {}}
|
|
fetched = await run_manager.get(record.run_id)
|
|
assert fetched.status == RunStatus.interrupted
|
|
run_manager.update_run_completion.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_pending_snapshot_failure_cannot_complete_run_successfully():
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
run_store = MemoryRunStore()
|
|
run_manager = RunManager(store=run_store)
|
|
record = await run_manager.create("thread-snapshot-failure")
|
|
app_config = AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}})
|
|
app_config.hitl_approval.enabled = True
|
|
|
|
class SnapshotFailureAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
yield {"messages": []}
|
|
|
|
async def aget_state(self, _config):
|
|
raise RuntimeError("checkpoint snapshot unavailable")
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=InMemorySaver(), app_config=app_config),
|
|
agent_factory=lambda *, config: SnapshotFailureAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
assert record.status == RunStatus.error
|
|
assert record.error == "checkpoint snapshot unavailable"
|
|
assert (await run_store.get(record.run_id))["status"] == "error"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_paused_metadata_is_not_written_after_ownership_loss():
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
run_store = MemoryRunStore()
|
|
run_manager = RunManager(
|
|
store=run_store,
|
|
worker_id="owner",
|
|
run_ownership_config=RunOwnershipConfig(heartbeat_enabled=True, lease_seconds=30, grace_seconds=0),
|
|
)
|
|
record = await run_manager.create_or_reject("thread-paused-ownership")
|
|
app_config = AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}})
|
|
app_config.hitl_approval.enabled = True
|
|
original_finalize = run_store.finalize_if_not_cancelled
|
|
|
|
async def finalize_then_lose_ownership(*args, **kwargs):
|
|
result = await original_finalize(*args, **kwargs)
|
|
run_store._runs[record.run_id].update(status="error", error="peer takeover", owner_worker_id="peer", metadata={"owner": "peer"})
|
|
await run_manager._mark_ownership_lost(record, reason="Test peer takeover", require_active=False)
|
|
return result
|
|
|
|
run_store.finalize_if_not_cancelled = finalize_then_lose_ownership
|
|
run_store.put = AsyncMock(wraps=run_store.put)
|
|
run_manager.persist_metadata = AsyncMock(wraps=run_manager.persist_metadata)
|
|
payload = {"kind": "hitl_approval", "request_id": "approval-1"}
|
|
|
|
class PausedAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
yield {"messages": []}
|
|
|
|
async def aget_state(self, _config):
|
|
return SimpleNamespace(
|
|
values={"messages": []},
|
|
config={},
|
|
metadata={},
|
|
next=("agent",),
|
|
tasks=(SimpleNamespace(interrupts=(SimpleNamespace(value=payload),)),),
|
|
)
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=InMemorySaver(), app_config=app_config),
|
|
agent_factory=lambda *, config: PausedAgent(),
|
|
graph_input={},
|
|
config={},
|
|
)
|
|
|
|
run_manager.persist_metadata.assert_not_awaited()
|
|
run_store.put.assert_not_awaited()
|
|
row = await run_store.get(record.run_id)
|
|
assert row["status"] == "error"
|
|
assert row["error"] == "peer takeover"
|
|
assert row["metadata"] == {"owner": "peer"}
|
|
|
|
|
|
@pytest.mark.anyio
|
|
@pytest.mark.parametrize("stream_modes", [None, ["messages-tuple"]], ids=["values", "messages-only"])
|
|
async def test_generic_interrupt_is_waiting_even_when_hitl_is_disabled(stream_modes):
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
run_store = MemoryRunStore()
|
|
run_manager = RunManager(store=run_store)
|
|
record = await run_manager.create("thread-generic-pause" if stream_modes is None else "thread-generic-pause-messages")
|
|
app_config = AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}})
|
|
|
|
class GenericPausedAgent:
|
|
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
|
yield {"messages": [], "__interrupt__": (Interrupt(value={"question": "Continue?"}, id="generic-pause"),)}
|
|
|
|
async def aget_state(self, _config):
|
|
return SimpleNamespace(
|
|
values={"messages": []},
|
|
config={},
|
|
metadata={},
|
|
next=("answer",),
|
|
tasks=(SimpleNamespace(interrupts=(SimpleNamespace(value={"question": "Continue?"}),)),),
|
|
)
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=InMemorySaver(), app_config=app_config),
|
|
agent_factory=lambda *, config: GenericPausedAgent(),
|
|
graph_input={},
|
|
config={},
|
|
stream_modes=stream_modes,
|
|
)
|
|
|
|
row = await run_store.get(record.run_id)
|
|
assert row["status"] == "waiting_for_input"
|
|
assert "hitl_approval" not in row["metadata"]
|
|
|
|
|
|
@pytest.mark.anyio
|
|
@pytest.mark.parametrize("mode", ["full", "delta"])
|
|
async def test_approval_pause_bypasses_terminal_goal_delivery_and_batch_completion(tmp_path, monkeypatch, mode):
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from app.subagent_batches.continuation import BatchContinuations
|
|
from deerflow.agents.thread_state import ThreadState
|
|
from deerflow.extensions.registry import ExtensionRegistry
|
|
from deerflow.runtime.goal import build_goal_state, read_thread_goal, write_thread_goal
|
|
|
|
paths = Paths(base_dir=tmp_path)
|
|
monkeypatch.setattr("deerflow.workspace_changes.recorder.get_paths", lambda: paths)
|
|
user_id = get_effective_user_id()
|
|
thread_id = f"thread-approval-pause-{mode}"
|
|
checkpointer = InMemorySaver()
|
|
event_store = MemoryRunEventStore()
|
|
run_store = MemoryRunStore()
|
|
run_manager = RunManager(store=run_store)
|
|
record = await run_manager.create(thread_id, user_id=user_id)
|
|
await write_thread_goal(checkpointer, thread_id, build_goal_state("Finish the report"), create_if_missing=True)
|
|
|
|
output_dir = paths.sandbox_outputs_dir(thread_id, user_id=user_id)
|
|
payload = {"kind": "hitl_approval", "request_id": f"approval-{mode}", "pending_tool_call_ids": ["call-write"]}
|
|
|
|
def produce_output(_state):
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
(output_dir / "report.md").write_text("Ready for review\n", encoding="utf-8")
|
|
return {}
|
|
|
|
def request_approval(_state):
|
|
interrupt(payload)
|
|
|
|
builder = StateGraph(ThreadState)
|
|
builder.add_node("produce_output", produce_output)
|
|
builder.add_node("request_approval", request_approval)
|
|
builder.add_edge(START, "produce_output")
|
|
builder.add_edge("produce_output", "request_approval")
|
|
builder.add_edge("request_approval", END)
|
|
graph = builder.compile(checkpointer=checkpointer)
|
|
|
|
requested_followups = {"batch-1": "requested"}
|
|
|
|
class BatchRepo:
|
|
async def list_continuation_batches_for_run(self, run_id):
|
|
assert run_id == record.run_id
|
|
return [{"id": "batch-1", "user_id": user_id}]
|
|
|
|
async def suppress_thread_continuations(self, _thread_id, *, goal_created_at):
|
|
requested_followups["batch-1"] = f"suppressed:{goal_created_at}"
|
|
|
|
class BatchService:
|
|
async def cancel_batch(self, *, batch_id, user_id):
|
|
assert batch_id == "batch-1"
|
|
assert user_id == record.user_id
|
|
requested_followups.pop(batch_id, None)
|
|
|
|
batch_app = SimpleNamespace(
|
|
state=SimpleNamespace(
|
|
subagent_batch_repo=BatchRepo(),
|
|
subagent_batch_service=BatchService(),
|
|
checkpointer=checkpointer,
|
|
)
|
|
)
|
|
batch_continuations = BatchContinuations(batch_app)
|
|
|
|
class LifecycleRecorder:
|
|
def __init__(self):
|
|
self.events = []
|
|
|
|
async def on_task_start(self, _app_store, _task_store, info):
|
|
self.events.append(("start", info.run_id))
|
|
|
|
async def on_task_stop(self, _app_store, _task_store, info, outcome):
|
|
self.events.append(("stop", info.run_id, outcome.value))
|
|
|
|
recorder = LifecycleRecorder()
|
|
registry = ExtensionRegistry()
|
|
with registry.attributed_to("test:hitl-pause"):
|
|
registry.task_lifecycle(recorder)
|
|
|
|
app_config = AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}})
|
|
app_config.hitl_approval.enabled = True
|
|
app_config.title.enabled = app_config.memory.enabled = app_config.memory.injection_enabled = False
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(
|
|
checkpointer=checkpointer,
|
|
event_store=event_store,
|
|
app_config=app_config,
|
|
extensions=registry.build(),
|
|
checkpoint_channel_mode=mode,
|
|
on_run_completed=batch_continuations.source_finished,
|
|
),
|
|
agent_factory=lambda *, config: graph,
|
|
graph_input={"messages": [HumanMessage(content="Write the report")]},
|
|
config={"configurable": {"thread_id": thread_id}},
|
|
)
|
|
|
|
row = await run_store.get(record.run_id)
|
|
assert row["status"] == "waiting_for_input"
|
|
assert row["error"] is None
|
|
assert row["metadata"]["hitl_approval"] == payload
|
|
paused_goal = await read_thread_goal(checkpointer, thread_id)
|
|
assert paused_goal["status"] == "active"
|
|
assert paused_goal["continuation_count"] == paused_goal["no_progress_count"] == 0
|
|
assert "last_evaluation" not in paused_goal
|
|
assert requested_followups == {"batch-1": "requested"}
|
|
assert recorder.events == [("start", record.run_id), ("stop", record.run_id, "failed")]
|
|
|
|
delivery = await _delivery_events(event_store, thread_id, record.run_id)
|
|
assert len(delivery) == 1
|
|
assert delivery[0]["content"]["produced_paths"] == ["/mnt/user-data/outputs/report.md"]
|
|
assert delivery[0]["content"]["satisfied"] is False
|
|
terminal = await event_store.list_events(thread_id, record.run_id, event_types=["run.terminal"])
|
|
assert [event["content"] for event in terminal] == [{"status": "waiting_for_input"}]
|
|
|
|
|
|
@pytest.mark.anyio
|
|
@pytest.mark.parametrize("mode", ["full", "delta"])
|
|
async def test_disabled_hitl_does_not_materialize_pending_approval_snapshot(mode):
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from deerflow.agents.thread_state import ThreadState
|
|
|
|
checkpointer = InMemorySaver()
|
|
run_manager = RunManager()
|
|
record = await run_manager.create(f"thread-disabled-hitl-{mode}")
|
|
|
|
def finish(_state):
|
|
return {"messages": [AIMessage(content="Done")]}
|
|
|
|
builder = StateGraph(ThreadState)
|
|
builder.add_node("finish", finish)
|
|
builder.add_edge(START, "finish")
|
|
builder.add_edge("finish", END)
|
|
compiled = builder.compile(checkpointer=checkpointer)
|
|
|
|
class SnapshotTrackingGraph:
|
|
def __init__(self):
|
|
self.stream_finished = False
|
|
self.post_stream_running_snapshots = 0
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(compiled, name)
|
|
|
|
async def astream(self, *args, **kwargs):
|
|
async for item in compiled.astream(*args, **kwargs):
|
|
yield item
|
|
self.stream_finished = True
|
|
|
|
async def aget_state(self, config):
|
|
if self.stream_finished and record.status == RunStatus.running:
|
|
self.post_stream_running_snapshots += 1
|
|
return await compiled.aget_state(config)
|
|
|
|
graph = SnapshotTrackingGraph()
|
|
app_config = AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}})
|
|
app_config.hitl_approval.enabled = False
|
|
app_config.title.enabled = app_config.memory.enabled = app_config.memory.injection_enabled = False
|
|
|
|
await run_agent(
|
|
_make_bridge(),
|
|
run_manager,
|
|
record,
|
|
ctx=RunContext(checkpointer=checkpointer, app_config=app_config, checkpoint_channel_mode=mode),
|
|
agent_factory=lambda *, config: graph,
|
|
graph_input={"messages": [HumanMessage(content="Finish normally")]},
|
|
config={"configurable": {"thread_id": record.thread_id}},
|
|
)
|
|
|
|
assert record.status == RunStatus.success
|
|
assert graph.post_stream_running_snapshots == 0
|