diff --git a/backend/docs/API.md b/backend/docs/API.md index b8aa522186..0cd5b732cd 100644 --- a/backend/docs/API.md +++ b/backend/docs/API.md @@ -356,7 +356,7 @@ does not accept this header and keeps the existing missing-stream close of **Run Option Compatibility:** - Supported concurrency strategies: `reject`, `rollback`, and `interrupt` - Compatibility default: `if_not_exists="create"`; this matches DeerFlow's current behavior -- Artifact delivery is enforced automatically when a run creates or modifies regular files under `/mnt/user-data/outputs`. `present_files` or `present_shared_artifact` must present at least one path produced by the current run (or a directory containing it), and the terminal receipt must be persisted; presenting only an unrelated file does not satisfy delivery. Runs without changed outputs retain ordinary conversational behavior. `artifact_delivery` is not a client-settable run option. +- Artifact delivery is enforced automatically when a run creates or modifies regular files under `/mnt/user-data/outputs`. `present_files` or `present_shared_artifact` must present at least one path produced by the current run (or a directory containing it), and the terminal receipt must be persisted; presenting only an unrelated file does not satisfy delivery. Runs without changed outputs retain ordinary conversational behavior. `artifact_delivery` is not a client-settable run option. A run failed this way is stored as `error`, and its stream carries an `error` event (name `RunError`) with the delivery message before `end`, matching the `/wait` error envelope and the step-limit event described under `recursion_limit`. - Unsupported options return `422`: `webhook`, `stream_resumable=true`, `after_seconds`, `feedback_keys`, any non-null `on_completion` value (including the SDK values `"complete"` and `"continue"`), `if_not_exists="reject"`, and `multitask_strategy="enqueue"` - `stream_resumable=false` is accepted: it is the LangGraph SDK's default and requests the non-resumable stream DeerFlow already serves - Undeclared SDK options, including `checkpoint_during` and `durability`, also return `422` instead of being silently discarded diff --git a/backend/packages/harness/deerflow/runtime/runs/worker.py b/backend/packages/harness/deerflow/runtime/runs/worker.py index b28599e0e1..bfc6dae3a6 100644 --- a/backend/packages/harness/deerflow/runtime/runs/worker.py +++ b/backend/packages/harness/deerflow/runtime/runs/worker.py @@ -360,6 +360,9 @@ async def _persist_delivery_receipt( _DELIVERY_INCOMPLETE_ERROR = "Artifact delivery incomplete: no produced output artifact was presented" _DELIVERY_RECEIPT_FAILED_ERROR = "Artifact delivery verification failed: terminal delivery receipt could not be persisted" _RECURSION_CAPPED_STOP_REASON = "recursion_capped" +# Stream error frames and the /wait error envelope share one name for a run that +# ends in error without an exception; the Gateway spells it the same way. +_RUN_ERROR_NAME = "RunError" _RECURSION_CAPPED_LISTED_OUTPUTS = 5 @@ -1879,6 +1882,12 @@ async def run_agent( if record.abort_event.is_set(): await _finish_cancellation(record.abort_action) elif llm_error_fallback_message or (journal is not None and journal.had_llm_error_fallback): + # Deliberately no `error` frame here, unlike the delivery check below. + # The fallback was streamed and stored as a visible assistant message + # carrying the provider failure (tagged `deerflow_error_fallback`), so + # this turn never looked like a silent success; a frame would only + # duplicate that notice. The record still commits `error` for + # accounting and retry purposes. error_msg = llm_error_fallback_message if error_msg is None and journal is not None: error_msg = journal.llm_error_fallback_message @@ -1937,6 +1946,17 @@ async def run_agent( ) if cancel_action is not None: await _finish_cancellation(cancel_action) + elif delivery_error is not None and not record.ownership_lost: + # The run record is terminal `error`, so the stream must say the + # same thing: the web UI raises its run-failure toast only for an + # `error` frame, and streaming IM channels mark a turn failed only + # on that frame. Without it they report a successful turn while + # the record (and /wait) report the failure. + delivery_frame = {"message": delivery_error, "name": _RUN_ERROR_NAME} + if record.status not in (RunStatus.pending, RunStatus.running) and getattr(record, "terminal_status_persisted", True): + await bridge.publish(run_id, "error", delivery_frame) + else: + terminal_error_frame = delivery_frame except asyncio.CancelledError: await _finish_cancellation(record.abort_action) @@ -2114,6 +2134,11 @@ async def run_agent( receipt_persisted = await _persist_delivery_receipt(event_store, thread_id=thread_id, run_id=run_id, content=delivery_content) if produced_output_paths and record.status == RunStatus.success and not receipt_persisted: await run_manager.set_status(run_id, RunStatus.error, error=_DELIVERY_RECEIPT_FAILED_ERROR, persist=False) + # Same invariant as the completion branch: the record is now + # `error`, so the stream must not end as a success. Frames + # already deferred and published below the finalizer run + # before END. + terminal_error_frame = {"message": _DELIVERY_RECEIPT_FAILED_ERROR, "name": _RUN_ERROR_NAME} if not record.ownership_lost and journal is not None and persist_completion: try: diff --git a/backend/tests/fixtures/replay/write_read_file.ultra.events.json b/backend/tests/fixtures/replay/write_read_file.ultra.events.json index 7cbeb27ae9..0439126a8c 100644 --- a/backend/tests/fixtures/replay/write_read_file.ultra.events.json +++ b/backend/tests/fixtures/replay/write_read_file.ultra.events.json @@ -249,6 +249,74 @@ "viewed_images" ] }, + { + "event": "values", + "keys": [ + "artifacts", + "background_tasks", + "delegations", + "messages", + "sandbox", + "skill_context", + "thread_data", + "title", + "tool_artifact_processed", + "tool_artifacts", + "uploaded_files", + "viewed_images" + ] + }, + { + "event": "values", + "keys": [ + "artifacts", + "background_tasks", + "delegations", + "messages", + "sandbox", + "skill_context", + "thread_data", + "title", + "tool_artifact_processed", + "tool_artifacts", + "uploaded_files", + "viewed_images" + ] + }, + { + "event": "values", + "keys": [ + "artifacts", + "background_tasks", + "delegations", + "messages", + "sandbox", + "skill_context", + "thread_data", + "title", + "tool_artifact_processed", + "tool_artifacts", + "uploaded_files", + "viewed_images" + ] + }, + { + "event": "values", + "keys": [ + "artifacts", + "background_tasks", + "delegations", + "messages", + "sandbox", + "skill_context", + "thread_data", + "title", + "tool_artifact_processed", + "tool_artifacts", + "uploaded_files", + "viewed_images" + ] + }, { "event": "end", "keys": null diff --git a/backend/tests/fixtures/replay/write_read_file.ultra.json b/backend/tests/fixtures/replay/write_read_file.ultra.json index 1eb4400ef9..03016757ec 100644 --- a/backend/tests/fixtures/replay/write_read_file.ultra.json +++ b/backend/tests/fixtures/replay/write_read_file.ultra.json @@ -141,6 +141,52 @@ "caller": "lead_agent", "conversation_hash": "bf8d15b438660f957ce93aee2145d37d69cceedf5d127c096466addd347f5291", "input_hash": "3e64bf27addeac7731f5d1acf4f4a9c04b0974a538244f52c6f7cf1d5ee7c019", + "output": { + "type": "ai", + "data": { + "content": "", + "additional_kwargs": {}, + "response_metadata": { + "finish_reason": "tool_calls", + "model_name": "sre/gpt-5", + "model_provider": "openai" + }, + "type": "ai", + "name": null, + "id": "lc_run--present-note-file", + "tool_calls": [ + { + "name": "present_files", + "args": { + "filepaths": [ + "/mnt/user-data/outputs/note.txt" + ] + }, + "id": "call_PresentNoteFile", + "type": "tool_call" + } + ], + "invalid_tool_calls": [], + "usage_metadata": { + "input_tokens": 3719, + "output_tokens": 35, + "total_tokens": 3754, + "input_token_details": { + "audio": 0, + "cache_read": 3584 + }, + "output_token_details": { + "audio": 0, + "reasoning": 0 + } + } + } + } + }, + { + "caller": "lead_agent", + "conversation_hash": "ee08f3a198eb8a120d07c755ccfa1918bb62d27aa930cd646d40c1389cc0122d", + "input_hash": "7c7e912bd806ef2c2de0217596096523f005a047c0409a5bbae830f4701adae9", "output": { "type": "ai", "data": { diff --git a/backend/tests/test_run_worker_delivery.py b/backend/tests/test_run_worker_delivery.py index 7402640af5..0819850f90 100644 --- a/backend/tests/test_run_worker_delivery.py +++ b/backend/tests/test_run_worker_delivery.py @@ -29,6 +29,22 @@ def _make_bridge(): return SimpleNamespace(publish=AsyncMock(), publish_end=AsyncMock(), cleanup=AsyncMock()) +class _RecordingBridge: + """A stream bridge that keeps every published event in order.""" + + def __init__(self): + self.events: list[tuple[str, object]] = [] + + async def publish(self, run_id, event, payload=None, **kwargs): + self.events.append((event, payload)) + + async def publish_end(self, run_id, **kwargs): + self.events.append(("end", None)) + + async def cleanup(self, run_id, **kwargs): + return None + + async def _delivery_events(store: MemoryRunEventStore, thread_id: str, run_id: str) -> list[dict]: events = await store.list_events(thread_id, run_id) return [e for e in events if e["event_type"] == "run.delivery"] @@ -282,6 +298,168 @@ async def test_changed_outputs_fail_closed_when_not_presented(monkeypatch): assert record.stop_reason is None +@pytest.mark.anyio +async def test_unpresented_outputs_publish_an_error_frame_before_end(monkeypatch): + """A run failed by delivery verification must say so on the stream. + + The record and ``/wait`` report the failure; the web UI raises its run-failure + toast and streaming IM channels mark a turn failed only on an ``error`` frame, + so without one they present the turn as a successful answer. + """ + run_manager = RunManager() + record = await run_manager.create("thread-1") + store = MemoryRunEventStore() + monkeypatch.setattr( + "deerflow.runtime.runs.worker._produced_output_paths", + AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]), + ) + bridge = _RecordingBridge() + + class ProseOnlyAgent: + async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False): + yield {"messages": [AIMessage(content="SESSION SUMMARY")]} + + await run_agent( + bridge, + run_manager, + record, + ctx=RunContext(checkpointer=None, event_store=store), + agent_factory=lambda *, config: ProseOnlyAgent(), + graph_input={}, + config={}, + ) + + assert record.status == RunStatus.error + assert [payload for event, payload in bridge.events if event == "error"] == [ + { + "message": "Artifact delivery incomplete: no produced output artifact was presented", + "name": "RunError", + } + ] + assert bridge.events[-1][0] == "end" + + +@pytest.mark.anyio +async def test_unpersisted_delivery_receipt_publishes_an_error_frame_before_end(monkeypatch): + """The receipt failure also flips the record to ``error``; the stream must follow.""" + run_manager = RunManager() + record = await run_manager.create("thread-1") + store = MemoryRunEventStore() + monkeypatch.setattr( + "deerflow.runtime.runs.worker._produced_output_paths", + AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]), + ) + monkeypatch.setattr( + "deerflow.runtime.runs.worker._persist_delivery_receipt", + AsyncMock(return_value=False), + ) + bridge = _RecordingBridge() + + class PresentingAgent: + async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False): + journal = config["context"]["__run_journal"] + ai = AIMessage(content="", tool_calls=[{"id": "call_1", "name": "present_files", "args": {}}]) + journal._remember_current_run_tool_calls(ai, caller="lead_agent") + journal.on_tool_end( + Command( + update={ + "artifacts": ["/mnt/user-data/outputs/report.md"], + "messages": [ToolMessage("Successfully presented files", tool_call_id="call_1")], + } + ), + run_id=uuid4(), + ) + yield {"messages": []} + + await run_agent( + bridge, + run_manager, + record, + ctx=RunContext(checkpointer=None, event_store=store), + agent_factory=lambda *, config: PresentingAgent(), + graph_input={}, + config={}, + ) + + assert record.status == RunStatus.error + assert record.error == "Artifact delivery verification failed: terminal delivery receipt could not be persisted" + assert [payload for event, payload in bridge.events if event == "error"] == [ + { + "message": "Artifact delivery verification failed: terminal delivery receipt could not be persisted", + "name": "RunError", + } + ] + assert bridge.events[-1][0] == "end" + + +@pytest.mark.anyio +async def test_unconfirmed_delivery_terminal_status_defers_the_error_frame_until_end(monkeypatch): + """A delivery failure defers the frame while its terminal status is not durable. + + The worker publishes the ``error`` frame directly only once the record is + terminal *and* persisted, and hands it to the finalizer otherwise. A + store-backed run whose first terminal write fails (a transient store outage) + therefore publishes the frame after the finalizer's retry confirmed the row — + and only because that retry succeeded, so a stream that never reaches ``end`` + never carries the frame. + """ + order: list[str] = [] + + class FlakyTerminalRunStore(MemoryRunStore): + def __init__(self): + super().__init__() + self.terminal_write_attempts = 0 + + async def update_status(self, run_id, status, *, error=None, stop_reason=None, **kwargs): + terminal = status not in {"pending", "running"} + if terminal: + self.terminal_write_attempts += 1 + if self.terminal_write_attempts == 1: + raise RuntimeError("transient run store outage") + updated = await super().update_status(run_id, status, error=error, stop_reason=stop_reason, **kwargs) + if terminal and updated: + order.append("durable-error") + return updated + + class OrderingBridge: + async def publish(self, _run_id, event, payload=None, **kwargs): + if event == "error": + order.append("stream-error") + + async def publish_end(self, _run_id, **kwargs): + order.append("stream-end") + + async def cleanup(self, _run_id, **kwargs): + return None + + run_store = FlakyTerminalRunStore() + run_manager = RunManager(store=run_store) + record = await run_manager.create("thread-1") + monkeypatch.setattr( + "deerflow.runtime.runs.worker._produced_output_paths", + AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]), + ) + + class ProseOnlyAgent: + async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False): + yield {"messages": [AIMessage(content="SESSION SUMMARY")]} + + await run_agent( + OrderingBridge(), + run_manager, + record, + ctx=RunContext(checkpointer=None), + agent_factory=lambda *, config: ProseOnlyAgent(), + graph_input={}, + config={}, + ) + + assert run_store.terminal_write_attempts == 2 + assert order == ["durable-error", "stream-error", "stream-end"] + assert record.status == RunStatus.error + assert (await run_store.get(record.run_id))["status"] == "error" + + @pytest.mark.anyio @pytest.mark.parametrize("modify_existing", [False, True], ids=["unproven-new-output", "observed-modified-output"]) @pytest.mark.parametrize("terminal_outcome", ["success", "error", "cancelled"])