fix(runtime): publish a failed artifact delivery on the run stream (#8252)

* fix(runtime): publish the delivery failure on the run stream

* docs(api): note the delivery-failure stream event

* test(runtime): cover the deferred delivery-failure frame

* docs(runtime): log the delivery fix and explain the fallback exemption

* test(e2e): present the recorded output in the replay scenario

The write/read-file fixture predates the delivery check, so the replayed
run has been ending as an error without presenting note.txt. This PR makes
that failure visible on the stream, which broke the replay golden and the
render e2e. Add the present_files step the current scenario produces and
regenerate the golden.

* docs: drop the CHANGELOG entry

CONTRIBUTING.md asks contributors to leave CHANGELOG.md and CHANGELOG_zh.md
to the maintainers at merge time, so the Unreleased ordering and the
English/Chinese pair stay in sync.
This commit is contained in:
一片一片 2026-11-16 18:39:06 +08:00 committed by GitHub
parent db02b77837
commit ea1833bd7a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 318 additions and 1 deletions

View File

@ -356,7 +356,7 @@ does not accept this header and keeps the existing missing-stream close of
**Run Option Compatibility:**
- Supported concurrency strategies: `reject`, `rollback`, and `interrupt`
- Compatibility default: `if_not_exists="create"`; this matches DeerFlow's current behavior
- Artifact delivery is enforced automatically when a run creates or modifies regular files under `/mnt/user-data/outputs`. `present_files` or `present_shared_artifact` must present at least one path produced by the current run (or a directory containing it), and the terminal receipt must be persisted; presenting only an unrelated file does not satisfy delivery. Runs without changed outputs retain ordinary conversational behavior. `artifact_delivery` is not a client-settable run option.
- Artifact delivery is enforced automatically when a run creates or modifies regular files under `/mnt/user-data/outputs`. `present_files` or `present_shared_artifact` must present at least one path produced by the current run (or a directory containing it), and the terminal receipt must be persisted; presenting only an unrelated file does not satisfy delivery. Runs without changed outputs retain ordinary conversational behavior. `artifact_delivery` is not a client-settable run option. A run failed this way is stored as `error`, and its stream carries an `error` event (name `RunError`) with the delivery message before `end`, matching the `/wait` error envelope and the step-limit event described under `recursion_limit`.
- Unsupported options return `422`: `webhook`, `stream_resumable=true`, `after_seconds`, `feedback_keys`, any non-null `on_completion` value (including the SDK values `"complete"` and `"continue"`), `if_not_exists="reject"`, and `multitask_strategy="enqueue"`
- `stream_resumable=false` is accepted: it is the LangGraph SDK's default and requests the non-resumable stream DeerFlow already serves
- Undeclared SDK options, including `checkpoint_during` and `durability`, also return `422` instead of being silently discarded

View File

@ -360,6 +360,9 @@ async def _persist_delivery_receipt(
_DELIVERY_INCOMPLETE_ERROR = "Artifact delivery incomplete: no produced output artifact was presented"
_DELIVERY_RECEIPT_FAILED_ERROR = "Artifact delivery verification failed: terminal delivery receipt could not be persisted"
_RECURSION_CAPPED_STOP_REASON = "recursion_capped"
# Stream error frames and the /wait error envelope share one name for a run that
# ends in error without an exception; the Gateway spells it the same way.
_RUN_ERROR_NAME = "RunError"
_RECURSION_CAPPED_LISTED_OUTPUTS = 5
@ -1879,6 +1882,12 @@ async def run_agent(
if record.abort_event.is_set():
await _finish_cancellation(record.abort_action)
elif llm_error_fallback_message or (journal is not None and journal.had_llm_error_fallback):
# Deliberately no `error` frame here, unlike the delivery check below.
# The fallback was streamed and stored as a visible assistant message
# carrying the provider failure (tagged `deerflow_error_fallback`), so
# this turn never looked like a silent success; a frame would only
# duplicate that notice. The record still commits `error` for
# accounting and retry purposes.
error_msg = llm_error_fallback_message
if error_msg is None and journal is not None:
error_msg = journal.llm_error_fallback_message
@ -1937,6 +1946,17 @@ async def run_agent(
)
if cancel_action is not None:
await _finish_cancellation(cancel_action)
elif delivery_error is not None and not record.ownership_lost:
# The run record is terminal `error`, so the stream must say the
# same thing: the web UI raises its run-failure toast only for an
# `error` frame, and streaming IM channels mark a turn failed only
# on that frame. Without it they report a successful turn while
# the record (and /wait) report the failure.
delivery_frame = {"message": delivery_error, "name": _RUN_ERROR_NAME}
if record.status not in (RunStatus.pending, RunStatus.running) and getattr(record, "terminal_status_persisted", True):
await bridge.publish(run_id, "error", delivery_frame)
else:
terminal_error_frame = delivery_frame
except asyncio.CancelledError:
await _finish_cancellation(record.abort_action)
@ -2114,6 +2134,11 @@ async def run_agent(
receipt_persisted = await _persist_delivery_receipt(event_store, thread_id=thread_id, run_id=run_id, content=delivery_content)
if produced_output_paths and record.status == RunStatus.success and not receipt_persisted:
await run_manager.set_status(run_id, RunStatus.error, error=_DELIVERY_RECEIPT_FAILED_ERROR, persist=False)
# Same invariant as the completion branch: the record is now
# `error`, so the stream must not end as a success. Frames
# already deferred and published below the finalizer run
# before END.
terminal_error_frame = {"message": _DELIVERY_RECEIPT_FAILED_ERROR, "name": _RUN_ERROR_NAME}
if not record.ownership_lost and journal is not None and persist_completion:
try:

View File

@ -249,6 +249,74 @@
"viewed_images"
]
},
{
"event": "values",
"keys": [
"artifacts",
"background_tasks",
"delegations",
"messages",
"sandbox",
"skill_context",
"thread_data",
"title",
"tool_artifact_processed",
"tool_artifacts",
"uploaded_files",
"viewed_images"
]
},
{
"event": "values",
"keys": [
"artifacts",
"background_tasks",
"delegations",
"messages",
"sandbox",
"skill_context",
"thread_data",
"title",
"tool_artifact_processed",
"tool_artifacts",
"uploaded_files",
"viewed_images"
]
},
{
"event": "values",
"keys": [
"artifacts",
"background_tasks",
"delegations",
"messages",
"sandbox",
"skill_context",
"thread_data",
"title",
"tool_artifact_processed",
"tool_artifacts",
"uploaded_files",
"viewed_images"
]
},
{
"event": "values",
"keys": [
"artifacts",
"background_tasks",
"delegations",
"messages",
"sandbox",
"skill_context",
"thread_data",
"title",
"tool_artifact_processed",
"tool_artifacts",
"uploaded_files",
"viewed_images"
]
},
{
"event": "end",
"keys": null

View File

@ -141,6 +141,52 @@
"caller": "lead_agent",
"conversation_hash": "bf8d15b438660f957ce93aee2145d37d69cceedf5d127c096466addd347f5291",
"input_hash": "3e64bf27addeac7731f5d1acf4f4a9c04b0974a538244f52c6f7cf1d5ee7c019",
"output": {
"type": "ai",
"data": {
"content": "",
"additional_kwargs": {},
"response_metadata": {
"finish_reason": "tool_calls",
"model_name": "sre/gpt-5",
"model_provider": "openai"
},
"type": "ai",
"name": null,
"id": "lc_run--present-note-file",
"tool_calls": [
{
"name": "present_files",
"args": {
"filepaths": [
"/mnt/user-data/outputs/note.txt"
]
},
"id": "call_PresentNoteFile",
"type": "tool_call"
}
],
"invalid_tool_calls": [],
"usage_metadata": {
"input_tokens": 3719,
"output_tokens": 35,
"total_tokens": 3754,
"input_token_details": {
"audio": 0,
"cache_read": 3584
},
"output_token_details": {
"audio": 0,
"reasoning": 0
}
}
}
}
},
{
"caller": "lead_agent",
"conversation_hash": "ee08f3a198eb8a120d07c755ccfa1918bb62d27aa930cd646d40c1389cc0122d",
"input_hash": "7c7e912bd806ef2c2de0217596096523f005a047c0409a5bbae830f4701adae9",
"output": {
"type": "ai",
"data": {

View File

@ -29,6 +29,22 @@ def _make_bridge():
return SimpleNamespace(publish=AsyncMock(), publish_end=AsyncMock(), cleanup=AsyncMock())
class _RecordingBridge:
"""A stream bridge that keeps every published event in order."""
def __init__(self):
self.events: list[tuple[str, object]] = []
async def publish(self, run_id, event, payload=None, **kwargs):
self.events.append((event, payload))
async def publish_end(self, run_id, **kwargs):
self.events.append(("end", None))
async def cleanup(self, run_id, **kwargs):
return None
async def _delivery_events(store: MemoryRunEventStore, thread_id: str, run_id: str) -> list[dict]:
events = await store.list_events(thread_id, run_id)
return [e for e in events if e["event_type"] == "run.delivery"]
@ -282,6 +298,168 @@ async def test_changed_outputs_fail_closed_when_not_presented(monkeypatch):
assert record.stop_reason is None
@pytest.mark.anyio
async def test_unpresented_outputs_publish_an_error_frame_before_end(monkeypatch):
"""A run failed by delivery verification must say so on the stream.
The record and ``/wait`` report the failure; the web UI raises its run-failure
toast and streaming IM channels mark a turn failed only on an ``error`` frame,
so without one they present the turn as a successful answer.
"""
run_manager = RunManager()
record = await run_manager.create("thread-1")
store = MemoryRunEventStore()
monkeypatch.setattr(
"deerflow.runtime.runs.worker._produced_output_paths",
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
)
bridge = _RecordingBridge()
class ProseOnlyAgent:
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
yield {"messages": [AIMessage(content="SESSION SUMMARY")]}
await run_agent(
bridge,
run_manager,
record,
ctx=RunContext(checkpointer=None, event_store=store),
agent_factory=lambda *, config: ProseOnlyAgent(),
graph_input={},
config={},
)
assert record.status == RunStatus.error
assert [payload for event, payload in bridge.events if event == "error"] == [
{
"message": "Artifact delivery incomplete: no produced output artifact was presented",
"name": "RunError",
}
]
assert bridge.events[-1][0] == "end"
@pytest.mark.anyio
async def test_unpersisted_delivery_receipt_publishes_an_error_frame_before_end(monkeypatch):
"""The receipt failure also flips the record to ``error``; the stream must follow."""
run_manager = RunManager()
record = await run_manager.create("thread-1")
store = MemoryRunEventStore()
monkeypatch.setattr(
"deerflow.runtime.runs.worker._produced_output_paths",
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
)
monkeypatch.setattr(
"deerflow.runtime.runs.worker._persist_delivery_receipt",
AsyncMock(return_value=False),
)
bridge = _RecordingBridge()
class PresentingAgent:
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
journal = config["context"]["__run_journal"]
ai = AIMessage(content="", tool_calls=[{"id": "call_1", "name": "present_files", "args": {}}])
journal._remember_current_run_tool_calls(ai, caller="lead_agent")
journal.on_tool_end(
Command(
update={
"artifacts": ["/mnt/user-data/outputs/report.md"],
"messages": [ToolMessage("Successfully presented files", tool_call_id="call_1")],
}
),
run_id=uuid4(),
)
yield {"messages": []}
await run_agent(
bridge,
run_manager,
record,
ctx=RunContext(checkpointer=None, event_store=store),
agent_factory=lambda *, config: PresentingAgent(),
graph_input={},
config={},
)
assert record.status == RunStatus.error
assert record.error == "Artifact delivery verification failed: terminal delivery receipt could not be persisted"
assert [payload for event, payload in bridge.events if event == "error"] == [
{
"message": "Artifact delivery verification failed: terminal delivery receipt could not be persisted",
"name": "RunError",
}
]
assert bridge.events[-1][0] == "end"
@pytest.mark.anyio
async def test_unconfirmed_delivery_terminal_status_defers_the_error_frame_until_end(monkeypatch):
"""A delivery failure defers the frame while its terminal status is not durable.
The worker publishes the ``error`` frame directly only once the record is
terminal *and* persisted, and hands it to the finalizer otherwise. A
store-backed run whose first terminal write fails (a transient store outage)
therefore publishes the frame after the finalizer's retry confirmed the row —
and only because that retry succeeded, so a stream that never reaches ``end``
never carries the frame.
"""
order: list[str] = []
class FlakyTerminalRunStore(MemoryRunStore):
def __init__(self):
super().__init__()
self.terminal_write_attempts = 0
async def update_status(self, run_id, status, *, error=None, stop_reason=None, **kwargs):
terminal = status not in {"pending", "running"}
if terminal:
self.terminal_write_attempts += 1
if self.terminal_write_attempts == 1:
raise RuntimeError("transient run store outage")
updated = await super().update_status(run_id, status, error=error, stop_reason=stop_reason, **kwargs)
if terminal and updated:
order.append("durable-error")
return updated
class OrderingBridge:
async def publish(self, _run_id, event, payload=None, **kwargs):
if event == "error":
order.append("stream-error")
async def publish_end(self, _run_id, **kwargs):
order.append("stream-end")
async def cleanup(self, _run_id, **kwargs):
return None
run_store = FlakyTerminalRunStore()
run_manager = RunManager(store=run_store)
record = await run_manager.create("thread-1")
monkeypatch.setattr(
"deerflow.runtime.runs.worker._produced_output_paths",
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
)
class ProseOnlyAgent:
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
yield {"messages": [AIMessage(content="SESSION SUMMARY")]}
await run_agent(
OrderingBridge(),
run_manager,
record,
ctx=RunContext(checkpointer=None),
agent_factory=lambda *, config: ProseOnlyAgent(),
graph_input={},
config={},
)
assert run_store.terminal_write_attempts == 2
assert order == ["durable-error", "stream-error", "stream-end"]
assert record.status == RunStatus.error
assert (await run_store.get(record.run_id))["status"] == "error"
@pytest.mark.anyio
@pytest.mark.parametrize("modify_existing", [False, True], ids=["unproven-new-output", "observed-modified-output"])
@pytest.mark.parametrize("terminal_outcome", ["success", "error", "cancelled"])