mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-16 16:31:18 +00:00
fix(runtime): publish a failed artifact delivery on the run stream (#8252)
* fix(runtime): publish the delivery failure on the run stream * docs(api): note the delivery-failure stream event * test(runtime): cover the deferred delivery-failure frame * docs(runtime): log the delivery fix and explain the fallback exemption * test(e2e): present the recorded output in the replay scenario The write/read-file fixture predates the delivery check, so the replayed run has been ending as an error without presenting note.txt. This PR makes that failure visible on the stream, which broke the replay golden and the render e2e. Add the present_files step the current scenario produces and regenerate the golden. * docs: drop the CHANGELOG entry CONTRIBUTING.md asks contributors to leave CHANGELOG.md and CHANGELOG_zh.md to the maintainers at merge time, so the Unreleased ordering and the English/Chinese pair stay in sync.
This commit is contained in:
parent
db02b77837
commit
ea1833bd7a
@ -356,7 +356,7 @@ does not accept this header and keeps the existing missing-stream close of
|
||||
**Run Option Compatibility:**
|
||||
- Supported concurrency strategies: `reject`, `rollback`, and `interrupt`
|
||||
- Compatibility default: `if_not_exists="create"`; this matches DeerFlow's current behavior
|
||||
- Artifact delivery is enforced automatically when a run creates or modifies regular files under `/mnt/user-data/outputs`. `present_files` or `present_shared_artifact` must present at least one path produced by the current run (or a directory containing it), and the terminal receipt must be persisted; presenting only an unrelated file does not satisfy delivery. Runs without changed outputs retain ordinary conversational behavior. `artifact_delivery` is not a client-settable run option.
|
||||
- Artifact delivery is enforced automatically when a run creates or modifies regular files under `/mnt/user-data/outputs`. `present_files` or `present_shared_artifact` must present at least one path produced by the current run (or a directory containing it), and the terminal receipt must be persisted; presenting only an unrelated file does not satisfy delivery. Runs without changed outputs retain ordinary conversational behavior. `artifact_delivery` is not a client-settable run option. A run failed this way is stored as `error`, and its stream carries an `error` event (name `RunError`) with the delivery message before `end`, matching the `/wait` error envelope and the step-limit event described under `recursion_limit`.
|
||||
- Unsupported options return `422`: `webhook`, `stream_resumable=true`, `after_seconds`, `feedback_keys`, any non-null `on_completion` value (including the SDK values `"complete"` and `"continue"`), `if_not_exists="reject"`, and `multitask_strategy="enqueue"`
|
||||
- `stream_resumable=false` is accepted: it is the LangGraph SDK's default and requests the non-resumable stream DeerFlow already serves
|
||||
- Undeclared SDK options, including `checkpoint_during` and `durability`, also return `422` instead of being silently discarded
|
||||
|
||||
@ -360,6 +360,9 @@ async def _persist_delivery_receipt(
|
||||
_DELIVERY_INCOMPLETE_ERROR = "Artifact delivery incomplete: no produced output artifact was presented"
|
||||
_DELIVERY_RECEIPT_FAILED_ERROR = "Artifact delivery verification failed: terminal delivery receipt could not be persisted"
|
||||
_RECURSION_CAPPED_STOP_REASON = "recursion_capped"
|
||||
# Stream error frames and the /wait error envelope share one name for a run that
|
||||
# ends in error without an exception; the Gateway spells it the same way.
|
||||
_RUN_ERROR_NAME = "RunError"
|
||||
_RECURSION_CAPPED_LISTED_OUTPUTS = 5
|
||||
|
||||
|
||||
@ -1879,6 +1882,12 @@ async def run_agent(
|
||||
if record.abort_event.is_set():
|
||||
await _finish_cancellation(record.abort_action)
|
||||
elif llm_error_fallback_message or (journal is not None and journal.had_llm_error_fallback):
|
||||
# Deliberately no `error` frame here, unlike the delivery check below.
|
||||
# The fallback was streamed and stored as a visible assistant message
|
||||
# carrying the provider failure (tagged `deerflow_error_fallback`), so
|
||||
# this turn never looked like a silent success; a frame would only
|
||||
# duplicate that notice. The record still commits `error` for
|
||||
# accounting and retry purposes.
|
||||
error_msg = llm_error_fallback_message
|
||||
if error_msg is None and journal is not None:
|
||||
error_msg = journal.llm_error_fallback_message
|
||||
@ -1937,6 +1946,17 @@ async def run_agent(
|
||||
)
|
||||
if cancel_action is not None:
|
||||
await _finish_cancellation(cancel_action)
|
||||
elif delivery_error is not None and not record.ownership_lost:
|
||||
# The run record is terminal `error`, so the stream must say the
|
||||
# same thing: the web UI raises its run-failure toast only for an
|
||||
# `error` frame, and streaming IM channels mark a turn failed only
|
||||
# on that frame. Without it they report a successful turn while
|
||||
# the record (and /wait) report the failure.
|
||||
delivery_frame = {"message": delivery_error, "name": _RUN_ERROR_NAME}
|
||||
if record.status not in (RunStatus.pending, RunStatus.running) and getattr(record, "terminal_status_persisted", True):
|
||||
await bridge.publish(run_id, "error", delivery_frame)
|
||||
else:
|
||||
terminal_error_frame = delivery_frame
|
||||
|
||||
except asyncio.CancelledError:
|
||||
await _finish_cancellation(record.abort_action)
|
||||
@ -2114,6 +2134,11 @@ async def run_agent(
|
||||
receipt_persisted = await _persist_delivery_receipt(event_store, thread_id=thread_id, run_id=run_id, content=delivery_content)
|
||||
if produced_output_paths and record.status == RunStatus.success and not receipt_persisted:
|
||||
await run_manager.set_status(run_id, RunStatus.error, error=_DELIVERY_RECEIPT_FAILED_ERROR, persist=False)
|
||||
# Same invariant as the completion branch: the record is now
|
||||
# `error`, so the stream must not end as a success. Frames
|
||||
# already deferred and published below the finalizer run
|
||||
# before END.
|
||||
terminal_error_frame = {"message": _DELIVERY_RECEIPT_FAILED_ERROR, "name": _RUN_ERROR_NAME}
|
||||
|
||||
if not record.ownership_lost and journal is not None and persist_completion:
|
||||
try:
|
||||
|
||||
@ -249,6 +249,74 @@
|
||||
"viewed_images"
|
||||
]
|
||||
},
|
||||
{
|
||||
"event": "values",
|
||||
"keys": [
|
||||
"artifacts",
|
||||
"background_tasks",
|
||||
"delegations",
|
||||
"messages",
|
||||
"sandbox",
|
||||
"skill_context",
|
||||
"thread_data",
|
||||
"title",
|
||||
"tool_artifact_processed",
|
||||
"tool_artifacts",
|
||||
"uploaded_files",
|
||||
"viewed_images"
|
||||
]
|
||||
},
|
||||
{
|
||||
"event": "values",
|
||||
"keys": [
|
||||
"artifacts",
|
||||
"background_tasks",
|
||||
"delegations",
|
||||
"messages",
|
||||
"sandbox",
|
||||
"skill_context",
|
||||
"thread_data",
|
||||
"title",
|
||||
"tool_artifact_processed",
|
||||
"tool_artifacts",
|
||||
"uploaded_files",
|
||||
"viewed_images"
|
||||
]
|
||||
},
|
||||
{
|
||||
"event": "values",
|
||||
"keys": [
|
||||
"artifacts",
|
||||
"background_tasks",
|
||||
"delegations",
|
||||
"messages",
|
||||
"sandbox",
|
||||
"skill_context",
|
||||
"thread_data",
|
||||
"title",
|
||||
"tool_artifact_processed",
|
||||
"tool_artifacts",
|
||||
"uploaded_files",
|
||||
"viewed_images"
|
||||
]
|
||||
},
|
||||
{
|
||||
"event": "values",
|
||||
"keys": [
|
||||
"artifacts",
|
||||
"background_tasks",
|
||||
"delegations",
|
||||
"messages",
|
||||
"sandbox",
|
||||
"skill_context",
|
||||
"thread_data",
|
||||
"title",
|
||||
"tool_artifact_processed",
|
||||
"tool_artifacts",
|
||||
"uploaded_files",
|
||||
"viewed_images"
|
||||
]
|
||||
},
|
||||
{
|
||||
"event": "end",
|
||||
"keys": null
|
||||
|
||||
@ -141,6 +141,52 @@
|
||||
"caller": "lead_agent",
|
||||
"conversation_hash": "bf8d15b438660f957ce93aee2145d37d69cceedf5d127c096466addd347f5291",
|
||||
"input_hash": "3e64bf27addeac7731f5d1acf4f4a9c04b0974a538244f52c6f7cf1d5ee7c019",
|
||||
"output": {
|
||||
"type": "ai",
|
||||
"data": {
|
||||
"content": "",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {
|
||||
"finish_reason": "tool_calls",
|
||||
"model_name": "sre/gpt-5",
|
||||
"model_provider": "openai"
|
||||
},
|
||||
"type": "ai",
|
||||
"name": null,
|
||||
"id": "lc_run--present-note-file",
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "present_files",
|
||||
"args": {
|
||||
"filepaths": [
|
||||
"/mnt/user-data/outputs/note.txt"
|
||||
]
|
||||
},
|
||||
"id": "call_PresentNoteFile",
|
||||
"type": "tool_call"
|
||||
}
|
||||
],
|
||||
"invalid_tool_calls": [],
|
||||
"usage_metadata": {
|
||||
"input_tokens": 3719,
|
||||
"output_tokens": 35,
|
||||
"total_tokens": 3754,
|
||||
"input_token_details": {
|
||||
"audio": 0,
|
||||
"cache_read": 3584
|
||||
},
|
||||
"output_token_details": {
|
||||
"audio": 0,
|
||||
"reasoning": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caller": "lead_agent",
|
||||
"conversation_hash": "ee08f3a198eb8a120d07c755ccfa1918bb62d27aa930cd646d40c1389cc0122d",
|
||||
"input_hash": "7c7e912bd806ef2c2de0217596096523f005a047c0409a5bbae830f4701adae9",
|
||||
"output": {
|
||||
"type": "ai",
|
||||
"data": {
|
||||
|
||||
@ -29,6 +29,22 @@ def _make_bridge():
|
||||
return SimpleNamespace(publish=AsyncMock(), publish_end=AsyncMock(), cleanup=AsyncMock())
|
||||
|
||||
|
||||
class _RecordingBridge:
|
||||
"""A stream bridge that keeps every published event in order."""
|
||||
|
||||
def __init__(self):
|
||||
self.events: list[tuple[str, object]] = []
|
||||
|
||||
async def publish(self, run_id, event, payload=None, **kwargs):
|
||||
self.events.append((event, payload))
|
||||
|
||||
async def publish_end(self, run_id, **kwargs):
|
||||
self.events.append(("end", None))
|
||||
|
||||
async def cleanup(self, run_id, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
async def _delivery_events(store: MemoryRunEventStore, thread_id: str, run_id: str) -> list[dict]:
|
||||
events = await store.list_events(thread_id, run_id)
|
||||
return [e for e in events if e["event_type"] == "run.delivery"]
|
||||
@ -282,6 +298,168 @@ async def test_changed_outputs_fail_closed_when_not_presented(monkeypatch):
|
||||
assert record.stop_reason is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_unpresented_outputs_publish_an_error_frame_before_end(monkeypatch):
|
||||
"""A run failed by delivery verification must say so on the stream.
|
||||
|
||||
The record and ``/wait`` report the failure; the web UI raises its run-failure
|
||||
toast and streaming IM channels mark a turn failed only on an ``error`` frame,
|
||||
so without one they present the turn as a successful answer.
|
||||
"""
|
||||
run_manager = RunManager()
|
||||
record = await run_manager.create("thread-1")
|
||||
store = MemoryRunEventStore()
|
||||
monkeypatch.setattr(
|
||||
"deerflow.runtime.runs.worker._produced_output_paths",
|
||||
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
|
||||
)
|
||||
bridge = _RecordingBridge()
|
||||
|
||||
class ProseOnlyAgent:
|
||||
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
||||
yield {"messages": [AIMessage(content="SESSION SUMMARY")]}
|
||||
|
||||
await run_agent(
|
||||
bridge,
|
||||
run_manager,
|
||||
record,
|
||||
ctx=RunContext(checkpointer=None, event_store=store),
|
||||
agent_factory=lambda *, config: ProseOnlyAgent(),
|
||||
graph_input={},
|
||||
config={},
|
||||
)
|
||||
|
||||
assert record.status == RunStatus.error
|
||||
assert [payload for event, payload in bridge.events if event == "error"] == [
|
||||
{
|
||||
"message": "Artifact delivery incomplete: no produced output artifact was presented",
|
||||
"name": "RunError",
|
||||
}
|
||||
]
|
||||
assert bridge.events[-1][0] == "end"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_unpersisted_delivery_receipt_publishes_an_error_frame_before_end(monkeypatch):
|
||||
"""The receipt failure also flips the record to ``error``; the stream must follow."""
|
||||
run_manager = RunManager()
|
||||
record = await run_manager.create("thread-1")
|
||||
store = MemoryRunEventStore()
|
||||
monkeypatch.setattr(
|
||||
"deerflow.runtime.runs.worker._produced_output_paths",
|
||||
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"deerflow.runtime.runs.worker._persist_delivery_receipt",
|
||||
AsyncMock(return_value=False),
|
||||
)
|
||||
bridge = _RecordingBridge()
|
||||
|
||||
class PresentingAgent:
|
||||
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
||||
journal = config["context"]["__run_journal"]
|
||||
ai = AIMessage(content="", tool_calls=[{"id": "call_1", "name": "present_files", "args": {}}])
|
||||
journal._remember_current_run_tool_calls(ai, caller="lead_agent")
|
||||
journal.on_tool_end(
|
||||
Command(
|
||||
update={
|
||||
"artifacts": ["/mnt/user-data/outputs/report.md"],
|
||||
"messages": [ToolMessage("Successfully presented files", tool_call_id="call_1")],
|
||||
}
|
||||
),
|
||||
run_id=uuid4(),
|
||||
)
|
||||
yield {"messages": []}
|
||||
|
||||
await run_agent(
|
||||
bridge,
|
||||
run_manager,
|
||||
record,
|
||||
ctx=RunContext(checkpointer=None, event_store=store),
|
||||
agent_factory=lambda *, config: PresentingAgent(),
|
||||
graph_input={},
|
||||
config={},
|
||||
)
|
||||
|
||||
assert record.status == RunStatus.error
|
||||
assert record.error == "Artifact delivery verification failed: terminal delivery receipt could not be persisted"
|
||||
assert [payload for event, payload in bridge.events if event == "error"] == [
|
||||
{
|
||||
"message": "Artifact delivery verification failed: terminal delivery receipt could not be persisted",
|
||||
"name": "RunError",
|
||||
}
|
||||
]
|
||||
assert bridge.events[-1][0] == "end"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_unconfirmed_delivery_terminal_status_defers_the_error_frame_until_end(monkeypatch):
|
||||
"""A delivery failure defers the frame while its terminal status is not durable.
|
||||
|
||||
The worker publishes the ``error`` frame directly only once the record is
|
||||
terminal *and* persisted, and hands it to the finalizer otherwise. A
|
||||
store-backed run whose first terminal write fails (a transient store outage)
|
||||
therefore publishes the frame after the finalizer's retry confirmed the row —
|
||||
and only because that retry succeeded, so a stream that never reaches ``end``
|
||||
never carries the frame.
|
||||
"""
|
||||
order: list[str] = []
|
||||
|
||||
class FlakyTerminalRunStore(MemoryRunStore):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.terminal_write_attempts = 0
|
||||
|
||||
async def update_status(self, run_id, status, *, error=None, stop_reason=None, **kwargs):
|
||||
terminal = status not in {"pending", "running"}
|
||||
if terminal:
|
||||
self.terminal_write_attempts += 1
|
||||
if self.terminal_write_attempts == 1:
|
||||
raise RuntimeError("transient run store outage")
|
||||
updated = await super().update_status(run_id, status, error=error, stop_reason=stop_reason, **kwargs)
|
||||
if terminal and updated:
|
||||
order.append("durable-error")
|
||||
return updated
|
||||
|
||||
class OrderingBridge:
|
||||
async def publish(self, _run_id, event, payload=None, **kwargs):
|
||||
if event == "error":
|
||||
order.append("stream-error")
|
||||
|
||||
async def publish_end(self, _run_id, **kwargs):
|
||||
order.append("stream-end")
|
||||
|
||||
async def cleanup(self, _run_id, **kwargs):
|
||||
return None
|
||||
|
||||
run_store = FlakyTerminalRunStore()
|
||||
run_manager = RunManager(store=run_store)
|
||||
record = await run_manager.create("thread-1")
|
||||
monkeypatch.setattr(
|
||||
"deerflow.runtime.runs.worker._produced_output_paths",
|
||||
AsyncMock(return_value=["/mnt/user-data/outputs/report.md"]),
|
||||
)
|
||||
|
||||
class ProseOnlyAgent:
|
||||
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
||||
yield {"messages": [AIMessage(content="SESSION SUMMARY")]}
|
||||
|
||||
await run_agent(
|
||||
OrderingBridge(),
|
||||
run_manager,
|
||||
record,
|
||||
ctx=RunContext(checkpointer=None),
|
||||
agent_factory=lambda *, config: ProseOnlyAgent(),
|
||||
graph_input={},
|
||||
config={},
|
||||
)
|
||||
|
||||
assert run_store.terminal_write_attempts == 2
|
||||
assert order == ["durable-error", "stream-error", "stream-end"]
|
||||
assert record.status == RunStatus.error
|
||||
assert (await run_store.get(record.run_id))["status"] == "error"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize("modify_existing", [False, True], ids=["unproven-new-output", "observed-modified-output"])
|
||||
@pytest.mark.parametrize("terminal_outcome", ["success", "error", "cancelled"])
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user