mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
fix(runtime): report a failed scheduled-goal cleanup on the run stream (#8322)
* fix(runtime): report a failed scheduled-goal cleanup on the run stream The finalizer flips a finished run to `error` when it cannot remove the occurrence's scheduled goal, but it published no `error` event. The stream then ended as a success while the record and `/wait` reported the failure: the web UI raises its run-failure toast and streaming IM channels mark a turn failed only on that event. Set the same deferred `terminal_error_frame` the unpersisted delivery receipt uses, so both finalizer-side terminal errors end the stream the way the record does. * fix(runtime): publish a staged error frame only for an error outcome An interrupt accepted while scheduled-goal cleanup runs keeps the run slot active. If cleanup then raises, the error is staged, but the cancellation branch persists `interrupted`; the stale frame made a cancelled run show a failure toast while the record and `/wait` reported interruption. Publish `terminal_error_frame` only when the committed outcome is the error it describes, which also covers a cancellation that wins after the delivery receipt staged its frame. The delta cases now set the worker's checkpoint mode, and a new test cancels the run from inside a failing cleanup. * fix(runtime): publish a staged error frame only for the committed error A rollback also commits `RunStatus.error`, so the status check alone still sent the staged cleanup message while the row and `/wait` said `Rolled back by user`. Require the frame's message to equal the committed error. The cancellation test now covers interrupt and rollback over both run stores and both checkpoint modes; the rollback cases fail without the message check.
This commit is contained in:
parent
524f60494f
commit
fdb9deab6d
@ -361,7 +361,7 @@ does not accept this header and keeps the existing missing-stream close of
|
||||
**Run Option Compatibility:**
|
||||
- Supported concurrency strategies: `reject`, `rollback`, and `interrupt`
|
||||
- Compatibility default: `if_not_exists="create"`; this matches DeerFlow's current behavior
|
||||
- Artifact delivery is enforced automatically when a run creates or modifies regular files under `/mnt/user-data/outputs`. `present_files` or `present_shared_artifact` must present at least one path produced by the current run (or a directory containing it), and the terminal receipt must be persisted; presenting only an unrelated file does not satisfy delivery. Runs without changed outputs retain ordinary conversational behavior. `artifact_delivery` is not a client-settable run option. A run failed this way is stored as `error`, and its stream carries an `error` event (name `RunError`) with the delivery message before `end`, matching the `/wait` error envelope and the step-limit event described under `recursion_limit`.
|
||||
- Artifact delivery is enforced automatically when a run creates or modifies regular files under `/mnt/user-data/outputs`. `present_files` or `present_shared_artifact` must present at least one path produced by the current run (or a directory containing it), and the terminal receipt must be persisted; presenting only an unrelated file does not satisfy delivery. Runs without changed outputs retain ordinary conversational behavior. `artifact_delivery` is not a client-settable run option. A run failed this way is stored as `error`, and its stream carries an `error` event (name `RunError`) with the delivery message before `end`, matching the `/wait` error envelope and the step-limit event described under `recursion_limit`. Any other terminal `error` the worker commits after the graph ends — an unpersisted delivery receipt, or a scheduled-goal cleanup that failed — publishes the same event with that error message.
|
||||
- Unsupported options return `422`: `webhook`, `stream_resumable=true`, `after_seconds`, `feedback_keys`, any non-null `on_completion` value (including the SDK values `"complete"` and `"continue"`), `if_not_exists="reject"`, and `multitask_strategy="enqueue"`
|
||||
- `stream_resumable=false` is accepted: it is the LangGraph SDK's default and requests the non-resumable stream DeerFlow already serves
|
||||
- Undeclared SDK options, including `checkpoint_during` and `durability`, also return `422` instead of being silently discarded
|
||||
|
||||
@ -359,6 +359,7 @@ async def _persist_delivery_receipt(
|
||||
|
||||
_DELIVERY_INCOMPLETE_ERROR = "Artifact delivery incomplete: no produced output artifact was presented"
|
||||
_DELIVERY_RECEIPT_FAILED_ERROR = "Artifact delivery verification failed: terminal delivery receipt could not be persisted"
|
||||
_SCHEDULED_GOAL_CLEANUP_FAILED_ERROR = "Scheduled goal cleanup failed"
|
||||
_RECURSION_CAPPED_STOP_REASON = "recursion_capped"
|
||||
# Stream error frames and the /wait error envelope share one name for a run that
|
||||
# ends in error without an exception; the Gateway spells it the same way.
|
||||
@ -2183,7 +2184,10 @@ async def run_agent(
|
||||
deferred_finalization_interrupt = await _await_task_stop_after_host_cancellation(cleanup_task, deferred_finalization_interrupt)
|
||||
except Exception:
|
||||
logger.warning("Could not remove scheduled goal for run %s", run_id, exc_info=True)
|
||||
await run_manager.set_status(run_id, RunStatus.error, error="Scheduled goal cleanup failed", persist=False)
|
||||
await run_manager.set_status(run_id, RunStatus.error, error=_SCHEDULED_GOAL_CLEANUP_FAILED_ERROR, persist=False)
|
||||
# Same invariant as the receipt failure above: the record
|
||||
# is now `error`, so the stream must not end as a success.
|
||||
terminal_error_frame = {"message": _SCHEDULED_GOAL_CLEANUP_FAILED_ERROR, "name": _RUN_ERROR_NAME}
|
||||
|
||||
if not record.ownership_lost and (event_store is not None or scheduled_goal is not None or record.rollback_finalization_pending):
|
||||
try:
|
||||
@ -2423,7 +2427,12 @@ async def run_agent(
|
||||
|
||||
terminal_stream_authorized = not record.ownership_lost and record.status not in (RunStatus.pending, RunStatus.running) and getattr(record, "terminal_status_persisted", True)
|
||||
if terminal_stream_authorized:
|
||||
if terminal_error_frame is not None:
|
||||
# A staged frame predates the final outcome, and a cancellation
|
||||
# can still win after it was staged: an interrupt replaces the
|
||||
# status and a rollback replaces the message (both can land
|
||||
# while scheduled-goal cleanup runs). Publish only the error
|
||||
# the record actually committed.
|
||||
if terminal_error_frame is not None and record.status == RunStatus.error and record.error == terminal_error_frame.get("message"):
|
||||
await bridge.publish(
|
||||
run_id,
|
||||
"error",
|
||||
|
||||
@ -22,7 +22,7 @@ from deerflow.persistence.run.sql import RunRepository
|
||||
from deerflow.runtime import goal
|
||||
from deerflow.runtime.checkpoint_state import CheckpointStateAccessor
|
||||
from deerflow.runtime.runs import worker
|
||||
from deerflow.runtime.runs.manager import ConflictError, RunManager, RunRecord
|
||||
from deerflow.runtime.runs.manager import CancelOutcome, ConflictError, RunManager, RunRecord
|
||||
from deerflow.runtime.runs.schemas import DisconnectMode, RunStatus
|
||||
from deerflow.runtime.runs.store.base import canonical_run_created_at
|
||||
from deerflow.runtime.runs.store.memory import MemoryRunStore
|
||||
@ -625,3 +625,132 @@ async def test_cancel_accepted_at_terminal_cas_wins_after_worker_abort_check(inf
|
||||
snapshot = await CheckpointStateAccessor.bind(compiled, checkpointer, mode=mode).aget({"configurable": {"thread_id": "result"}})
|
||||
answers = [message.content for message in snapshot.values.get("messages", []) if isinstance(message, AIMessage)]
|
||||
assert ("Report verified" in answers) is (action == "interrupt")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("mode", ["full", "delta"])
|
||||
async def test_scheduled_goal_cleanup_failure_reaches_the_stream_as_an_error(infrastructure, monkeypatch, mode):
|
||||
"""A terminal error the finalizer commits must not end the stream as a success.
|
||||
|
||||
The finalizer flips the record to ``error`` when it cannot remove the
|
||||
occurrence's goal. Without a matching ``error`` event, the web UI raises no
|
||||
run-failure toast and streaming IM channels mark the turn successful while
|
||||
the stored run (and ``/wait``) report the failure.
|
||||
"""
|
||||
store, checkpointer = infrastructure
|
||||
manager = RunManager(store=store, worker_id="scheduler")
|
||||
metadata = {"scheduled_task_id": "task", "scheduled_task_run_id": "occurrence", "scheduled_goal_objective": "Scheduled report"}
|
||||
record = await manager.create_or_reject("result", user_id="alice", metadata=metadata)
|
||||
events = []
|
||||
|
||||
class RecordingBridge:
|
||||
async def publish(self, _run_id, event, payload=None, **kwargs):
|
||||
events.append((event, payload))
|
||||
|
||||
async def publish_end(self, _run_id, **kwargs):
|
||||
events.append(("end", None))
|
||||
|
||||
async def cleanup(self, _run_id, **kwargs):
|
||||
return None
|
||||
|
||||
async def failing_cleanup(**kwargs):
|
||||
raise RuntimeError("goal store unavailable")
|
||||
|
||||
monkeypatch.setattr(worker, "_clear_scheduled_goal", failing_cleanup)
|
||||
|
||||
async def node(state):
|
||||
return {"messages": [AIMessage(content="Report ready")], "title": "Report"}
|
||||
|
||||
compiled = _graph(node, checkpointer, mode)
|
||||
await worker.run_agent(
|
||||
RecordingBridge(),
|
||||
manager,
|
||||
record,
|
||||
ctx=worker.RunContext(
|
||||
checkpointer=checkpointer,
|
||||
checkpoint_channel_mode=mode,
|
||||
scheduled_task_runtime={
|
||||
"task_id": "task",
|
||||
"occurrence_id": "occurrence",
|
||||
"user_id": "alice",
|
||||
"goal_objective": metadata["scheduled_goal_objective"],
|
||||
},
|
||||
),
|
||||
agent_factory=lambda config: compiled,
|
||||
graph_input={"messages": [HumanMessage(content="Report")]},
|
||||
config={"configurable": {"thread_id": "result"}, "context": {"non_interactive": True}},
|
||||
)
|
||||
|
||||
assert record.status == RunStatus.error
|
||||
assert record.error == "Scheduled goal cleanup failed"
|
||||
assert [payload for event, payload in events if event == "error"] == [{"message": "Scheduled goal cleanup failed", "name": "RunError"}]
|
||||
assert events[-1][0] == "end"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("mode", ["full", "delta"])
|
||||
@pytest.mark.parametrize("action", ["interrupt", "rollback"])
|
||||
async def test_cancellation_during_failing_cleanup_ends_as_the_cancellation(infrastructure, monkeypatch, mode, action):
|
||||
"""A staged error frame must not describe a run a cancellation replaced.
|
||||
|
||||
A cancellation is accepted while the occurrence's goal cleanup runs (the run
|
||||
slot stays active until cleanup finishes). Cleanup then fails, so the error is
|
||||
staged; the cancellation branch then commits the cancellation's outcome —
|
||||
``interrupted`` for an interrupt, ``error`` with ``Rolled back by user`` for a
|
||||
rollback. The stale frame belongs to an outcome that lost: the stream has to
|
||||
agree with the record and ``/wait``.
|
||||
"""
|
||||
store, checkpointer = infrastructure
|
||||
manager = RunManager(store=store, worker_id="scheduler")
|
||||
metadata = {"scheduled_task_id": "task", "scheduled_task_run_id": "occurrence", "scheduled_goal_objective": "Scheduled report"}
|
||||
record = await manager.create_or_reject("result", user_id="alice", metadata=metadata)
|
||||
events = []
|
||||
cancels = []
|
||||
|
||||
class RecordingBridge:
|
||||
async def publish(self, _run_id, event, payload=None, **kwargs):
|
||||
events.append((event, payload))
|
||||
|
||||
async def publish_end(self, _run_id, **kwargs):
|
||||
events.append(("end", None))
|
||||
|
||||
async def cleanup(self, _run_id, **kwargs):
|
||||
return None
|
||||
|
||||
async def cancel_then_fail(**kwargs):
|
||||
cancels.append(await manager.cancel(record.run_id, action=action))
|
||||
raise RuntimeError("goal store unavailable")
|
||||
|
||||
monkeypatch.setattr(worker, "_clear_scheduled_goal", cancel_then_fail)
|
||||
|
||||
async def node(state):
|
||||
return {"messages": [AIMessage(content="Report ready")], "title": "Report"}
|
||||
|
||||
compiled = _graph(node, checkpointer, mode)
|
||||
await worker.run_agent(
|
||||
RecordingBridge(),
|
||||
manager,
|
||||
record,
|
||||
ctx=worker.RunContext(
|
||||
checkpointer=checkpointer,
|
||||
checkpoint_channel_mode=mode,
|
||||
scheduled_task_runtime={
|
||||
"task_id": "task",
|
||||
"occurrence_id": "occurrence",
|
||||
"user_id": "alice",
|
||||
"goal_objective": metadata["scheduled_goal_objective"],
|
||||
},
|
||||
),
|
||||
agent_factory=lambda config: compiled,
|
||||
graph_input={"messages": [HumanMessage(content="Report")]},
|
||||
config={"configurable": {"thread_id": "result"}, "context": {"non_interactive": True}},
|
||||
)
|
||||
|
||||
assert cancels == [CancelOutcome.cancelled]
|
||||
if action == "interrupt":
|
||||
assert record.status == RunStatus.interrupted
|
||||
else:
|
||||
assert record.status == RunStatus.error
|
||||
assert record.error.startswith("Rolled back by user")
|
||||
assert [payload for event, payload in events if event == "error"] == []
|
||||
assert events[-1][0] == "end"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user