diff --git a/backend/docs/API.md b/backend/docs/API.md index bb9dec403a..044f9d3cd3 100644 --- a/backend/docs/API.md +++ b/backend/docs/API.md @@ -361,7 +361,7 @@ does not accept this header and keeps the existing missing-stream close of **Run Option Compatibility:** - Supported concurrency strategies: `reject`, `rollback`, and `interrupt` - Compatibility default: `if_not_exists="create"`; this matches DeerFlow's current behavior -- Artifact delivery is enforced automatically when a run creates or modifies regular files under `/mnt/user-data/outputs`. `present_files` or `present_shared_artifact` must present at least one path produced by the current run (or a directory containing it), and the terminal receipt must be persisted; presenting only an unrelated file does not satisfy delivery. Runs without changed outputs retain ordinary conversational behavior. `artifact_delivery` is not a client-settable run option. A run failed this way is stored as `error`, and its stream carries an `error` event (name `RunError`) with the delivery message before `end`, matching the `/wait` error envelope and the step-limit event described under `recursion_limit`. +- Artifact delivery is enforced automatically when a run creates or modifies regular files under `/mnt/user-data/outputs`. `present_files` or `present_shared_artifact` must present at least one path produced by the current run (or a directory containing it), and the terminal receipt must be persisted; presenting only an unrelated file does not satisfy delivery. Runs without changed outputs retain ordinary conversational behavior. `artifact_delivery` is not a client-settable run option. A run failed this way is stored as `error`, and its stream carries an `error` event (name `RunError`) with the delivery message before `end`, matching the `/wait` error envelope and the step-limit event described under `recursion_limit`. Any other terminal `error` the worker commits after the graph ends — an unpersisted delivery receipt, or a scheduled-goal cleanup that failed — publishes the same event with that error message. - Unsupported options return `422`: `webhook`, `stream_resumable=true`, `after_seconds`, `feedback_keys`, any non-null `on_completion` value (including the SDK values `"complete"` and `"continue"`), `if_not_exists="reject"`, and `multitask_strategy="enqueue"` - `stream_resumable=false` is accepted: it is the LangGraph SDK's default and requests the non-resumable stream DeerFlow already serves - Undeclared SDK options, including `checkpoint_during` and `durability`, also return `422` instead of being silently discarded diff --git a/backend/packages/harness/deerflow/runtime/runs/worker.py b/backend/packages/harness/deerflow/runtime/runs/worker.py index c3ef54803d..6307be023b 100644 --- a/backend/packages/harness/deerflow/runtime/runs/worker.py +++ b/backend/packages/harness/deerflow/runtime/runs/worker.py @@ -359,6 +359,7 @@ async def _persist_delivery_receipt( _DELIVERY_INCOMPLETE_ERROR = "Artifact delivery incomplete: no produced output artifact was presented" _DELIVERY_RECEIPT_FAILED_ERROR = "Artifact delivery verification failed: terminal delivery receipt could not be persisted" +_SCHEDULED_GOAL_CLEANUP_FAILED_ERROR = "Scheduled goal cleanup failed" _RECURSION_CAPPED_STOP_REASON = "recursion_capped" # Stream error frames and the /wait error envelope share one name for a run that # ends in error without an exception; the Gateway spells it the same way. @@ -2183,7 +2184,10 @@ async def run_agent( deferred_finalization_interrupt = await _await_task_stop_after_host_cancellation(cleanup_task, deferred_finalization_interrupt) except Exception: logger.warning("Could not remove scheduled goal for run %s", run_id, exc_info=True) - await run_manager.set_status(run_id, RunStatus.error, error="Scheduled goal cleanup failed", persist=False) + await run_manager.set_status(run_id, RunStatus.error, error=_SCHEDULED_GOAL_CLEANUP_FAILED_ERROR, persist=False) + # Same invariant as the receipt failure above: the record + # is now `error`, so the stream must not end as a success. + terminal_error_frame = {"message": _SCHEDULED_GOAL_CLEANUP_FAILED_ERROR, "name": _RUN_ERROR_NAME} if not record.ownership_lost and (event_store is not None or scheduled_goal is not None or record.rollback_finalization_pending): try: @@ -2423,7 +2427,12 @@ async def run_agent( terminal_stream_authorized = not record.ownership_lost and record.status not in (RunStatus.pending, RunStatus.running) and getattr(record, "terminal_status_persisted", True) if terminal_stream_authorized: - if terminal_error_frame is not None: + # A staged frame predates the final outcome, and a cancellation + # can still win after it was staged: an interrupt replaces the + # status and a rollback replaces the message (both can land + # while scheduled-goal cleanup runs). Publish only the error + # the record actually committed. + if terminal_error_frame is not None and record.status == RunStatus.error and record.error == terminal_error_frame.get("message"): await bridge.publish( run_id, "error", diff --git a/backend/tests/test_scheduled_goal_handoff.py b/backend/tests/test_scheduled_goal_handoff.py index 90bfa2bafd..1fb79e0b2c 100644 --- a/backend/tests/test_scheduled_goal_handoff.py +++ b/backend/tests/test_scheduled_goal_handoff.py @@ -22,7 +22,7 @@ from deerflow.persistence.run.sql import RunRepository from deerflow.runtime import goal from deerflow.runtime.checkpoint_state import CheckpointStateAccessor from deerflow.runtime.runs import worker -from deerflow.runtime.runs.manager import ConflictError, RunManager, RunRecord +from deerflow.runtime.runs.manager import CancelOutcome, ConflictError, RunManager, RunRecord from deerflow.runtime.runs.schemas import DisconnectMode, RunStatus from deerflow.runtime.runs.store.base import canonical_run_created_at from deerflow.runtime.runs.store.memory import MemoryRunStore @@ -625,3 +625,132 @@ async def test_cancel_accepted_at_terminal_cas_wins_after_worker_abort_check(inf snapshot = await CheckpointStateAccessor.bind(compiled, checkpointer, mode=mode).aget({"configurable": {"thread_id": "result"}}) answers = [message.content for message in snapshot.values.get("messages", []) if isinstance(message, AIMessage)] assert ("Report verified" in answers) is (action == "interrupt") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["full", "delta"]) +async def test_scheduled_goal_cleanup_failure_reaches_the_stream_as_an_error(infrastructure, monkeypatch, mode): + """A terminal error the finalizer commits must not end the stream as a success. + + The finalizer flips the record to ``error`` when it cannot remove the + occurrence's goal. Without a matching ``error`` event, the web UI raises no + run-failure toast and streaming IM channels mark the turn successful while + the stored run (and ``/wait``) report the failure. + """ + store, checkpointer = infrastructure + manager = RunManager(store=store, worker_id="scheduler") + metadata = {"scheduled_task_id": "task", "scheduled_task_run_id": "occurrence", "scheduled_goal_objective": "Scheduled report"} + record = await manager.create_or_reject("result", user_id="alice", metadata=metadata) + events = [] + + class RecordingBridge: + async def publish(self, _run_id, event, payload=None, **kwargs): + events.append((event, payload)) + + async def publish_end(self, _run_id, **kwargs): + events.append(("end", None)) + + async def cleanup(self, _run_id, **kwargs): + return None + + async def failing_cleanup(**kwargs): + raise RuntimeError("goal store unavailable") + + monkeypatch.setattr(worker, "_clear_scheduled_goal", failing_cleanup) + + async def node(state): + return {"messages": [AIMessage(content="Report ready")], "title": "Report"} + + compiled = _graph(node, checkpointer, mode) + await worker.run_agent( + RecordingBridge(), + manager, + record, + ctx=worker.RunContext( + checkpointer=checkpointer, + checkpoint_channel_mode=mode, + scheduled_task_runtime={ + "task_id": "task", + "occurrence_id": "occurrence", + "user_id": "alice", + "goal_objective": metadata["scheduled_goal_objective"], + }, + ), + agent_factory=lambda config: compiled, + graph_input={"messages": [HumanMessage(content="Report")]}, + config={"configurable": {"thread_id": "result"}, "context": {"non_interactive": True}}, + ) + + assert record.status == RunStatus.error + assert record.error == "Scheduled goal cleanup failed" + assert [payload for event, payload in events if event == "error"] == [{"message": "Scheduled goal cleanup failed", "name": "RunError"}] + assert events[-1][0] == "end" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["full", "delta"]) +@pytest.mark.parametrize("action", ["interrupt", "rollback"]) +async def test_cancellation_during_failing_cleanup_ends_as_the_cancellation(infrastructure, monkeypatch, mode, action): + """A staged error frame must not describe a run a cancellation replaced. + + A cancellation is accepted while the occurrence's goal cleanup runs (the run + slot stays active until cleanup finishes). Cleanup then fails, so the error is + staged; the cancellation branch then commits the cancellation's outcome — + ``interrupted`` for an interrupt, ``error`` with ``Rolled back by user`` for a + rollback. The stale frame belongs to an outcome that lost: the stream has to + agree with the record and ``/wait``. + """ + store, checkpointer = infrastructure + manager = RunManager(store=store, worker_id="scheduler") + metadata = {"scheduled_task_id": "task", "scheduled_task_run_id": "occurrence", "scheduled_goal_objective": "Scheduled report"} + record = await manager.create_or_reject("result", user_id="alice", metadata=metadata) + events = [] + cancels = [] + + class RecordingBridge: + async def publish(self, _run_id, event, payload=None, **kwargs): + events.append((event, payload)) + + async def publish_end(self, _run_id, **kwargs): + events.append(("end", None)) + + async def cleanup(self, _run_id, **kwargs): + return None + + async def cancel_then_fail(**kwargs): + cancels.append(await manager.cancel(record.run_id, action=action)) + raise RuntimeError("goal store unavailable") + + monkeypatch.setattr(worker, "_clear_scheduled_goal", cancel_then_fail) + + async def node(state): + return {"messages": [AIMessage(content="Report ready")], "title": "Report"} + + compiled = _graph(node, checkpointer, mode) + await worker.run_agent( + RecordingBridge(), + manager, + record, + ctx=worker.RunContext( + checkpointer=checkpointer, + checkpoint_channel_mode=mode, + scheduled_task_runtime={ + "task_id": "task", + "occurrence_id": "occurrence", + "user_id": "alice", + "goal_objective": metadata["scheduled_goal_objective"], + }, + ), + agent_factory=lambda config: compiled, + graph_input={"messages": [HumanMessage(content="Report")]}, + config={"configurable": {"thread_id": "result"}, "context": {"non_interactive": True}}, + ) + + assert cancels == [CancelOutcome.cancelled] + if action == "interrupt": + assert record.status == RunStatus.interrupted + else: + assert record.status == RunStatus.error + assert record.error.startswith("Rolled back by user") + assert [payload for event, payload in events if event == "error"] == [] + assert events[-1][0] == "end"