diff --git a/backend/app/gateway/checkpoint_mode_errors.py b/backend/app/gateway/checkpoint_mode_errors.py new file mode 100644 index 0000000000..47fa8606f0 --- /dev/null +++ b/backend/app/gateway/checkpoint_mode_errors.py @@ -0,0 +1,23 @@ +"""Map checkpoint-mode guard failures to precise HTTP statuses. + +A mismatch means the thread's persisted checkpoints conflict with the process's +frozen mode (operator-actionable, 409); a reconfiguration means the process +itself is mid mode-flip (transient, 503). Both must surface their message — a +generic 500 would force operators to grep logs to discover the root cause after +a mode flip. +""" + +from __future__ import annotations + +from fastapi import HTTPException + +from deerflow.runtime.checkpoint_mode import CheckpointModeMismatchError, CheckpointModeReconfigurationError + +CHECKPOINT_MODE_ERRORS = (CheckpointModeMismatchError, CheckpointModeReconfigurationError) + + +def checkpoint_mode_http_error(exc: Exception, thread_id: str) -> HTTPException: + """Return the HTTP error for a checkpoint-mode guard failure.""" + if isinstance(exc, CheckpointModeMismatchError): + return HTTPException(status_code=409, detail=f"Thread {thread_id}: {exc}") + return HTTPException(status_code=503, detail=str(exc)) diff --git a/backend/app/gateway/routers/thread_runs.py b/backend/app/gateway/routers/thread_runs.py index 2825baf50d..26a56aed26 100644 --- a/backend/app/gateway/routers/thread_runs.py +++ b/backend/app/gateway/routers/thread_runs.py @@ -42,6 +42,12 @@ from app.gateway.checkpoint_lineage import ( resolve_history_versions, resolve_stamp_candidate_versions, ) +from app.gateway.checkpoint_mode_errors import ( + CHECKPOINT_MODE_ERRORS as _CHECKPOINT_MODE_ERRORS, +) +from app.gateway.checkpoint_mode_errors import ( + checkpoint_mode_http_error as _checkpoint_mode_http_error, +) from app.gateway.context_usage import build_context_usage from app.gateway.conversation_reader import ( default_history_hidden_run_ids as _default_history_hidden_run_ids, @@ -746,7 +752,10 @@ async def _find_base_checkpoint_before_human( *, head_checkpoint: Any | None = None, ) -> Any: - accessor, base_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id) + try: + accessor, base_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id) + except _CHECKPOINT_MODE_ERRORS as exc: + raise _checkpoint_mode_http_error(exc, thread_id) from exc if head_checkpoint is not None: try: return await find_checkpoint_before_message( @@ -780,6 +789,8 @@ async def _find_base_checkpoint_before_human( versions, parent_versions = await resolve_stamp_candidate_versions(accessor, item, parent, version_cache) if not _is_duration_only_checkpoint(item, history_index, versions=versions, parent_versions=parent_versions): checkpoints.append(item) + except _CHECKPOINT_MODE_ERRORS as exc: + raise _checkpoint_mode_http_error(exc, thread_id) from exc except Exception as exc: logger.exception("Failed to list checkpoints for regenerate thread %s", thread_id) raise HTTPException(status_code=500, detail="Failed to inspect checkpoint history") from exc @@ -870,9 +881,14 @@ async def _find_interrupted_target_run_id( async def _prepare_regenerate_payload(thread_id: str, message_id: str, request: Request) -> RegeneratePrepareResponse: - accessor, latest_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id) try: + accessor, latest_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id) latest_checkpoint = await accessor.aget(latest_config) + except _CHECKPOINT_MODE_ERRORS as exc: + raise _checkpoint_mode_http_error(exc, thread_id) from exc + except HTTPException: + # Config-loading and other route-level errors keep their status/detail. + raise except Exception as exc: logger.exception("Failed to read latest checkpoint for regenerate thread %s", thread_id) raise HTTPException(status_code=500, detail="Failed to read latest checkpoint") from exc @@ -962,9 +978,14 @@ async def _prepare_edit_regenerate_payload( if not normalized_text: raise HTTPException(status_code=409, detail="Edited message cannot be empty") - accessor, latest_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id) try: + accessor, latest_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id) latest_checkpoint = await accessor.aget(latest_config) + except _CHECKPOINT_MODE_ERRORS as exc: + raise _checkpoint_mode_http_error(exc, thread_id) from exc + except HTTPException: + # Config-loading and other route-level errors keep their status/detail. + raise except Exception as exc: logger.exception("Failed to read latest checkpoint for edit replay thread %s", thread_id) raise HTTPException(status_code=500, detail="Failed to read latest checkpoint") from exc diff --git a/backend/app/gateway/routers/threads.py b/backend/app/gateway/routers/threads.py index e228818e26..d2ff57cc4d 100644 --- a/backend/app/gateway/routers/threads.py +++ b/backend/app/gateway/routers/threads.py @@ -38,6 +38,12 @@ from app.gateway.checkpoint_lineage import ( resolve_history_versions, resolve_stamp_candidate_versions, ) +from app.gateway.checkpoint_mode_errors import ( + CHECKPOINT_MODE_ERRORS as _CHECKPOINT_MODE_ERRORS, +) +from app.gateway.checkpoint_mode_errors import ( + checkpoint_mode_http_error as _checkpoint_mode_http_error, +) from app.gateway.deps import get_checkpointer, get_config, get_current_user_from_request, get_run_event_store, get_run_manager, get_run_store from app.gateway.history_seed import reserve_history_seed, write_history_seed from app.gateway.hitl_approval import find_pending_interrupts @@ -59,7 +65,6 @@ from deerflow.config.paths import Paths, get_paths from deerflow.config.summarization_config import ContextSize from deerflow.persistence.thread_meta import PROJECT_FILTER_UNSET, THREAD_ARCHIVED_METADATA_KEY, THREAD_PINNED_METADATA_KEY, THREAD_PROJECT_METADATA_KEY, ThreadOwnershipConflictError from deerflow.runtime import ThreadOperationKind, serialize_channel_values_for_api -from deerflow.runtime.checkpoint_mode import CheckpointModeMismatchError, CheckpointModeReconfigurationError from deerflow.runtime.checkpoint_state import graph_reducer_channels, graph_state_schema, graph_writable_channels, validate_state_mutation_node from deerflow.runtime.context_compaction import ( ContextCompactionDisabled, @@ -102,8 +107,6 @@ from deerflow.utils.time import coerce_iso, now_iso logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/threads", tags=["threads"]) -_CHECKPOINT_MODE_ERRORS = (CheckpointModeMismatchError, CheckpointModeReconfigurationError) - def _optional_run_event_store(request: Request) -> Any: """Return the run event store, or ``None`` when the app has none wired. @@ -116,20 +119,6 @@ def _optional_run_event_store(request: Request) -> Any: return getattr(request.app.state, "run_event_store", None) -def _checkpoint_mode_http_error(exc: Exception, thread_id: str) -> HTTPException: - """Map checkpoint-mode guard failures to precise HTTP statuses. - - A mismatch means the thread's persisted checkpoints conflict with the - process's frozen mode (operator-actionable, 409); a reconfiguration means - the process itself is mid mode-flip (transient, 503). Both must surface - their message — a generic 500 would force operators to grep logs to - discover the root cause after a mode flip. - """ - if isinstance(exc, CheckpointModeMismatchError): - return HTTPException(status_code=409, detail=f"Thread {thread_id}: {exc}") - return HTTPException(status_code=503, detail=str(exc)) - - # Metadata keys that the server controls; clients are not allowed to set # them. Pydantic ``@field_validator("metadata")`` strips them on every # inbound model below so a malicious client cannot reflect a forged diff --git a/backend/tests/test_gateway_checkpoint_mode.py b/backend/tests/test_gateway_checkpoint_mode.py index 8f6ff165bf..c5a6d8a999 100644 --- a/backend/tests/test_gateway_checkpoint_mode.py +++ b/backend/tests/test_gateway_checkpoint_mode.py @@ -19,6 +19,7 @@ from unittest.mock import AsyncMock import pytest from _router_auth_helpers import make_authed_test_app +from fastapi import HTTPException from fastapi.testclient import TestClient from langchain_core.messages import AIMessage, HumanMessage from langgraph.checkpoint.memory import InMemorySaver @@ -27,7 +28,7 @@ from langgraph.store.memory import InMemoryStore from app.gateway import context_usage from app.gateway import services as gateway_services -from app.gateway.routers import threads +from app.gateway.routers import thread_runs, threads from deerflow.agents.thread_state import get_thread_state_schema from deerflow.config.app_config import AppConfig, reset_app_config, set_app_config from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore @@ -319,3 +320,79 @@ def test_mutation_accessor_fails_closed_when_thread_metadata_lookup_fails(monkey ) resolve_factory.assert_not_called() + + +def test_full_mode_run_prepare_routes_reject_delta_thread_with_409(_stub_app_config, monkeypatch: pytest.MonkeyPatch) -> None: + """The regenerate-prepare routes must surface the mode mismatch as 409. + + They read the same thread state as the threads router, so a full-mode + process opening a delta thread must produce the same actionable 409 rather + than a generic 500 that hides the mode flip. + """ + app = make_authed_test_app() + store = InMemoryStore() + checkpointer = InMemorySaver() + app.state.store = store + app.state.checkpointer = checkpointer + app.state.thread_store.get = AsyncMock(return_value=None) + app.state.checkpoint_channel_mode = "full" + app.state.run_event_store = SimpleNamespace() + app.state.run_manager = RunManager(store=MemoryRunStore()) + app.include_router(threads.router) + app.include_router(thread_runs.router) + + full_graph = _build_reply_graph("full", checkpointer) + monkeypatch.setattr( + gateway_services, + "resolve_agent_factory", + lambda assistant_id=None: lambda config: full_graph, + ) + + delta_graph = _build_reply_graph("delta", checkpointer) + config: dict[str, Any] = {"configurable": {"thread_id": _THREAD_ID}} + inject_checkpoint_mode(config, "delta") + asyncio.run(delta_graph.ainvoke({"messages": [HumanMessage(content="question", id="h0")]}, config)) + + with TestClient(app) as client: + regenerate = client.post( + f"/api/threads/{_THREAD_ID}/runs/regenerate/prepare", + json={"message_id": "a0"}, + ) + assert regenerate.status_code == 409, regenerate.text + assert "requires delta mode" in regenerate.json()["detail"] + assert _THREAD_ID in regenerate.json()["detail"] + + edit_regenerate = client.post( + f"/api/threads/{_THREAD_ID}/runs/edit-regenerate/prepare", + json={"human_message_id": "h0", "replacement_text": "edited"}, + ) + assert edit_regenerate.status_code == 409, edit_regenerate.text + assert "requires delta mode" in edit_regenerate.json()["detail"] + assert _THREAD_ID in edit_regenerate.json()["detail"] + + +def test_run_prepare_propagates_accessor_http_errors(_stub_app_config, monkeypatch: pytest.MonkeyPatch) -> None: + """Route-level errors from the accessor keep their status and detail. + + Configuration loading raises a typed HTTP error; the generic checkpoint-read + handler must not turn it into a misleading 500. + """ + app = make_authed_test_app() + app.state.thread_store.get = AsyncMock(return_value=None) + app.state.run_manager = RunManager(store=MemoryRunStore()) + app.state.checkpoint_channel_mode = "full" + app.state.run_event_store = SimpleNamespace() + app.include_router(thread_runs.router) + + async def raise_unavailable(*args: Any, **kwargs: Any) -> Any: + raise HTTPException(status_code=503, detail="Configuration not available") + + monkeypatch.setattr(thread_runs, "build_thread_checkpoint_state_accessor", raise_unavailable) + + with TestClient(app) as client: + response = client.post( + f"/api/threads/{_THREAD_ID}/runs/regenerate/prepare", + json={"message_id": "a0"}, + ) + assert response.status_code == 503, response.text + assert response.json()["detail"] == "Configuration not available" diff --git a/frontend/src/content/en/harness/checkpoints/troubleshooting.mdx b/frontend/src/content/en/harness/checkpoints/troubleshooting.mdx index 6766357a8a..ab39dd144a 100644 --- a/frontend/src/content/en/harness/checkpoints/troubleshooting.mdx +++ b/frontend/src/content/en/harness/checkpoints/troubleshooting.mdx @@ -18,7 +18,7 @@ This chapter is organized by **what you see**. Each entry gives the cause, the f | A run fails with a restart-required message | a config edit the frozen process rejects when it builds the graph | [A run fails after a config edit](#a-run-fails-after-a-config-edit) | | A config edit seems ignored | mode and cadence are frozen at agent-build time | [A config change appears to have no effect](#a-config-change-appears-to-have-no-effect) | | A history-cache edit seems ignored | the cache is built with the checkpointer at startup | [A history-cache change appears to have no effect](#a-history-cache-change-appears-to-have-no-effect) | -| Regenerate endpoints return `500` | the run-runs router does not map the mode errors | [Regenerate endpoints return 500 instead of 409](#regenerate-endpoints-return-500-instead-of-409) | +| Token usage is null or `/runs/wait` hides a mode mismatch | those routes do not map the mode guard | [Checkpoint-mode mismatches on auxiliary endpoints](#checkpoint-mode-mismatches-on-auxiliary-endpoints) | | Delta did not shrink storage | still in full mode, or the writes table was not counted | [Delta did not shrink storage](#delta-did-not-shrink-storage) | | Materialization got slower | snapshot cadence raised too far | [Materialization got slower after raising the cadence](#materialization-got-slower-after-raising-the-cadence) | | The replaced answer comes back after regenerate | delta fork replay (#4458) | [The old answer reappears after regenerate](#the-old-answer-reappears-after-regenerate) | @@ -91,20 +91,13 @@ This is not the frozen-for-correctness case: nothing about the cache can corrupt **Fix**: restart the Gateway after editing the section — and on the embedded or TUI path, restart that process, whose checkpointer singleton is also created once and reused. Only `database.checkpoint_graph_cache.accessor_graph_max` applies without a restart, because it is re-read on every eviction check. -### Regenerate endpoints return 500 instead of 409 +### Checkpoint-mode mismatches on auxiliary endpoints -**Symptom**: regenerate preparation returns `500` with `detail: "Failed to read latest checkpoint"` or `detail: "Failed to inspect checkpoint history"`, even though the same thread yields `409` elsewhere. +**Symptom**: `GET /api/threads/{thread_id}/token-usage` returns `200` with `context_usage: null`, or `/runs/wait` returns `200` with an error body, while the same thread yields `409` from the state and thread endpoints. -**Cause**: `POST /api/threads/{thread_id}/runs/regenerate/prepare` (and `/runs/edit-regenerate/prepare`) build the state accessor outside any `try`, and this module does not map the checkpoint-mode errors, so a mismatch surfaces as an unhandled `500`. The log lines are +**Cause**: these routes do not map the checkpoint-mode guard. The token-usage path logs `Failed to load checkpoint for context usage on thread %s` and returns an empty usage; `/runs/wait` reports the run's error body instead of the HTTP status. The regenerate and edit-regenerate preparation routes used to fail the same way with `500`; they now reuse the shared `checkpoint_mode_errors` mapper and return the same `409`/`503` as the state endpoints. -``` -Failed to read latest checkpoint for regenerate thread %s -Failed to list checkpoints for regenerate thread %s -``` - -Two other endpoints fail more quietly for the same reason: `GET /api/threads/{thread_id}/token-usage` returns `200` with `context_usage: null` (only `Failed to load checkpoint for context usage on thread %s` is logged), and `/runs/wait` returns `200` with an error body rather than `409`. - -**Fix**: read the `500` as a mode mismatch. Check the thread's marker, then restart the process in the thread's mode. There is no local fix that turns these into `409`. +**Fix**: confirm the mode mismatch first — the thread/state endpoints return `409` with the mode text, and a null usage alone is not conclusive because configuration loading or token counting can fail too. Once confirmed, restart the process in the thread's mode. ## Storage and materialization