mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
fix(gateway): map checkpoint-mode errors in run-prepare routes (#8296)
* fix(gateway): map checkpoint-mode errors in run-prepare routes The regenerate and edit-regenerate prepare routes read the same thread checkpoints as the threads router but built the accessor and read the latest checkpoint under a generic handler, so a delta/full mismatch surfaced as a 500 instead of the 409 that names the mode flip. Move the mapper to a shared module, reuse it for the accessor, latest-checkpoint and history-scan failures, and update the troubleshooting page now that the routes behave like the other state surfaces. * fix(gateway): preserve route-level HTTP errors in run-prepare The checkpoint-read fallback now sits around the accessor build, so a config-loading HTTPException from the accessor was being rewritten as a generic 500. Re-raise HTTPException before the generic handler and cover the 503 passthrough with a test. * docs: confirm the mode mismatch before advising a restart A null token usage or a /runs/wait error body can also come from a configuration or token-counting failure, so the troubleshooting entry should point at the 409 from the thread/state endpoints first. * fix(gateway): remove duplicate checkpoint imports after merge --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
parent
209255a874
commit
f95d4f6353
23
backend/app/gateway/checkpoint_mode_errors.py
Normal file
23
backend/app/gateway/checkpoint_mode_errors.py
Normal file
@ -0,0 +1,23 @@
|
||||
"""Map checkpoint-mode guard failures to precise HTTP statuses.
|
||||
|
||||
A mismatch means the thread's persisted checkpoints conflict with the process's
|
||||
frozen mode (operator-actionable, 409); a reconfiguration means the process
|
||||
itself is mid mode-flip (transient, 503). Both must surface their message — a
|
||||
generic 500 would force operators to grep logs to discover the root cause after
|
||||
a mode flip.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from deerflow.runtime.checkpoint_mode import CheckpointModeMismatchError, CheckpointModeReconfigurationError
|
||||
|
||||
CHECKPOINT_MODE_ERRORS = (CheckpointModeMismatchError, CheckpointModeReconfigurationError)
|
||||
|
||||
|
||||
def checkpoint_mode_http_error(exc: Exception, thread_id: str) -> HTTPException:
|
||||
"""Return the HTTP error for a checkpoint-mode guard failure."""
|
||||
if isinstance(exc, CheckpointModeMismatchError):
|
||||
return HTTPException(status_code=409, detail=f"Thread {thread_id}: {exc}")
|
||||
return HTTPException(status_code=503, detail=str(exc))
|
||||
@ -42,6 +42,12 @@ from app.gateway.checkpoint_lineage import (
|
||||
resolve_history_versions,
|
||||
resolve_stamp_candidate_versions,
|
||||
)
|
||||
from app.gateway.checkpoint_mode_errors import (
|
||||
CHECKPOINT_MODE_ERRORS as _CHECKPOINT_MODE_ERRORS,
|
||||
)
|
||||
from app.gateway.checkpoint_mode_errors import (
|
||||
checkpoint_mode_http_error as _checkpoint_mode_http_error,
|
||||
)
|
||||
from app.gateway.context_usage import build_context_usage
|
||||
from app.gateway.conversation_reader import (
|
||||
default_history_hidden_run_ids as _default_history_hidden_run_ids,
|
||||
@ -746,7 +752,10 @@ async def _find_base_checkpoint_before_human(
|
||||
*,
|
||||
head_checkpoint: Any | None = None,
|
||||
) -> Any:
|
||||
accessor, base_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id)
|
||||
try:
|
||||
accessor, base_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id)
|
||||
except _CHECKPOINT_MODE_ERRORS as exc:
|
||||
raise _checkpoint_mode_http_error(exc, thread_id) from exc
|
||||
if head_checkpoint is not None:
|
||||
try:
|
||||
return await find_checkpoint_before_message(
|
||||
@ -780,6 +789,8 @@ async def _find_base_checkpoint_before_human(
|
||||
versions, parent_versions = await resolve_stamp_candidate_versions(accessor, item, parent, version_cache)
|
||||
if not _is_duration_only_checkpoint(item, history_index, versions=versions, parent_versions=parent_versions):
|
||||
checkpoints.append(item)
|
||||
except _CHECKPOINT_MODE_ERRORS as exc:
|
||||
raise _checkpoint_mode_http_error(exc, thread_id) from exc
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to list checkpoints for regenerate thread %s", thread_id)
|
||||
raise HTTPException(status_code=500, detail="Failed to inspect checkpoint history") from exc
|
||||
@ -870,9 +881,14 @@ async def _find_interrupted_target_run_id(
|
||||
|
||||
|
||||
async def _prepare_regenerate_payload(thread_id: str, message_id: str, request: Request) -> RegeneratePrepareResponse:
|
||||
accessor, latest_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id)
|
||||
try:
|
||||
accessor, latest_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id)
|
||||
latest_checkpoint = await accessor.aget(latest_config)
|
||||
except _CHECKPOINT_MODE_ERRORS as exc:
|
||||
raise _checkpoint_mode_http_error(exc, thread_id) from exc
|
||||
except HTTPException:
|
||||
# Config-loading and other route-level errors keep their status/detail.
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to read latest checkpoint for regenerate thread %s", thread_id)
|
||||
raise HTTPException(status_code=500, detail="Failed to read latest checkpoint") from exc
|
||||
@ -962,9 +978,14 @@ async def _prepare_edit_regenerate_payload(
|
||||
if not normalized_text:
|
||||
raise HTTPException(status_code=409, detail="Edited message cannot be empty")
|
||||
|
||||
accessor, latest_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id)
|
||||
try:
|
||||
accessor, latest_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id)
|
||||
latest_checkpoint = await accessor.aget(latest_config)
|
||||
except _CHECKPOINT_MODE_ERRORS as exc:
|
||||
raise _checkpoint_mode_http_error(exc, thread_id) from exc
|
||||
except HTTPException:
|
||||
# Config-loading and other route-level errors keep their status/detail.
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to read latest checkpoint for edit replay thread %s", thread_id)
|
||||
raise HTTPException(status_code=500, detail="Failed to read latest checkpoint") from exc
|
||||
|
||||
@ -38,6 +38,12 @@ from app.gateway.checkpoint_lineage import (
|
||||
resolve_history_versions,
|
||||
resolve_stamp_candidate_versions,
|
||||
)
|
||||
from app.gateway.checkpoint_mode_errors import (
|
||||
CHECKPOINT_MODE_ERRORS as _CHECKPOINT_MODE_ERRORS,
|
||||
)
|
||||
from app.gateway.checkpoint_mode_errors import (
|
||||
checkpoint_mode_http_error as _checkpoint_mode_http_error,
|
||||
)
|
||||
from app.gateway.deps import get_checkpointer, get_config, get_current_user_from_request, get_run_event_store, get_run_manager, get_run_store
|
||||
from app.gateway.history_seed import reserve_history_seed, write_history_seed
|
||||
from app.gateway.hitl_approval import find_pending_interrupts
|
||||
@ -59,7 +65,6 @@ from deerflow.config.paths import Paths, get_paths
|
||||
from deerflow.config.summarization_config import ContextSize
|
||||
from deerflow.persistence.thread_meta import PROJECT_FILTER_UNSET, THREAD_ARCHIVED_METADATA_KEY, THREAD_PINNED_METADATA_KEY, THREAD_PROJECT_METADATA_KEY, ThreadOwnershipConflictError
|
||||
from deerflow.runtime import ThreadOperationKind, serialize_channel_values_for_api
|
||||
from deerflow.runtime.checkpoint_mode import CheckpointModeMismatchError, CheckpointModeReconfigurationError
|
||||
from deerflow.runtime.checkpoint_state import graph_reducer_channels, graph_state_schema, graph_writable_channels, validate_state_mutation_node
|
||||
from deerflow.runtime.context_compaction import (
|
||||
ContextCompactionDisabled,
|
||||
@ -102,8 +107,6 @@ from deerflow.utils.time import coerce_iso, now_iso
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/threads", tags=["threads"])
|
||||
|
||||
_CHECKPOINT_MODE_ERRORS = (CheckpointModeMismatchError, CheckpointModeReconfigurationError)
|
||||
|
||||
|
||||
def _optional_run_event_store(request: Request) -> Any:
|
||||
"""Return the run event store, or ``None`` when the app has none wired.
|
||||
@ -116,20 +119,6 @@ def _optional_run_event_store(request: Request) -> Any:
|
||||
return getattr(request.app.state, "run_event_store", None)
|
||||
|
||||
|
||||
def _checkpoint_mode_http_error(exc: Exception, thread_id: str) -> HTTPException:
|
||||
"""Map checkpoint-mode guard failures to precise HTTP statuses.
|
||||
|
||||
A mismatch means the thread's persisted checkpoints conflict with the
|
||||
process's frozen mode (operator-actionable, 409); a reconfiguration means
|
||||
the process itself is mid mode-flip (transient, 503). Both must surface
|
||||
their message — a generic 500 would force operators to grep logs to
|
||||
discover the root cause after a mode flip.
|
||||
"""
|
||||
if isinstance(exc, CheckpointModeMismatchError):
|
||||
return HTTPException(status_code=409, detail=f"Thread {thread_id}: {exc}")
|
||||
return HTTPException(status_code=503, detail=str(exc))
|
||||
|
||||
|
||||
# Metadata keys that the server controls; clients are not allowed to set
|
||||
# them. Pydantic ``@field_validator("metadata")`` strips them on every
|
||||
# inbound model below so a malicious client cannot reflect a forged
|
||||
|
||||
@ -19,6 +19,7 @@ from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from _router_auth_helpers import make_authed_test_app
|
||||
from fastapi import HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
@ -27,7 +28,7 @@ from langgraph.store.memory import InMemoryStore
|
||||
|
||||
from app.gateway import context_usage
|
||||
from app.gateway import services as gateway_services
|
||||
from app.gateway.routers import threads
|
||||
from app.gateway.routers import thread_runs, threads
|
||||
from deerflow.agents.thread_state import get_thread_state_schema
|
||||
from deerflow.config.app_config import AppConfig, reset_app_config, set_app_config
|
||||
from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore
|
||||
@ -319,3 +320,79 @@ def test_mutation_accessor_fails_closed_when_thread_metadata_lookup_fails(monkey
|
||||
)
|
||||
|
||||
resolve_factory.assert_not_called()
|
||||
|
||||
|
||||
def test_full_mode_run_prepare_routes_reject_delta_thread_with_409(_stub_app_config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The regenerate-prepare routes must surface the mode mismatch as 409.
|
||||
|
||||
They read the same thread state as the threads router, so a full-mode
|
||||
process opening a delta thread must produce the same actionable 409 rather
|
||||
than a generic 500 that hides the mode flip.
|
||||
"""
|
||||
app = make_authed_test_app()
|
||||
store = InMemoryStore()
|
||||
checkpointer = InMemorySaver()
|
||||
app.state.store = store
|
||||
app.state.checkpointer = checkpointer
|
||||
app.state.thread_store.get = AsyncMock(return_value=None)
|
||||
app.state.checkpoint_channel_mode = "full"
|
||||
app.state.run_event_store = SimpleNamespace()
|
||||
app.state.run_manager = RunManager(store=MemoryRunStore())
|
||||
app.include_router(threads.router)
|
||||
app.include_router(thread_runs.router)
|
||||
|
||||
full_graph = _build_reply_graph("full", checkpointer)
|
||||
monkeypatch.setattr(
|
||||
gateway_services,
|
||||
"resolve_agent_factory",
|
||||
lambda assistant_id=None: lambda config: full_graph,
|
||||
)
|
||||
|
||||
delta_graph = _build_reply_graph("delta", checkpointer)
|
||||
config: dict[str, Any] = {"configurable": {"thread_id": _THREAD_ID}}
|
||||
inject_checkpoint_mode(config, "delta")
|
||||
asyncio.run(delta_graph.ainvoke({"messages": [HumanMessage(content="question", id="h0")]}, config))
|
||||
|
||||
with TestClient(app) as client:
|
||||
regenerate = client.post(
|
||||
f"/api/threads/{_THREAD_ID}/runs/regenerate/prepare",
|
||||
json={"message_id": "a0"},
|
||||
)
|
||||
assert regenerate.status_code == 409, regenerate.text
|
||||
assert "requires delta mode" in regenerate.json()["detail"]
|
||||
assert _THREAD_ID in regenerate.json()["detail"]
|
||||
|
||||
edit_regenerate = client.post(
|
||||
f"/api/threads/{_THREAD_ID}/runs/edit-regenerate/prepare",
|
||||
json={"human_message_id": "h0", "replacement_text": "edited"},
|
||||
)
|
||||
assert edit_regenerate.status_code == 409, edit_regenerate.text
|
||||
assert "requires delta mode" in edit_regenerate.json()["detail"]
|
||||
assert _THREAD_ID in edit_regenerate.json()["detail"]
|
||||
|
||||
|
||||
def test_run_prepare_propagates_accessor_http_errors(_stub_app_config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Route-level errors from the accessor keep their status and detail.
|
||||
|
||||
Configuration loading raises a typed HTTP error; the generic checkpoint-read
|
||||
handler must not turn it into a misleading 500.
|
||||
"""
|
||||
app = make_authed_test_app()
|
||||
app.state.thread_store.get = AsyncMock(return_value=None)
|
||||
app.state.run_manager = RunManager(store=MemoryRunStore())
|
||||
app.state.checkpoint_channel_mode = "full"
|
||||
app.state.run_event_store = SimpleNamespace()
|
||||
app.include_router(thread_runs.router)
|
||||
|
||||
async def raise_unavailable(*args: Any, **kwargs: Any) -> Any:
|
||||
raise HTTPException(status_code=503, detail="Configuration not available")
|
||||
|
||||
monkeypatch.setattr(thread_runs, "build_thread_checkpoint_state_accessor", raise_unavailable)
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.post(
|
||||
f"/api/threads/{_THREAD_ID}/runs/regenerate/prepare",
|
||||
json={"message_id": "a0"},
|
||||
)
|
||||
assert response.status_code == 503, response.text
|
||||
assert response.json()["detail"] == "Configuration not available"
|
||||
|
||||
@ -18,7 +18,7 @@ This chapter is organized by **what you see**. Each entry gives the cause, the f
|
||||
| A run fails with a restart-required message | a config edit the frozen process rejects when it builds the graph | [A run fails after a config edit](#a-run-fails-after-a-config-edit) |
|
||||
| A config edit seems ignored | mode and cadence are frozen at agent-build time | [A config change appears to have no effect](#a-config-change-appears-to-have-no-effect) |
|
||||
| A history-cache edit seems ignored | the cache is built with the checkpointer at startup | [A history-cache change appears to have no effect](#a-history-cache-change-appears-to-have-no-effect) |
|
||||
| Regenerate endpoints return `500` | the run-runs router does not map the mode errors | [Regenerate endpoints return 500 instead of 409](#regenerate-endpoints-return-500-instead-of-409) |
|
||||
| Token usage is null or `/runs/wait` hides a mode mismatch | those routes do not map the mode guard | [Checkpoint-mode mismatches on auxiliary endpoints](#checkpoint-mode-mismatches-on-auxiliary-endpoints) |
|
||||
| Delta did not shrink storage | still in full mode, or the writes table was not counted | [Delta did not shrink storage](#delta-did-not-shrink-storage) |
|
||||
| Materialization got slower | snapshot cadence raised too far | [Materialization got slower after raising the cadence](#materialization-got-slower-after-raising-the-cadence) |
|
||||
| The replaced answer comes back after regenerate | delta fork replay (#4458) | [The old answer reappears after regenerate](#the-old-answer-reappears-after-regenerate) |
|
||||
@ -91,20 +91,13 @@ This is not the frozen-for-correctness case: nothing about the cache can corrupt
|
||||
|
||||
**Fix**: restart the Gateway after editing the section — and on the embedded or TUI path, restart that process, whose checkpointer singleton is also created once and reused. Only `database.checkpoint_graph_cache.accessor_graph_max` applies without a restart, because it is re-read on every eviction check.
|
||||
|
||||
### Regenerate endpoints return 500 instead of 409
|
||||
### Checkpoint-mode mismatches on auxiliary endpoints
|
||||
|
||||
**Symptom**: regenerate preparation returns `500` with `detail: "Failed to read latest checkpoint"` or `detail: "Failed to inspect checkpoint history"`, even though the same thread yields `409` elsewhere.
|
||||
**Symptom**: `GET /api/threads/{thread_id}/token-usage` returns `200` with `context_usage: null`, or `/runs/wait` returns `200` with an error body, while the same thread yields `409` from the state and thread endpoints.
|
||||
|
||||
**Cause**: `POST /api/threads/{thread_id}/runs/regenerate/prepare` (and `/runs/edit-regenerate/prepare`) build the state accessor outside any `try`, and this module does not map the checkpoint-mode errors, so a mismatch surfaces as an unhandled `500`. The log lines are
|
||||
**Cause**: these routes do not map the checkpoint-mode guard. The token-usage path logs `Failed to load checkpoint for context usage on thread %s` and returns an empty usage; `/runs/wait` reports the run's error body instead of the HTTP status. The regenerate and edit-regenerate preparation routes used to fail the same way with `500`; they now reuse the shared `checkpoint_mode_errors` mapper and return the same `409`/`503` as the state endpoints.
|
||||
|
||||
```
|
||||
Failed to read latest checkpoint for regenerate thread %s
|
||||
Failed to list checkpoints for regenerate thread %s
|
||||
```
|
||||
|
||||
Two other endpoints fail more quietly for the same reason: `GET /api/threads/{thread_id}/token-usage` returns `200` with `context_usage: null` (only `Failed to load checkpoint for context usage on thread %s` is logged), and `/runs/wait` returns `200` with an error body rather than `409`.
|
||||
|
||||
**Fix**: read the `500` as a mode mismatch. Check the thread's marker, then restart the process in the thread's mode. There is no local fix that turns these into `409`.
|
||||
**Fix**: confirm the mode mismatch first — the thread/state endpoints return `409` with the mode text, and a null usage alone is not conclusive because configuration loading or token counting can fail too. Once confirmed, restart the process in the thread's mode.
|
||||
|
||||
## Storage and materialization
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user