From 625c07b9934d59891554cb24aa03d48cc05dbc68 Mon Sep 17 00:00:00 2001
From: Vanzeren <53075619+Vanzeren@users.noreply.github.com>
Date: Mon, 27 Jul 2026 11:32:10 +0800
Subject: [PATCH 01/15] fix(runtime): resume original title when regenerating
(#4480)
* fix(runtime): rusume original title when regenerating
* test(runtime): cover regenerated title sync
---
README.md | 2 +-
backend/AGENTS.md | 2 +-
backend/app/gateway/routers/thread_runs.py | 11 +++++-
backend/tests/test_run_worker_delta_resume.py | 24 ++++++++++---
.../tests/test_thread_regenerate_prepare.py | 34 +++++++++++++++++++
5 files changed, 66 insertions(+), 7 deletions(-)
diff --git a/README.md b/README.md
index 14d98e3e0..d31e6d961 100644
--- a/README.md
+++ b/README.md
@@ -778,7 +778,7 @@ Interrupted first-turn runs still persist a fallback conversation title, so stop
Streaming Markdown responses animate only newly arrived words; text that is already visible is not faded out and replayed when the next chunk extends the same block.
-In the Web UI, completed assistant turns can be branched into a new main conversation. The new thread starts from that turn's checkpoint and keeps the preceding replay checkpoint, so the branched response can be regenerated immediately. Legacy or imported histories without checkpoint parent links use a bounded chronological fallback; if no earlier replay checkpoint exists, branching still succeeds with the legacy single-checkpoint shape, while regeneration remains unavailable for that inherited response. Existing single-checkpoint branches are left unchanged rather than attempting an unsafe checkpoint copy. Because workspace files are not checkpointed, the branch only receives a best-effort copy of the current workspace when you branch from the latest turn; branching from an older turn keeps just the restored message history so the branch never inherits files that were created in a later part of the conversation.
+In the Web UI, completed assistant turns can be branched into a new main conversation. The new thread starts from that turn's checkpoint and keeps the preceding replay checkpoint, so the branched response can be regenerated immediately. Regenerating the latest response preserves the thread's current title, including a title you renamed manually after the original response. Legacy or imported histories without checkpoint parent links use a bounded chronological fallback; if no earlier replay checkpoint exists, branching still succeeds with the legacy single-checkpoint shape, while regeneration remains unavailable for that inherited response. Existing single-checkpoint branches are left unchanged rather than attempting an unsafe checkpoint copy. Because workspace files are not checkpointed, the branch only receives a best-effort copy of the current workspace when you branch from the latest turn; branching from an older turn keeps just the restored message history so the branch never inherits files that were created in a later part of the conversation.
The Web UI reports completed task time once per run. This is total wall-clock time—including model reasoning, tool calls, and waiting—not a per-step or model-only thinking duration. Reasoning content remains available through its own separate disclosure.
diff --git a/backend/AGENTS.md b/backend/AGENTS.md
index 174646b3f..56ebe7e19 100644
--- a/backend/AGENTS.md
+++ b/backend/AGENTS.md
@@ -439,7 +439,7 @@ Localhost persistence deliberately reads the direct request `Host` and ignores `
| **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types |
| **Suggestions** (`/api/suggestions`) | `GET /config` - returns global suggestions config boolean; `POST /threads/{id}/suggestions` - generate follow-up questions; rich list/block model content is normalized and inline reasoning (`...`, including unclosed/truncated blocks from reasoning models like MiniMax-M3) is stripped before JSON parsing |
| **Input Polish** (`/api/input-polish`) | `POST /` - rewrite a composer draft before it is sent. This is a short authenticated `runs:create` LLM request using `input_polish` config; it does not create a LangGraph run, persist a message, or modify thread state. Shares the non-graph one-shot LLM path (`deerflow.utils.oneshot_llm.run_oneshot_llm`) with the suggestions route so model build + Langfuse metadata + invoke stay in one place; validates the same stripped view of the draft it sends to the model, and preserves literal `` substrings in the rewrite (`strip_think_blocks(truncate_unclosed=False)`) |
-| **Thread Runs** (`/api/threads/{id}/runs`) | `POST /` - create background run; `POST /stream` - create + SSE stream; `POST /wait` - create + block; `POST /regenerate/prepare` - prepare clean input + checkpoint metadata for regenerating the latest assistant answer; `GET /` - list runs; `GET /{rid}` - run details; `POST /{rid}/cancel` - cancel; `GET /{rid}/join` - join SSE; `GET /{rid}/messages` - paginated per-run messages `{data, has_more}`; `GET /{rid}/events` - full event stream; `GET /{rid}/workspace-changes` - workspace/output file change summary and optional diffs; `GET /../messages` - legacy thread message array; `GET /../messages/page` - backward thread-global `seq` history page with middleware/subagent-AI/successful-regenerate filtering and page-run-scoped feedback enrichment; subagent AI callbacks remain available through run events while parent `task` ToolMessages stay visible for card restoration; `GET /../token-usage` - aggregate tokens |
+| **Thread Runs** (`/api/threads/{id}/runs`) | `POST /` - create background run; `POST /stream` - create + SSE stream; `POST /wait` - create + block; `POST /regenerate/prepare` - prepare clean input + checkpoint metadata for regenerating the latest assistant answer, carrying the latest non-empty thread title in graph input so resuming an older checkpoint cannot roll back a later manual rename (#4457); `GET /` - list runs; `GET /{rid}` - run details; `POST /{rid}/cancel` - cancel; `GET /{rid}/join` - join SSE; `GET /{rid}/messages` - paginated per-run messages `{data, has_more}`; `GET /{rid}/events` - full event stream; `GET /{rid}/workspace-changes` - workspace/output file change summary and optional diffs; `GET /../messages` - legacy thread message array; `GET /../messages/page` - backward thread-global `seq` history page with middleware/subagent-AI/successful-regenerate filtering and page-run-scoped feedback enrichment; subagent AI callbacks remain available through run events while parent `task` ToolMessages stay visible for card restoration; `GET /../token-usage` - aggregate tokens |
| **Feedback** (`/api/threads/{id}/runs/{rid}/feedback`) | `PUT /` - upsert feedback; `DELETE /` - delete user feedback; `POST /` - create feedback; `GET /` - list feedback; `GET /stats` - aggregate stats; `DELETE /{fid}` - delete specific |
| **Runs** (`/api/runs`) | `POST /stream` - stateless run + SSE; `POST /wait` - stateless run + block; `GET /{rid}/messages` - paginated messages by run_id `{data, has_more}` (cursor: `after_seq`/`before_seq`); `GET /{rid}/feedback` - list feedback by run_id |
| **GitHub Webhooks** (`/api/webhooks/github`) | `POST /` - receive GitHub App / repo webhook deliveries. Verifies `X-Hub-Signature-256` against `GITHUB_WEBHOOK_SECRET`; exempt from auth + CSRF because authenticity is enforced by HMAC. The route is fail-closed: mounted only when `GITHUB_WEBHOOK_SECRET` is set, or when explicit dev opt-in `DEER_FLOW_ALLOW_UNVERIFIED_GITHUB_WEBHOOKS=1` is set. Recognized events include `ping`, `issues`, `issue_comment`, `pull_request`, `pull_request_review`, and `pull_request_review_comment`; unknown events return 200 with `handled=false`. Fan-out runtime failures return 503, keeping the delivery recorded as failed for manual/API/scripted redelivery (GitHub does not automatically retry any failed delivery, 5xx included); permanent/non-retryable conditions such as `channels.github.enabled: false`, unknown events, malformed payloads, or unavailable channel service return 200 with a skipped/handled response. |
diff --git a/backend/app/gateway/routers/thread_runs.py b/backend/app/gateway/routers/thread_runs.py
index 3060e0fe9..77905389e 100644
--- a/backend/app/gateway/routers/thread_runs.py
+++ b/backend/app/gateway/routers/thread_runs.py
@@ -490,8 +490,17 @@ async def _prepare_regenerate_payload(thread_id: str, message_id: str, request:
"regenerate_from_run_id": target_run_id,
"regenerate_checkpoint_id": checkpoint["checkpoint_id"],
}
+ regenerate_input: dict[str, Any] = {"messages": [_clean_human_message_for_regenerate(previous_human)]}
+ latest_values = latest_checkpoint.values if isinstance(latest_checkpoint.values, dict) else {}
+ latest_title = latest_values.get("title")
+ if isinstance(latest_title, str) and latest_title:
+ # Regenerate resumes from the checkpoint before the target human turn.
+ # That checkpoint can predate a manual rename, so replay the current
+ # title as graph input instead of letting checkpoint rollback restore
+ # the older automatically generated title (#4457).
+ regenerate_input["title"] = latest_title
return RegeneratePrepareResponse(
- input={"messages": [_clean_human_message_for_regenerate(previous_human)]},
+ input=regenerate_input,
checkpoint=checkpoint,
metadata=metadata,
target_run_id=target_run_id,
diff --git a/backend/tests/test_run_worker_delta_resume.py b/backend/tests/test_run_worker_delta_resume.py
index d64152bf4..6113fb21e 100644
--- a/backend/tests/test_run_worker_delta_resume.py
+++ b/backend/tests/test_run_worker_delta_resume.py
@@ -41,6 +41,7 @@ def anyio_backend():
class _DeltaChannelState(TypedDict):
messages: Annotated[list[AnyMessage], DeltaChannel(merge_message_writes, snapshot_frequency=1000)]
+ title: str | None
class _FullChannelState(TypedDict):
@@ -226,12 +227,18 @@ async def test_run_agent_streams_from_the_linearized_delta_resume(tmp_path):
branch_writer = CheckpointStateAccessor.bind(mutation_graph, checkpointer, mode="delta")
replay_base_config = await branch_writer.aupdate(
_run_config("worker-branch"),
- {"messages": Overwrite(list(source_pre_turn.values["messages"]))},
+ {
+ "messages": Overwrite(list(source_pre_turn.values["messages"])),
+ "title": "Original title",
+ },
as_node="branch",
)
await branch_writer.aupdate(
replay_base_config,
- {"messages": Overwrite(list(source_head.values["messages"]))},
+ {
+ "messages": Overwrite(list(source_head.values["messages"])),
+ "title": "User renamed title",
+ },
as_node="branch",
)
@@ -247,6 +254,10 @@ async def test_run_agent_streams_from_the_linearized_delta_resume(tmp_path):
publish_end=AsyncMock(),
cleanup=AsyncMock(),
)
+ thread_store = SimpleNamespace(
+ update_display_name=AsyncMock(),
+ update_status=AsyncMock(),
+ )
created_graphs: list[Any] = []
def agent_factory(*, config):
@@ -260,9 +271,12 @@ async def test_run_agent_streams_from_the_linearized_delta_resume(tmp_path):
bridge,
run_manager,
record,
- ctx=RunContext(checkpointer=checkpointer, checkpoint_channel_mode="delta"),
+ ctx=RunContext(checkpointer=checkpointer, thread_store=thread_store, checkpoint_channel_mode="delta"),
agent_factory=agent_factory,
- graph_input={"messages": [HumanMessage(content="q2", id="h2")]},
+ graph_input={
+ "messages": [HumanMessage(content="q2", id="h2")],
+ "title": "User renamed title",
+ },
config=_run_config("worker-branch", base.config["configurable"]["checkpoint_id"]),
stream_modes=["values"],
),
@@ -273,6 +287,8 @@ async def test_run_agent_streams_from_the_linearized_delta_resume(tmp_path):
final_accessor = CheckpointStateAccessor.bind(created_graphs[-1], checkpointer, mode="delta")
final = await final_accessor.aget(_run_config("worker-branch"))
assert _ids(final) == ["h1", "a1", "h2", "a2-new"]
+ assert final.values["title"] == "User renamed title"
+ thread_store.update_display_name.assert_awaited_once_with("worker-branch", "User renamed title")
async def test_run_agent_serializes_resume_preparation_with_checkpoint_writes(monkeypatch):
diff --git a/backend/tests/test_thread_regenerate_prepare.py b/backend/tests/test_thread_regenerate_prepare.py
index 7dec0652f..21958343b 100644
--- a/backend/tests/test_thread_regenerate_prepare.py
+++ b/backend/tests/test_thread_regenerate_prepare.py
@@ -390,12 +390,46 @@ def test_prepare_regenerate_payload_returns_clean_input_and_base_checkpoint():
"regenerate_from_run_id": "run-old",
"regenerate_checkpoint_id": "ckpt-base",
}
+ assert "title" not in response.input
regenerated_human = response.input["messages"][0]
assert regenerated_human["id"] == "human-1"
assert regenerated_human["content"] == [{"type": "text", "text": "/data-analysis analyze data.csv"}]
assert regenerated_human["additional_kwargs"] == {"files": [{"filename": "data.csv", "path": "/mnt/user-data/uploads/data.csv"}]}
+def test_prepare_regenerate_payload_preserves_latest_thread_title():
+ from app.gateway.routers import thread_runs
+
+ human = HumanMessage(id="human-1", content="question")
+ ai = AIMessage(id="ai-1", content="answer v1")
+ base = _checkpoint("ckpt-base", [])
+ latest = _checkpoint("ckpt-ai", [human, ai])
+ latest.checkpoint["channel_values"]["title"] = "User renamed title"
+ checkpointer = FakeCheckpointer([latest, base])
+ event_store = FakeEventStore(
+ [
+ {
+ "run_id": "run-old",
+ "event_type": "llm.ai.response",
+ "category": "message",
+ "content": {"id": "ai-1", "type": "ai", "content": "answer v1"},
+ "metadata": {"caller": "lead_agent"},
+ }
+ ]
+ )
+
+ response = asyncio.run(
+ thread_runs._prepare_regenerate_payload(
+ "thread-1",
+ "ai-1",
+ _request(checkpointer, event_store),
+ )
+ )
+
+ assert response.checkpoint["checkpoint_id"] == "ckpt-base"
+ assert response.input["title"] == "User renamed title"
+
+
def test_prepare_regenerate_payload_does_not_mutate_legacy_single_checkpoint_branch():
from app.gateway.routers.thread_runs import _prepare_regenerate_payload
From e01173d8b2ac372c4711c061c28e83f7a6b9cf2c Mon Sep 17 00:00:00 2001
From: Vanzeren <53075619+Vanzeren@users.noreply.github.com>
Date: Mon, 27 Jul 2026 11:47:49 +0800
Subject: [PATCH 02/15] bench(checkpoint): production-shaped full/delta
benchmark with configurable snapshot frequency (#4467)
* feat(checkpoint): production-shaped full/delta benchmark with configurable snapshot frequency
- Group benchmark scripts into per-family folders (checkpoint/, sandbox/)
- Extract shared benchmark infrastructure into checkpoint_bench_common.py
- Add checkpoint_delta_snapshot_frequency config (default 1000, process-frozen);
freeze it in make_lead_agent and DeerFlowClient; key the state-schema
adaptation cache by resolved frequency
- New bench_production.py: per-case child processes run N ainvoke turns through
the real lead-agent graph (scripted deterministic model, real AsyncSqliteSaver),
then measure GET /state + POST /history through the real Gateway route stack
in one event loop (httpx ASGITransport), cold/warm accessor-cache split,
cross-mode digest gates
- New summarize_production.py: delta/full ratios plus decision metrics
(snapshot_write_spike, cache_effect_ms, checkpoint_write_share,
auto-discovered history per-limit ratios)
* fix(checkpoint): address production benchmark review
---
README.md | 4 +
backend/AGENTS.md | 66 +-
.../deerflow/agents/lead_agent/agent.py | 3 +-
.../harness/deerflow/agents/thread_state.py | 67 +-
backend/packages/harness/deerflow/client.py | 3 +-
.../deerflow/config/database_config.py | 10 +
.../bench_channels.py} | 195 +----
.../benchmark/checkpoint/bench_production.py | 716 ++++++++++++++++++
.../checkpoint/checkpoint_bench_common.py | 208 +++++
.../summarize_channels.py} | 6 +-
.../checkpoint/summarize_production.py | 199 +++++
.../bench_provider.py} | 4 +-
.../summarize.py} | 8 +-
backend/tests/conftest.py | 14 +
.../tests/test_bench_checkpoint_channels.py | 2 +-
.../tests/test_bench_checkpoint_production.py | 237 ++++++
backend/tests/test_bench_sandbox_provider.py | 4 +-
backend/tests/test_checkpoint_mode.py | 25 +
backend/tests/test_checkpointer.py | 2 +
backend/tests/test_client.py | 18 +
backend/tests/test_delta_channel_state.py | 44 ++
backend/tests/test_lead_agent_skills.py | 24 +
.../test_summarize_checkpoint_channels.py | 2 +-
.../test_summarize_checkpoint_production.py | 128 ++++
backend/tests/test_token_usage.py | 1 +
25 files changed, 1785 insertions(+), 205 deletions(-)
rename backend/scripts/benchmark/{bench_checkpoint_channels.py => checkpoint/bench_channels.py} (78%)
create mode 100755 backend/scripts/benchmark/checkpoint/bench_production.py
create mode 100755 backend/scripts/benchmark/checkpoint/checkpoint_bench_common.py
rename backend/scripts/benchmark/{summarize_checkpoint_channels.py => checkpoint/summarize_channels.py} (97%)
create mode 100755 backend/scripts/benchmark/checkpoint/summarize_production.py
rename backend/scripts/benchmark/{bench_sandbox_provider.py => sandbox/bench_provider.py} (99%)
rename backend/scripts/benchmark/{summarize_bench.py => sandbox/summarize.py} (95%)
create mode 100644 backend/tests/test_bench_checkpoint_production.py
create mode 100644 backend/tests/test_summarize_checkpoint_production.py
diff --git a/README.md b/README.md
index d31e6d961..94857b64e 100644
--- a/README.md
+++ b/README.md
@@ -255,6 +255,10 @@ single-label cluster hosts, and Docker/Podman internal hostnames do not inherit
honor environment proxy settings.
Backend processes automatically pick up `config.yaml` changes on the next config access, so model metadata updates do not require a manual restart during development.
+The checkpoint storage settings `database.checkpoint_channel_mode` and
+`database.checkpoint_delta_snapshot_frequency` are exceptions: both are frozen
+when the process first builds an agent (including through `DeerFlowClient`) and
+require a process restart to change safely.
> [!TIP]
> On Linux, if Docker-based commands fail with `permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock`, add your user to the `docker` group and re-login before retrying. See [CONTRIBUTING.md](CONTRIBUTING.md#linux-docker-daemon-permission-denied) for the full fix.
diff --git a/backend/AGENTS.md b/backend/AGENTS.md
index 56ebe7e19..79948b0f3 100644
--- a/backend/AGENTS.md
+++ b/backend/AGENTS.md
@@ -902,7 +902,7 @@ This invokes `alembic revision --autogenerate` against the live ORM models. Revi
Checkpointer storage runs in one of two channel modes, selected by `checkpoint_channel_mode` in `config.yaml` (default `full`). `delta` mode adopts LangGraph 1.2's `DeltaChannel` for `messages`: checkpoints store a sentinel + per-step writes instead of the full message list, so storage/serde grows O(N) instead of O(N²) in turns. All checkpointer backends (memory/sqlite/postgres) serve both modes unchanged — the semantics live in the compiled graph's channel table, not in the saver.
-**Mode is process-frozen and restart-required.** `make_lead_agent` freezes the resolved mode (`runtime/checkpoint_mode.py::freeze_checkpoint_channel_mode`) before compiling the graph with the mode-matched schema (`agents/thread_state.py::get_thread_state_schema`, plus `adapt_state_schema_for_mode` / `normalize_middleware_state_schemas` for middleware state). A second, different mode in the same process raises `CheckpointModeReconfigurationError`. To switch: edit config, restart.
+**Mode is process-frozen and restart-required.** `make_lead_agent` and the embedded `DeerFlowClient` freeze the resolved mode (`runtime/checkpoint_mode.py::freeze_checkpoint_channel_mode`) and the delta snapshot frequency (`agents/thread_state.py::freeze_delta_snapshot_frequency`, from the `checkpoint_delta_snapshot_frequency` config knob, default 1000 — restart-required like the mode) before compiling the graph with the mode-matched schema (`agents/thread_state.py::get_thread_state_schema`, plus `adapt_state_schema_for_mode` / `normalize_middleware_state_schemas` for middleware state). Adapted middleware schemas are cached by schema, mode, and resolved snapshot frequency so a pre-freeze ephemeral graph cannot leave a stale default-frequency schema behind. A second, different mode or frequency in the same process raises `CheckpointModeReconfigurationError`. To switch: edit config, restart.
**Compatibility is asymmetric and fail-closed.** Every checkpoint written in delta mode carries metadata marker `deerflow_checkpoint_channel_mode: "delta"` (injected via `inject_checkpoint_mode`; absence of marker = full, so pre-feature checkpoints need no migration). Before any state read/write, `ensure_checkpoint_mode_compatible` rejects a full-mode process opening a delta thread with `CheckpointModeMismatchError` (surfaced as HTTP 409 with the cause and thread id by the threads router; `CheckpointModeReconfigurationError` maps to 503) — a full-mode raw read of a delta blob would silently return empty/partial `messages`. The reverse direction is allowed: delta-mode processes read full checkpoints transparently (old full checkpoints seed the delta channel), so full → delta is the smooth migration path; delta → full requires materializing/converting the data first. Detection also honors upstream's `counters_since_delta_snapshot.messages` metadata, and an explicit config marker takes precedence over any ambient context value.
@@ -920,11 +920,11 @@ Checkpointer storage runs in one of two channel modes, selected by `checkpoint_c
- `runtime/checkpoint_mode.py` — mode freeze, marker injection, delta detection, compatibility gate, both error types
- `runtime/checkpoint_state.py` — `CheckpointStateAccessor`, `build_state_mutation_graph`, `RollbackPoint`
- `checkpoint_patches.py` (package root) — checkpoint-machinery patches: delta-history folding for `InMemorySaver` (delegating to the base walk), stable message IDs across materialization, upstream first-write drop fix, and `BinaryOperatorAggregate` unwrapping an `Overwrite` first write into an empty (MISSING) channel — Union-typed reducer channels (`sandbox`/`goal`/`todos`/`promoted`) have no constructible default, so a replace-style write into a fresh branch thread or a never-written channel stored the wrapper literally and crashed the next consumer (#4380; probe-guarded, stands down if upstream fixes it)
-- `agents/thread_state.py` — `ThreadState`/`DeltaThreadState`, `DELTA_MESSAGES_FIELD` (`DeltaChannel` with `snapshot_frequency=1000`), schema adaptation helpers
+- `agents/thread_state.py` — `ThreadState`/`DeltaThreadState`, `DELTA_MESSAGES_FIELD` (`DeltaChannel` whose `snapshot_frequency` resolves from the `checkpoint_delta_snapshot_frequency` config knob via `freeze_delta_snapshot_frequency`/`resolved_delta_snapshot_frequency`; 1000 is only the default), schema adaptation helpers
- `runtime/context_compaction.py` — compaction via accessor + mutation graph (reference consumer)
- Tests: `tests/test_checkpoint_mode.py` (freeze/detect/gate), `tests/test_checkpoint_state.py` (accessor/mutation graph), `tests/test_delta_channel_checkpointers.py` (saver parity), `tests/test_threads_checkpoint_mode.py`, `tests/test_gateway_checkpoint_mode.py` (dual-mode e2e parity), `tests/test_context_compaction.py` (mutation-graph write, no scheduling), `tests/test_run_worker_rollback.py`
-**Checkpoint channel benchmark**: `scripts/benchmark/bench_checkpoint_channels.py`
+**Checkpoint channel benchmark**: `scripts/benchmark/checkpoint/bench_channels.py`
runs paired `full`/`delta` message-only StateGraphs in a fresh child process per
case, using sync `InMemorySaver` or `SqliteSaver` so reducer, serialization, and
saver costs stay separate from Gateway/async scheduling. It reports deterministic
@@ -937,7 +937,7 @@ estimated cumulative full-payload cap skips both modes of an oversized pair when
full-payload cap, so size those runs explicitly. Use `--allow-large-cases` only
on a provisioned machine. Duplicate CSV matrix values are ignored with a warning;
use `--repetitions` for repeated samples. Summarize paired successful repetitions
-with `scripts/benchmark/summarize_checkpoint_channels.py` (all ratios are
+with `scripts/benchmark/checkpoint/summarize_channels.py` (all ratios are
`delta/full`). `--profile-dir /tmp/checkpoint-profiles` writes one cProfile
artifact per case for attribution. Profiled rows carry `profiled: true`, and the
summarizer automatically excludes them from baseline summaries with a warning.
@@ -948,19 +948,61 @@ Example:
```bash
cd backend
-PYTHONPATH=. uv run python scripts/benchmark/bench_checkpoint_channels.py \
+PYTHONPATH=. uv run python scripts/benchmark/checkpoint/bench_channels.py \
--backends sqlite --updates 100,500,999,1000,1001 --payload-bytes 128 \
--repetitions 7 --output /tmp/checkpoint-bench.jsonl
-PYTHONPATH=. uv run python scripts/benchmark/summarize_checkpoint_channels.py \
+PYTHONPATH=. uv run python scripts/benchmark/checkpoint/summarize_channels.py \
/tmp/checkpoint-bench.jsonl
```
-The sync storage benchmark is not an end-to-end Gateway benchmark. Complete
-`ThreadState`/`DeltaThreadState`, async saver scheduling, history, mutation,
-rollback, migration, and branch-heavy cases belong to the production-shaped
-follow-up layer. Harness tests live in `tests/test_bench_checkpoint_channels.py`
-and `tests/test_summarize_checkpoint_channels.py`; timing thresholds are not CI
-gates.
+The production-shaped layer lives in
+`scripts/benchmark/checkpoint/bench_production.py`: per-case child processes
+run graph-level `ainvoke` turns through the real lead-agent graph (scripted
+deterministic model, real `AsyncSqliteSaver`), then measure
+`GET /threads/{id}/state` and `POST /threads/{id}/history` through the real
+Gateway route stack in the same event loop (httpx ASGITransport), split into
+cold/warm accessor-graph-cache samples. It sweeps `snapshot_frequency`
+(config: `checkpoint_delta_snapshot_frequency`, process-frozen like the
+mode), pairs every delta frequency against the same full row, and fails both
+rows of a pair when materialized or wire digests diverge. Each case must have
+more than the two discarded warm-up turns, and SQLite DB/WAL/SHM sizes are
+captured while the saver is still open so they represent the online storage
+footprint. Summarize with
+`scripts/benchmark/checkpoint/summarize_production.py` (ratios are
+`delta/full`; it also emits `snapshot_write_spike` and `cache_effect_ms`,
+the decision inputs for the production snapshot-frequency and accessor-cache
+defaults). Harness tests live in `tests/test_bench_checkpoint_production.py`
+and `tests/test_summarize_checkpoint_production.py`; timing thresholds are
+not CI gates. The matrix test pins that every `(repetition, turns)` group
+contains both modes and that their execution order flips between consecutive
+groups, including across repetition boundaries.
+
+Operational limits learned from the first runs (the default matrix is too
+large to run blindly):
+
+- The default `--timeout-seconds 900` is insufficient for delta mode at
+ `snapshot_frequency=1000` once turns reach 500 (measured: delta-500 takes
+ ~1100-1200s; delta-2000 takes ~45min). Pass an explicit
+ `--timeout-seconds` for any large matrix, and treat the turns=2000 corner
+ as practical only at small snapshot frequencies.
+- Full-mode 2000-turn runs produce a ~33GB sqlite DB. Point `TMPDIR` at real
+ disk, not tmpfs (the benchmark uses `tempfile.TemporaryDirectory`, which
+ honors `TMPDIR`), or the run dies mid-case.
+- The history route clamps `limit` to 100 (`le=100` on
+ `ThreadHistoryRequest.limit`), so `--history-limits` values above 100 are
+ measured and reported by their effective (clamped) limit.
+
+Example:
+
+```bash
+cd backend
+PYTHONPATH=. uv run python scripts/benchmark/checkpoint/bench_production.py \
+ --turns 10,100,500,1000,2000 --payload-bytes 128 \
+ --snapshot-frequencies 10,50,100,500,1000 \
+ --repetitions 7 --output /tmp/production-bench.jsonl
+PYTHONPATH=. uv run python scripts/benchmark/checkpoint/summarize_production.py \
+ /tmp/production-bench.jsonl
+```
### Terminal Workbench / TUI (`packages/harness/deerflow/tui/`)
diff --git a/backend/packages/harness/deerflow/agents/lead_agent/agent.py b/backend/packages/harness/deerflow/agents/lead_agent/agent.py
index b20c2e337..7aefb8eb4 100644
--- a/backend/packages/harness/deerflow/agents/lead_agent/agent.py
+++ b/backend/packages/harness/deerflow/agents/lead_agent/agent.py
@@ -46,7 +46,7 @@ from deerflow.agents.middlewares.todo_middleware import TodoMiddleware
from deerflow.agents.middlewares.token_usage_middleware import TokenUsageMiddleware
from deerflow.agents.middlewares.tool_error_handling_middleware import build_lead_runtime_middlewares
from deerflow.agents.middlewares.view_image_middleware import ViewImageMiddleware
-from deerflow.agents.thread_state import get_thread_state_schema, normalize_middleware_state_schemas
+from deerflow.agents.thread_state import freeze_delta_snapshot_frequency, get_thread_state_schema, normalize_middleware_state_schemas
from deerflow.authz.tool_filter import apply_tool_authorization
from deerflow.config.agents_config import load_agent_config, validate_agent_name
from deerflow.config.app_config import AppConfig, get_app_config
@@ -518,6 +518,7 @@ def make_lead_agent(config: RunnableConfig):
runtime_app_config.database.checkpoint_channel_mode,
)
mode = freeze_checkpoint_channel_mode(requested_mode)
+ freeze_delta_snapshot_frequency(runtime_app_config.database.checkpoint_delta_snapshot_frequency)
inject_checkpoint_mode(config, mode)
return _make_lead_agent(config, app_config=runtime_app_config)
diff --git a/backend/packages/harness/deerflow/agents/thread_state.py b/backend/packages/harness/deerflow/agents/thread_state.py
index 5784c267d..10ba5ca70 100644
--- a/backend/packages/harness/deerflow/agents/thread_state.py
+++ b/backend/packages/harness/deerflow/agents/thread_state.py
@@ -352,16 +352,61 @@ def merge_message_writes(state: list[AnyMessage], writes: Sequence[Any]) -> list
return [message for message in messages if message is not None]
-DELTA_MESSAGES_FIELD = Annotated[
- list[AnyMessage],
- DeltaChannel(merge_message_writes, snapshot_frequency=1000),
-]
+DEFAULT_DELTA_SNAPSHOT_FREQUENCY = 1000
+
+_frozen_delta_snapshot_frequency: int | None = None
+
+
+def delta_messages_field(snapshot_frequency: int) -> Any:
+ return Annotated[
+ list[AnyMessage],
+ DeltaChannel(merge_message_writes, snapshot_frequency=snapshot_frequency),
+ ]
+
+
+DELTA_MESSAGES_FIELD = delta_messages_field(DEFAULT_DELTA_SNAPSHOT_FREQUENCY)
class DeltaThreadState(ThreadState):
messages: DELTA_MESSAGES_FIELD
+def freeze_delta_snapshot_frequency(frequency: int) -> int:
+ """Freeze the process-wide delta snapshot frequency (restart-required)."""
+ # Lazy import: deerflow.runtime.__init__ imports checkpoint_state, which
+ # imports this module — a top-level import would be circular.
+ from deerflow.runtime.checkpoint_mode import CheckpointModeReconfigurationError
+
+ global _frozen_delta_snapshot_frequency
+ if frequency <= 0:
+ raise ValueError("snapshot frequency must be positive")
+ if _frozen_delta_snapshot_frequency is None:
+ _frozen_delta_snapshot_frequency = frequency
+ elif _frozen_delta_snapshot_frequency != frequency:
+ raise CheckpointModeReconfigurationError(f"checkpoint delta snapshot frequency is frozen at {_frozen_delta_snapshot_frequency}; refusing to reconfigure to {frequency} — restart the process to change it")
+ return _frozen_delta_snapshot_frequency
+
+
+def resolved_delta_snapshot_frequency() -> int:
+ return _frozen_delta_snapshot_frequency or DEFAULT_DELTA_SNAPSHOT_FREQUENCY
+
+
+def reset_delta_snapshot_frequency_for_tests() -> None:
+ global _frozen_delta_snapshot_frequency
+ _frozen_delta_snapshot_frequency = None
+
+
+@cache
+def _delta_thread_state_for_frequency(snapshot_frequency: int) -> type:
+ if snapshot_frequency == DEFAULT_DELTA_SNAPSHOT_FREQUENCY:
+ # Preserve the canonical class so identity checks (and its public
+ # re-export) keep holding at the default frequency.
+ return DeltaThreadState
+ annotations = get_type_hints(ThreadState, include_extras=True)
+ annotations["messages"] = delta_messages_field(snapshot_frequency)
+ return TypedDict(f"DeltaThreadState_f{snapshot_frequency}", annotations, total=True)
+
+
THREAD_STATE_REDUCER_FIELDS = frozenset(
{
"messages",
@@ -378,17 +423,23 @@ THREAD_STATE_REDUCER_FIELDS = frozenset(
def get_thread_state_schema(mode: CheckpointChannelMode) -> type:
- return DeltaThreadState if mode == "delta" else ThreadState
+ if mode == "delta":
+ return _delta_thread_state_for_frequency(resolved_delta_snapshot_frequency())
+ return ThreadState
-@cache
def adapt_state_schema_for_mode(schema: type, mode: CheckpointChannelMode) -> type:
if mode == "full":
return schema
+ return _adapt_state_schema_for_mode(schema, mode, resolved_delta_snapshot_frequency())
+
+
+@cache
+def _adapt_state_schema_for_mode(schema: type, mode: CheckpointChannelMode, snapshot_frequency: int) -> type:
annotations = get_type_hints(schema, include_extras=True)
- annotations["messages"] = DELTA_MESSAGES_FIELD
+ annotations["messages"] = delta_messages_field(snapshot_frequency)
return TypedDict(
- f"Delta{schema.__module__.replace('.', '_')}_{schema.__name__}",
+ f"Delta{schema.__module__.replace('.', '_')}_{schema.__name__}_f{snapshot_frequency}",
annotations,
total=getattr(schema, "__total__", True),
)
diff --git a/backend/packages/harness/deerflow/client.py b/backend/packages/harness/deerflow/client.py
index 50a28438d..cbdc71b3e 100644
--- a/backend/packages/harness/deerflow/client.py
+++ b/backend/packages/harness/deerflow/client.py
@@ -37,7 +37,7 @@ from langchain_core.runnables import RunnableConfig
from deerflow.agents.lead_agent.agent import build_middlewares
from deerflow.agents.lead_agent.prompt import apply_prompt_template, get_enabled_skills_for_config
-from deerflow.agents.thread_state import get_thread_state_schema, normalize_middleware_state_schemas
+from deerflow.agents.thread_state import freeze_delta_snapshot_frequency, get_thread_state_schema, normalize_middleware_state_schemas
from deerflow.authz.principal import build_principal_from_context
from deerflow.config.agents_config import AGENT_NAME_PATTERN
from deerflow.config.app_config import get_app_config, is_trace_correlation_enabled, reload_app_config
@@ -192,6 +192,7 @@ class DeerFlowClient:
reload_app_config(config_path)
self._app_config = get_app_config()
self._checkpoint_channel_mode = freeze_checkpoint_channel_mode(self._app_config.database.checkpoint_channel_mode)
+ freeze_delta_snapshot_frequency(self._app_config.database.checkpoint_delta_snapshot_frequency)
if agent_name is not None and not AGENT_NAME_PATTERN.match(agent_name):
raise ValueError(f"Invalid agent name '{agent_name}'. Must match pattern: {AGENT_NAME_PATTERN.pattern}")
diff --git a/backend/packages/harness/deerflow/config/database_config.py b/backend/packages/harness/deerflow/config/database_config.py
index cb578fff5..c02f81b77 100644
--- a/backend/packages/harness/deerflow/config/database_config.py
+++ b/backend/packages/harness/deerflow/config/database_config.py
@@ -53,6 +53,16 @@ class DatabaseConfig(BaseModel):
"processes sharing one checkpoint database must use the same value."
),
)
+ checkpoint_delta_snapshot_frequency: int = Field(
+ default=1000,
+ ge=1,
+ description=(
+ "DeltaChannel snapshot frequency used in checkpoint 'delta' mode: "
+ "every N message-channel writes the saver stores a full snapshot "
+ "instead of a delta sentinel. Restart-required, and all processes "
+ "sharing one checkpoint database must use the same value."
+ ),
+ )
sqlite_dir: str = Field(
default=".deer-flow/data",
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
diff --git a/backend/scripts/benchmark/bench_checkpoint_channels.py b/backend/scripts/benchmark/checkpoint/bench_channels.py
similarity index 78%
rename from backend/scripts/benchmark/bench_checkpoint_channels.py
rename to backend/scripts/benchmark/checkpoint/bench_channels.py
index 17acc0d6c..dce2f3802 100644
--- a/backend/scripts/benchmark/bench_checkpoint_channels.py
+++ b/backend/scripts/benchmark/checkpoint/bench_channels.py
@@ -8,12 +8,12 @@ warming the other.
Examples::
- PYTHONPATH=. uv run python scripts/benchmark/bench_checkpoint_channels.py \
+ PYTHONPATH=. uv run python scripts/benchmark/checkpoint/bench_channels.py \
--updates 10,100,500,999,1000,1001,2000 \
--payload-bytes 128,4096 \
--output checkpoint-bench.jsonl
- PYTHONPATH=. uv run python scripts/benchmark/bench_checkpoint_channels.py \
+ PYTHONPATH=. uv run python scripts/benchmark/checkpoint/bench_channels.py \
--backends sqlite --updates 1000 --payload-bytes 128 \
--repetitions 7 --output snapshot-boundary.jsonl
@@ -27,15 +27,11 @@ and memory use.
from __future__ import annotations
import argparse
-import cProfile
import gc
-import hashlib
import importlib.metadata
import json
import os
import platform
-import statistics
-import subprocess
import sys
import tempfile
import time
@@ -55,10 +51,19 @@ from deerflow.agents.thread_state import merge_message_writes
from deerflow.runtime.checkpoint_mode import inject_checkpoint_mode
from deerflow.runtime.checkpoint_state import CheckpointStateAccessor
-try:
- import resource
-except ImportError: # pragma: no cover - Windows only
- resource = None # type: ignore[assignment]
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import checkpoint_bench_common as _common # noqa: E402
+
+_GIT_SHA_ENV = _common.GIT_SHA_ENV
+_parse_positive_int_csv = _common.parse_positive_int_csv
+_parse_choice_csv = _common.parse_choice_csv
+_canonical_messages_digest = _common.canonical_messages_digest
+_percentile = _common.percentile
+_window_median = _common.window_median
+_resolve_git_sha = _common.resolve_git_sha
+_safe_error = _common.safe_error
+_file_size = _common.file_size
+_peak_rss_bytes = _common.peak_rss_bytes
Mode = Literal["full", "delta"]
Backend = Literal["memory", "sqlite"]
@@ -67,7 +72,6 @@ SCHEMA_VERSION = 1
BENCHMARK_VERSION = 1
PRODUCTION_SNAPSHOT_FREQUENCY = 1000
DEFAULT_MAX_ESTIMATED_FULL_BYTES = 1024**3
-_GIT_SHA_ENV = "DEERFLOW_CHECKPOINT_BENCH_GIT_SHA"
_MODES: tuple[Mode, ...] = ("full", "delta")
_BACKENDS: tuple[Backend, ...] = ("memory", "sqlite")
_STORAGE_STAT_FIELDS = (
@@ -117,53 +121,6 @@ class BenchmarkCase:
raise ValueError(f"unsupported scenario: {self.scenario!r}")
-def _parse_positive_int_csv(value: str, *, option: str) -> list[int]:
- if not value or value.startswith(",") or value.endswith(",") or ",," in value:
- raise ValueError(f"{option} must be a comma-separated list of positive integers")
- result: list[int] = []
- seen: set[int] = set()
- duplicates: list[int] = []
- try:
- parsed = [int(part.strip()) for part in value.split(",")]
- except ValueError as exc:
- raise ValueError(f"{option} must be a comma-separated list of positive integers") from exc
- if any(item <= 0 for item in parsed):
- raise ValueError(f"{option} values must be positive integers")
- for item in parsed:
- if item not in seen:
- result.append(item)
- seen.add(item)
- elif item not in duplicates:
- duplicates.append(item)
- if duplicates:
- print(
- f"{option}: ignored duplicate value(s): {', '.join(str(item) for item in duplicates)}; use --repetitions for repeated samples.",
- file=sys.stderr,
- )
- return result
-
-
-def _parse_choice_csv(value: str, *, option: str, choices: tuple[str, ...]) -> list[str]:
- if not value or value.startswith(",") or value.endswith(",") or ",," in value:
- raise ValueError(f"{option} must contain one or more of: {', '.join(choices)}")
- result: list[str] = []
- duplicates: list[str] = []
- for raw in value.split(","):
- item = raw.strip()
- if item not in choices:
- raise ValueError(f"{option} contains unsupported value {item!r}; expected: {', '.join(choices)}")
- if item not in result:
- result.append(item)
- elif item not in duplicates:
- duplicates.append(item)
- if duplicates:
- print(
- f"{option}: ignored duplicate value(s): {', '.join(duplicates)}; use --repetitions for repeated samples.",
- file=sys.stderr,
- )
- return result
-
-
def _expand_cases(
*,
modes: list[str],
@@ -219,47 +176,6 @@ def _message_for_update(index: int, payload_bytes: int) -> BaseMessage:
return AIMessage(id=message_id, content=content)
-def _canonical_messages_digest(messages: list[AnyMessage]) -> str:
- canonical = [
- {
- "id": message.id,
- "type": message.type,
- "content": message.content,
- }
- for message in messages
- ]
- payload = json.dumps(canonical, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
- return hashlib.sha256(payload).hexdigest()
-
-
-def _percentile(values: list[float], percentile: float) -> float:
- if not values:
- return 0.0
- ordered = sorted(values)
- if len(ordered) == 1:
- return ordered[0]
- rank = (len(ordered) - 1) * percentile / 100
- lower = int(rank)
- upper = min(lower + 1, len(ordered) - 1)
- fraction = rank - lower
- return ordered[lower] * (1 - fraction) + ordered[upper] * fraction
-
-
-def _window_median(values: list[float], window: Literal["first", "middle", "last"]) -> float:
- if not values:
- return 0.0
- width = max(1, len(values) // 10)
- if window == "first":
- selected = values[:width]
- elif window == "last":
- selected = values[-width:]
- else:
- center = len(values) // 2
- start = max(0, center - width // 2)
- selected = values[start : start + width]
- return statistics.median(selected)
-
-
def _noop(_state: dict[str, Any]) -> dict[str, Any]:
return {}
@@ -283,20 +199,6 @@ def _config(case: BenchmarkCase) -> dict[str, Any]:
return config
-def _resolve_git_sha() -> str:
- try:
- return subprocess.run(
- ["git", "rev-parse", "HEAD"],
- cwd=Path(__file__).resolve().parents[3],
- check=True,
- capture_output=True,
- text=True,
- timeout=5,
- ).stdout.strip()
- except (OSError, subprocess.SubprocessError):
- return "unknown"
-
-
def _base_row(case: BenchmarkCase) -> dict[str, Any]:
git_sha = os.environ.get(_GIT_SHA_ENV) or _resolve_git_sha()
try:
@@ -324,13 +226,6 @@ def _base_row(case: BenchmarkCase) -> dict[str, Any]:
}
-def _safe_error(error: BaseException | str, *, work_dir: Path | None = None) -> str:
- message = str(error).replace(str(Path.home()), "")
- if work_dir is not None:
- message = message.replace(str(work_dir), "")
- return message[:2000]
-
-
def _collect_storage_stats(collector: Callable[[], dict[str, int]]) -> dict[str, Any]:
"""Keep timing data usable when a saver's diagnostic layout changes."""
try:
@@ -387,20 +282,6 @@ def _sqlite_storage_stats(saver: SqliteSaver, thread_id: str) -> dict[str, int]:
}
-def _file_size(path: Path) -> int:
- try:
- return path.stat().st_size
- except FileNotFoundError:
- return 0
-
-
-def _peak_rss_bytes() -> int | None:
- if resource is None:
- return None
- peak_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
- return int(peak_rss) if sys.platform == "darwin" else int(peak_rss * 1024)
-
-
def _write_and_read(case: BenchmarkCase, saver: Any, messages: list[BaseMessage]) -> tuple[dict[str, Any], list[AnyMessage]]:
graph = _build_graph(case.mode, saver)
accessor = CheckpointStateAccessor.bind(graph, saver, mode=case.mode)
@@ -532,12 +413,7 @@ def _run_case(case: BenchmarkCase, *, work_dir: Path) -> dict[str, Any]:
def _run_profiled_case(case: BenchmarkCase, *, work_dir: Path, profile_path: Path) -> dict[str, Any]:
- profile_path.parent.mkdir(parents=True, exist_ok=True)
- profiler = cProfile.Profile()
- row = profiler.runcall(_run_case, case, work_dir=work_dir)
- row["profiled"] = True
- profiler.dump_stats(profile_path)
- return row
+ return _common.run_profiled(_run_case, case, work_dir=work_dir, profile_path=profile_path)
def _comparison_key(row: dict[str, Any]) -> tuple[Any, ...]:
@@ -583,37 +459,16 @@ def _profile_filename(case: BenchmarkCase) -> str:
def _run_child_case(case: BenchmarkCase, *, timeout_seconds: float, git_sha: str, profile_dir: Path | None = None) -> dict[str, Any]:
- encoded_case = json.dumps(asdict(case), separators=(",", ":"))
- command = [sys.executable, str(Path(__file__).resolve()), "--worker-case", encoded_case]
+ worker_args = ["--worker-case", json.dumps(asdict(case), separators=(",", ":"))]
if profile_dir is not None:
- command.extend(["--worker-profile", str(profile_dir / _profile_filename(case))])
- started = time.perf_counter()
- child_env = os.environ.copy()
- child_env[_GIT_SHA_ENV] = git_sha
- try:
- completed = subprocess.run(
- command,
- check=False,
- capture_output=True,
- text=True,
- timeout=timeout_seconds,
- env=child_env,
- )
- except subprocess.TimeoutExpired:
- return _failure_row(case, f"child process timed out after {timeout_seconds:g} seconds")
- child_process_ms = (time.perf_counter() - started) * 1000
- output_lines = [line for line in completed.stdout.splitlines() if line.strip()]
- if not output_lines:
- return _failure_row(case, f"child process returned {completed.returncode} without a result")
- try:
- row = json.loads(output_lines[-1])
- except json.JSONDecodeError:
- return _failure_row(case, f"child process returned {completed.returncode} with malformed JSON")
- row["child_process_ms"] = child_process_ms
- if completed.returncode != 0 and row.get("success"):
- row["success"] = False
- row["error"] = f"child process exited with status {completed.returncode}"
- return row
+ worker_args.extend(["--worker-profile", str(profile_dir / _profile_filename(case))])
+ return _common.run_child_case(
+ script=Path(__file__).resolve(),
+ worker_args=worker_args,
+ failure_row=lambda error: _failure_row(case, error),
+ timeout_seconds=timeout_seconds,
+ git_sha=git_sha,
+ )
def _build_parser() -> argparse.ArgumentParser:
diff --git a/backend/scripts/benchmark/checkpoint/bench_production.py b/backend/scripts/benchmark/checkpoint/bench_production.py
new file mode 100755
index 000000000..3283e649a
--- /dev/null
+++ b/backend/scripts/benchmark/checkpoint/bench_production.py
@@ -0,0 +1,716 @@
+#!/usr/bin/env python3
+"""Production-shaped full/delta checkpoint benchmark.
+
+Measures user-visible run, state, and history latency on the production path:
+the real lead-agent graph (scripted deterministic model), a real
+AsyncSqliteSaver, and the real Gateway route stack for reads. Every case runs
+in a fresh child process with a fresh database, mirroring the
+restart-required checkpoint mode boundary.
+
+Examples::
+
+ PYTHONPATH=. uv run python scripts/benchmark/checkpoint/bench_production.py \
+ --turns 100,1000 --payload-bytes 128 \
+ --snapshot-frequencies 10,100,1000 \
+ --repetitions 7 --output production-bench.jsonl
+
+ PYTHONPATH=. uv run python scripts/benchmark/checkpoint/summarize_production.py \
+ production-bench.jsonl
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import tempfile
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Literal
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import checkpoint_bench_common as common # noqa: E402
+
+Mode = Literal["full", "delta"]
+
+SCHEMA_VERSION = 1
+BENCHMARK_VERSION = 1
+BENCHMARK_NAME = "checkpoint_production"
+DEFAULT_HISTORY_LIMITS = (10, 50, 200)
+DEFAULT_READ_REPETITIONS = 25
+WARMUP_TURNS = 2
+_MODES: tuple[Mode, ...] = ("full", "delta")
+
+
+@dataclass(frozen=True)
+class ProductionCase:
+ mode: Mode
+ turns: int
+ payload_bytes: int
+ snapshot_frequency: int | None
+ history_limits: tuple[int, ...]
+ read_repetitions: int
+ repetition: int
+ seed: int
+
+ def __post_init__(self) -> None:
+ if self.mode not in _MODES:
+ raise ValueError(f"unsupported mode: {self.mode!r}")
+ if self.turns <= WARMUP_TURNS:
+ raise ValueError(f"turns must exceed the {WARMUP_TURNS}-turn warm-up")
+ if self.payload_bytes <= 0:
+ raise ValueError("payload_bytes must be positive")
+ if self.mode == "full" and self.snapshot_frequency is not None:
+ raise ValueError("snapshot_frequency is a delta-only axis")
+ if self.mode == "delta" and (self.snapshot_frequency is None or self.snapshot_frequency <= 0):
+ raise ValueError("delta mode requires a positive snapshot_frequency")
+ if not self.history_limits or any(limit <= 0 for limit in self.history_limits):
+ raise ValueError("history_limits must be positive")
+ if self.read_repetitions <= 0:
+ raise ValueError("read_repetitions must be positive")
+ if self.repetition < 0:
+ raise ValueError("repetition must be non-negative")
+
+
+def expand_cases(
+ *,
+ modes: list[str],
+ turn_counts: list[int],
+ payload_bytes: list[int],
+ snapshot_frequencies: list[int],
+ repetitions: int,
+ seed: int = 1,
+ history_limits: tuple[int, ...] = DEFAULT_HISTORY_LIMITS,
+ read_repetitions: int = DEFAULT_READ_REPETITIONS,
+) -> list[ProductionCase]:
+ """Build the matrix with alternating mode order to reduce order bias."""
+ cases: list[ProductionCase] = []
+ group_index = 0
+ for repetition in range(repetitions):
+ for payload in payload_bytes:
+ for turns in turn_counts:
+ ordered_modes = list(modes)
+ if group_index % 2 == 1:
+ ordered_modes.reverse()
+ for mode in ordered_modes:
+ frequencies: list[int | None] = snapshot_frequencies if mode == "delta" else [None]
+ for frequency in frequencies:
+ cases.append(
+ ProductionCase(
+ mode=mode, # type: ignore[arg-type]
+ turns=turns,
+ payload_bytes=payload,
+ snapshot_frequency=frequency,
+ history_limits=history_limits,
+ read_repetitions=read_repetitions,
+ repetition=repetition,
+ seed=seed,
+ )
+ )
+ group_index += 1
+ return cases
+
+
+def _base_row(case: ProductionCase) -> dict:
+ import importlib.metadata
+ import os
+ import platform
+
+ git_sha = os.environ.get(common.GIT_SHA_ENV) or common.resolve_git_sha()
+ try:
+ langgraph_version = importlib.metadata.version("langgraph")
+ except importlib.metadata.PackageNotFoundError:
+ langgraph_version = "unknown"
+ return {
+ "schema_version": SCHEMA_VERSION,
+ "benchmark_version": BENCHMARK_VERSION,
+ "benchmark": BENCHMARK_NAME,
+ "success": True,
+ "error": None,
+ "profiled": False,
+ "git_sha": git_sha,
+ "python_version": platform.python_version(),
+ "langgraph_version": langgraph_version,
+ "platform": platform.platform(),
+ "mode": case.mode,
+ "snapshot_frequency": case.snapshot_frequency,
+ "turns": case.turns,
+ "payload_bytes": case.payload_bytes,
+ "history_limits": list(case.history_limits),
+ "read_repetitions": case.read_repetitions,
+ "repetition": case.repetition,
+ "seed": case.seed,
+ }
+
+
+def _failure_row(case: ProductionCase, error: str) -> dict:
+ row = _base_row(case)
+ row["success"] = False
+ row["error"] = common.safe_error(error)
+ return row
+
+
+def _comparison_key(row: dict) -> tuple:
+ # snapshot_frequency is intentionally excluded: every delta frequency
+ # pairs against the same full row, and all frequencies must agree on
+ # the materialized digests.
+ return (row.get("turns"), row.get("payload_bytes"), row.get("repetition"))
+
+
+def validate_cross_mode_rows(rows: list[dict]) -> None:
+ grouped: dict[tuple, list[dict]] = {}
+ for row in rows:
+ grouped.setdefault(_comparison_key(row), []).append(row)
+ for group in grouped.values():
+ successful = [row for row in group if row.get("success")]
+ modes = {row.get("mode") for row in successful}
+ if not {"full", "delta"}.issubset(modes):
+ continue
+ signatures = {(row.get("message_count"), row.get("seed_content_sha256"), row.get("state_wire_sha256"), row.get("history_wire_sha256")) for row in successful}
+ if len(signatures) == 1:
+ continue
+ for row in successful:
+ row["success"] = False
+ row["error"] = "cross-mode materialized state mismatch"
+
+
+def _profile_filename(case: ProductionCase) -> str:
+ return f"production-{case.mode}-turns-{case.turns}-payload-{case.payload_bytes}-freq-{case.snapshot_frequency}-rep-{case.repetition}.prof"
+
+
+def _worker_main(encoded_case: str, *, profile_path: Path | None = None) -> int:
+ try:
+ raw = json.loads(encoded_case)
+ history_limits = raw.get("history_limits")
+ if history_limits is not None:
+ raw["history_limits"] = tuple(history_limits)
+ case = ProductionCase(**raw)
+ except (TypeError, ValueError) as exc:
+ print(json.dumps({"schema_version": SCHEMA_VERSION, "benchmark_version": BENCHMARK_VERSION, "benchmark": BENCHMARK_NAME, "success": False, "error": common.safe_error(exc)}, separators=(",", ":")))
+ return 2
+ with tempfile.TemporaryDirectory(prefix="deerflow-checkpoint-production-") as temp_dir:
+ if profile_path is None:
+ row = _run_case(case, work_dir=Path(temp_dir))
+ else:
+ row = common.run_profiled(_run_case, case, work_dir=Path(temp_dir), profile_path=profile_path)
+ print(json.dumps(row, ensure_ascii=False, separators=(",", ":")))
+ return 0 if row.get("success") else 1
+
+
+def _build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description="Production-shaped full/delta checkpoint benchmark")
+ parser.add_argument("--modes", default="full,delta")
+ parser.add_argument("--turns", default="10,100,500,1000,2000")
+ parser.add_argument("--payload-bytes", default="128")
+ parser.add_argument("--snapshot-frequencies", default="10,50,100,500,1000")
+ parser.add_argument("--history-limits", default=",".join(str(v) for v in DEFAULT_HISTORY_LIMITS))
+ parser.add_argument("--read-repetitions", type=int, default=DEFAULT_READ_REPETITIONS)
+ parser.add_argument("--repetitions", type=int, default=3)
+ parser.add_argument("--seed", type=int, default=1)
+ parser.add_argument("--timeout-seconds", type=float, default=900)
+ parser.add_argument("--profile-dir", type=Path)
+ parser.add_argument("--output", type=Path)
+ parser.add_argument("--worker-case", help=argparse.SUPPRESS)
+ parser.add_argument("--worker-profile", type=Path, help=argparse.SUPPRESS)
+ return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = _build_parser()
+ args = parser.parse_args(argv)
+ if args.worker_case is not None:
+ return _worker_main(args.worker_case, profile_path=args.worker_profile)
+ if args.output is None:
+ parser.error("--output is required")
+ try:
+ modes = common.parse_choice_csv(args.modes, option="--modes", choices=_MODES)
+ turn_counts = common.parse_positive_int_csv(args.turns, option="--turns")
+ payload_bytes = common.parse_positive_int_csv(args.payload_bytes, option="--payload-bytes")
+ frequencies = common.parse_positive_int_csv(args.snapshot_frequencies, option="--snapshot-frequencies")
+ history_limits = tuple(common.parse_positive_int_csv(args.history_limits, option="--history-limits"))
+ if args.repetitions <= 0 or args.read_repetitions <= 0 or args.timeout_seconds <= 0:
+ raise ValueError("repetition and timeout values must be positive")
+ except ValueError as exc:
+ parser.error(str(exc))
+
+ cases = expand_cases(
+ modes=modes,
+ turn_counts=turn_counts,
+ payload_bytes=payload_bytes,
+ snapshot_frequencies=frequencies,
+ repetitions=args.repetitions,
+ seed=args.seed,
+ history_limits=history_limits,
+ read_repetitions=args.read_repetitions,
+ )
+ git_sha = common.resolve_git_sha()
+ rows: list[dict] = []
+ for index, case in enumerate(cases, start=1):
+ print(
+ f"[{index}/{len(cases)}] {case.mode} turns={case.turns} payload={case.payload_bytes} freq={case.snapshot_frequency} repetition={case.repetition}",
+ file=sys.stderr,
+ )
+ worker_args = ["--worker-case", json.dumps(asdict(case), separators=(",", ":"))]
+ if args.profile_dir is not None:
+ worker_args.extend(["--worker-profile", str(args.profile_dir / _profile_filename(case))])
+ rows.append(
+ common.run_child_case(
+ script=Path(__file__).resolve(),
+ worker_args=worker_args,
+ failure_row=lambda error, c=case: _failure_row(c, error),
+ timeout_seconds=args.timeout_seconds,
+ git_sha=git_sha,
+ )
+ )
+ validate_cross_mode_rows(rows)
+
+ args.output.parent.mkdir(parents=True, exist_ok=True)
+ with args.output.open("w", encoding="utf-8") as output_file:
+ for row in rows:
+ output_file.write(json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n")
+ failures = sum(1 for row in rows if not row.get("success"))
+ print(f"Wrote {len(rows)} result row(s) to {args.output}; failures={failures}", file=sys.stderr)
+ return 1 if failures else 0
+
+
+# ---------------------------------------------------------------------------
+# Worker: production-shaped run + read phases (single event loop)
+# ---------------------------------------------------------------------------
+
+import asyncio # noqa: E402
+import hashlib # noqa: E402
+import time # noqa: E402
+from collections import defaultdict # noqa: E402
+from typing import Any # noqa: E402
+from unittest.mock import patch # noqa: E402
+
+import httpx # noqa: E402
+from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel # noqa: E402
+from langchain_core.messages import AIMessage, HumanMessage # noqa: E402
+from langchain_core.outputs import ChatGeneration, ChatResult # noqa: E402
+
+
+class _ScriptedBenchModel(FakeMessagesListChatModel):
+ """Deterministic model: call k returns a fixed-size AIMessage.
+
+ Content and IDs depend only on the call index, so full and delta modes
+ produce byte-identical message streams. Serves the ainvoke path via
+ BaseChatModel's default ``_agenerate`` fallback (``run_in_executor`` over
+ ``_generate``); streaming is not supported.
+ """
+
+ def __init__(self, *, payload_bytes: int) -> None:
+ super().__init__(responses=[])
+ self._bench_payload_bytes = payload_bytes
+ self._bench_call_index = 0
+
+ def bind_tools(self, tools: Any, **kwargs: Any) -> Any:
+ return self
+
+ def _generate(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any) -> ChatResult:
+ index = self._bench_call_index
+ self._bench_call_index += 1
+ message = AIMessage(id=f"bench-ai-{index:08d}", content="x" * self._bench_payload_bytes)
+ return ChatResult(generations=[ChatGeneration(message=message)])
+
+
+class _TimingSaver:
+ """Delegating saver wrapper recording write timing.
+
+ LangGraph issues ``aput``/``aput_writes`` as concurrent asyncio tasks, so
+ per-call latencies overlap (queueing on the single sqlite connection is
+ counted in every concurrent call). Summing them double-counts and can
+ exceed the turn wall clock. ``intervals`` keeps absolute (start, end) per
+ call so the run phase can compute the merged busy time — the honest,
+ wall-bounded write cost. ``timings_ms`` remains the cumulative-per-method
+ ledger used for read-path attribution.
+ """
+
+ def __init__(self, saver: Any) -> None:
+ self._saver = saver
+ self.timings_ms: dict[str, float] = defaultdict(float)
+ self.intervals: list[tuple[str, float, float]] = []
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(self._saver, name)
+
+ async def _timed(self, label: str, fn: Any, *args: Any, **kwargs: Any) -> Any:
+ start = time.perf_counter()
+ try:
+ return await fn(*args, **kwargs)
+ finally:
+ end = time.perf_counter()
+ self.timings_ms[label] += (end - start) * 1000
+ self.intervals.append((label, start * 1000, end * 1000))
+
+ async def aput(self, *args: Any, **kwargs: Any) -> Any:
+ return await self._timed("aput", self._saver.aput, *args, **kwargs)
+
+ async def aput_writes(self, *args: Any, **kwargs: Any) -> Any:
+ return await self._timed("aput_writes", self._saver.aput_writes, *args, **kwargs)
+
+ async def aget_tuple(self, *args: Any, **kwargs: Any) -> Any:
+ return await self._timed("aget_tuple", self._saver.aget_tuple, *args, **kwargs)
+
+ def alist(self, *args: Any, **kwargs: Any) -> Any:
+ # alist is an async generator on real savers; wrap it to time the
+ # full iteration when consumed.
+ agen = self._saver.alist(*args, **kwargs)
+ timing = self.timings_ms
+
+ async def _wrapped() -> Any:
+ start_total = time.perf_counter()
+ try:
+ async for item in agen:
+ yield item
+ finally:
+ timing["alist"] += (time.perf_counter() - start_total) * 1000
+
+ return _wrapped()
+
+
+def _thread_id(case: ProductionCase) -> str:
+ return f"checkpoint-production-{case.seed}-{case.repetition}"
+
+
+async def _run_phase(case: ProductionCase, saver: Any, timing: _TimingSaver, work_dir: Path) -> dict[str, Any]:
+ """Seed the thread through the real lead-agent graph; measure per-turn latency."""
+ from deerflow.agents.lead_agent.agent import make_lead_agent
+ from deerflow.config.app_config import AppConfig, set_app_config
+ from deerflow.runtime.checkpoint_mode import inject_checkpoint_mode
+
+ app_config = AppConfig.model_validate(
+ {
+ # Mirrors tests/test_lead_agent_model_resolution.py::_make_model:
+ # `use` is a required ModelConfig field, so a provider/api_key
+ # style entry does not validate.
+ "models": [
+ {
+ "name": "bench-model",
+ "display_name": "bench-model",
+ "description": None,
+ "use": "langchain_openai:ChatOpenAI",
+ "model": "bench",
+ "supports_thinking": False,
+ "supports_vision": False,
+ }
+ ],
+ "sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
+ # Title and memory middleware would fire background LLM calls and
+ # debounce timers; both pollute per-turn latency, so disable them.
+ "title": {"enabled": False},
+ "memory": {"enabled": False},
+ "database": {
+ "backend": "sqlite",
+ "sqlite_dir": str(work_dir),
+ "checkpoint_channel_mode": case.mode,
+ "checkpoint_delta_snapshot_frequency": case.snapshot_frequency or 1000,
+ },
+ }
+ )
+ set_app_config(app_config)
+
+ model = _ScriptedBenchModel(payload_bytes=case.payload_bytes)
+
+ def _fake_create_chat_model(**kwargs: Any) -> Any:
+ return model
+
+ with patch("deerflow.agents.lead_agent.agent.create_chat_model", _fake_create_chat_model):
+ graph = make_lead_agent({"configurable": {}})
+ # Attach the timed saver post-construction, mirroring the run worker
+ # (deerflow.runs.worker assigns ``agent.checkpointer`` the same way), so
+ # every checkpoint write flows through ``timing``.
+ graph.checkpointer = timing
+
+ config: dict[str, Any] = {"configurable": {"thread_id": _thread_id(case)}}
+ inject_checkpoint_mode(config, case.mode)
+
+ turn_ms: list[float] = []
+ write_ms: list[float] = []
+ write_methods = {"aput", "aput_writes"}
+ for turn in range(case.turns):
+ intervals_before = len(timing.intervals)
+ start = time.perf_counter()
+ await graph.ainvoke(
+ {"messages": [HumanMessage(id=f"bench-h-{turn:08d}", content="y" * case.payload_bytes)]},
+ config,
+ )
+ turn_ms.append((time.perf_counter() - start) * 1000)
+ write_ms.append(_merged_busy_ms(iv for iv in timing.intervals[intervals_before:] if iv[0] in write_methods))
+
+ return {"graph": graph, "config": config, "turn_ms": turn_ms[WARMUP_TURNS:], "write_ms": write_ms[WARMUP_TURNS:]}
+
+
+def _merged_busy_ms(intervals: Any) -> float:
+ """Union of (start, end) write intervals in ms — wall-bounded write cost.
+
+ Concurrent write tasks overlap; merging their intervals yields the time
+ the event loop was actually inside aput/aput_writes, which cannot exceed
+ the turn wall clock the way a naive latency sum does.
+ """
+ merged: list[list[float]] = []
+ for _label, start, end in sorted(intervals, key=lambda iv: iv[1]):
+ if merged and start <= merged[-1][1]:
+ if end > merged[-1][1]:
+ merged[-1][1] = end
+ else:
+ merged.append([start, end])
+ return sum(end - start for start, end in merged)
+
+
+# ``ThreadHistoryRequest.limit`` is validated with ``le=100``: the production
+# route cannot serve a larger page. Configured history limits above the cap
+# are measured at the cap and labelled by the *effective* limit so no metric
+# field ever claims a request the API would reject.
+_HISTORY_ROUTE_MAX_LIMIT = 100
+
+
+def _wire_messages_digest(messages: Any) -> str:
+ """Digest wire-format messages, excluding volatile checkpoint ids/timestamps."""
+ canonical = [[message.get("type"), message.get("id"), message.get("content") if isinstance(message.get("content"), str) else json.dumps(message.get("content"), sort_keys=True)] for message in messages or []]
+ payload = json.dumps(canonical, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+ return hashlib.sha256(payload).hexdigest()
+
+
+def _combined_digest(digests: list[str]) -> str:
+ return hashlib.sha256("".join(digests).encode("utf-8")).hexdigest()
+
+
+def _make_gateway_app(saver: Any, mode: str, store: Any) -> Any:
+ """Minimal authed gateway app, mirroring tests/_router_auth_helpers.py.
+
+ Reimplemented inline: benchmark scripts must not import from tests/.
+ """
+ from unittest.mock import MagicMock
+ from uuid import uuid4
+
+ from fastapi import FastAPI, Request, Response
+ from starlette.middleware.base import BaseHTTPMiddleware
+
+ from app.gateway.auth.models import User
+ from app.gateway.authz import AuthContext, Permissions
+ from app.gateway.routers import threads
+ from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore
+ from deerflow.runtime.user_context import reset_current_user, set_current_user
+
+ permissions = [Permissions.THREADS_READ, Permissions.THREADS_WRITE, Permissions.THREADS_DELETE]
+
+ class _StubAuthMiddleware(BaseHTTPMiddleware):
+ async def dispatch(self, request: Request, call_next: Any) -> Response:
+ user = User(email="bench@example.com", password_hash="x", system_role="user", id=uuid4())
+ request.state.user = user
+ request.state.auth = AuthContext(user=user, permissions=list(permissions))
+ # Mirror the production AuthMiddleware: the user contextvar must be
+ # set or thread-meta lookups hit user_id=AUTO and raise per request.
+ token = set_current_user(user)
+ try:
+ return await call_next(request)
+ finally:
+ reset_current_user(token)
+
+ app = FastAPI()
+ app.add_middleware(_StubAuthMiddleware)
+ app.state.store = store
+ app.state.checkpointer = saver
+ app.state.thread_store = MemoryThreadMetaStore(store)
+ app.state.checkpoint_channel_mode = mode
+ app.state.run_event_store = MagicMock()
+ app.include_router(threads.router)
+ return app
+
+
+async def _read_phase(case: ProductionCase, saver: Any, timing: _TimingSaver) -> dict[str, Any]:
+ """Measure state/history endpoint latency through the real route stack.
+
+ Runs in the same event loop as the run phase (the AsyncSqliteSaver is
+ bound to its creating loop), driving the real threads router over an
+ ASGI transport. ``saver`` must be the RAW saver, not the timing
+ wrapper: ``Pregel.aget_state`` gates delta replay behind
+ ``isinstance(self.checkpointer, BaseCheckpointSaver)``
+ (langgraph/pregel/main.py), so the duck-typed ``_TimingSaver``
+ silently materializes delta channels as empty. The wrapper therefore
+ stays on the run phase only and the row records
+ ``saver_read_attribution: False``; the ``*_saver_ms`` fields remain in
+ the schema but report 0.0.
+ """
+ from langgraph.store.memory import InMemoryStore
+
+ from app.gateway import services as gateway_services
+
+ store = InMemoryStore()
+ app = _make_gateway_app(saver, case.mode, store)
+
+ graph_build_ms: list[float] = []
+ original_resolve = gateway_services.resolve_agent_factory
+ real_factory = original_resolve(None)
+
+ def _timed_factory(config: Any = None) -> Any:
+ start = time.perf_counter()
+ graph = real_factory(config)
+ graph_build_ms.append((time.perf_counter() - start) * 1000)
+ return graph
+
+ # The accessor graph cache is keyed (assistant_id, mode); the factory
+ # identity check compares the callable, so one stable wrapper instance.
+ gateway_services.resolve_agent_factory = lambda assistant_id=None: _timed_factory
+
+ # The accessor graph is compiled lazily inside the first request and
+ # rebuilds the lead agent, so the scripted model must be patched in for
+ # the whole read phase, exactly as in _run_phase.
+ model = _ScriptedBenchModel(payload_bytes=case.payload_bytes)
+
+ def _fake_create_chat_model(**kwargs: Any) -> Any:
+ return model
+
+ thread_id = _thread_id(case)
+ transport = httpx.ASGITransport(app=app)
+ result: dict[str, Any] = {}
+ try:
+ with patch("deerflow.agents.lead_agent.agent.create_chat_model", _fake_create_chat_model):
+ async with httpx.AsyncClient(transport=transport, base_url="http://bench") as client:
+
+ async def _timed_request(label: str, method: str, url: str, **kwargs: Any) -> tuple[Any, float, float]:
+ del label # timing only; labels aid debugging failures
+ saver_before = timing.timings_ms["aget_tuple"] + timing.timings_ms["alist"]
+ start = time.perf_counter()
+ response = await client.request(method, url, **kwargs)
+ elapsed_ms = (time.perf_counter() - start) * 1000
+ # The read phase drives the RAW saver (see docstring), so
+ # the timing wrapper never advances here: saver_ms — and
+ # thus every read-path *_saver_ms field — is structurally 0.0.
+ saver_ms = timing.timings_ms["aget_tuple"] + timing.timings_ms["alist"] - saver_before
+ response.raise_for_status()
+ return response, elapsed_ms, saver_ms
+
+ # --- state endpoint: one cold sample after clearing the accessor cache
+ gateway_services._state_accessor_graph_cache.clear()
+ graph_build_ms.clear()
+ response, state_cold_ms, state_cold_saver_ms = await _timed_request("state-cold", "GET", f"/api/threads/{thread_id}/state")
+ # Cold/warm contract, asserted structurally: the cold request
+ # MUST build the accessor graph exactly once (otherwise the
+ # cold sample did not exercise the cold path), and warm
+ # requests MUST NOT build again (otherwise the cache is not
+ # hit and warm samples silently measure the cold path).
+ if len(graph_build_ms) != 1:
+ raise AssertionError(f"state cold read triggered {len(graph_build_ms)} accessor graph builds; expected exactly 1")
+ cold_graph_build_ms = graph_build_ms[0]
+ state_payload = response.json()
+ state_digest = _wire_messages_digest((state_payload.get("values") or {}).get("messages"))
+
+ state_warm: list[float] = []
+ for _ in range(case.read_repetitions):
+ _response, elapsed_ms, _saver_ms = await _timed_request("state-warm", "GET", f"/api/threads/{thread_id}/state")
+ state_warm.append(elapsed_ms)
+ if len(graph_build_ms) != 1:
+ raise AssertionError("state warm reads triggered an accessor graph rebuild; cache was not hit")
+
+ # --- history endpoint per limit, same cold/warm split
+ history_metrics: dict[str, Any] = {}
+ history_digests: list[str] = []
+ effective_limits = list(dict.fromkeys(min(limit, _HISTORY_ROUTE_MAX_LIMIT) for limit in case.history_limits))
+ for limit_index, limit in enumerate(effective_limits):
+ gateway_services._state_accessor_graph_cache.clear()
+ response, cold_ms, cold_saver_ms = await _timed_request(
+ f"history-cold-{limit}",
+ "POST",
+ f"/api/threads/{thread_id}/history",
+ json={"limit": limit},
+ )
+ expected_builds = 2 + limit_index
+ if len(graph_build_ms) != expected_builds:
+ raise AssertionError(f"history cold read (limit={limit}) triggered {len(graph_build_ms)} total accessor graph builds; expected {expected_builds}")
+ entries = response.json()
+ digest_source: list[Any] = []
+ for entry in entries:
+ digest_source.extend((entry.get("values") or {}).get("messages") or [])
+ history_digests.append(_wire_messages_digest(digest_source))
+ warm: list[float] = []
+ for _ in range(case.read_repetitions):
+ _r, elapsed_ms, _s = await _timed_request(
+ f"history-warm-{limit}",
+ "POST",
+ f"/api/threads/{thread_id}/history",
+ json={"limit": limit},
+ )
+ warm.append(elapsed_ms)
+ if len(graph_build_ms) != expected_builds:
+ raise AssertionError(f"history warm reads (limit={limit}) triggered an accessor graph rebuild; cache was not hit")
+ history_metrics[f"history_limit_{limit}_cold_ms"] = cold_ms
+ history_metrics[f"history_limit_{limit}_cold_saver_ms"] = cold_saver_ms
+ history_metrics[f"history_limit_{limit}_warm_p50_ms"] = common.percentile(warm, 50)
+ history_metrics[f"history_limit_{limit}_warm_p95_ms"] = common.percentile(warm, 95)
+
+ result.update(
+ {
+ "state_cold_ms": state_cold_ms,
+ "state_cold_saver_ms": state_cold_saver_ms,
+ "state_cold_graph_build_ms": cold_graph_build_ms,
+ "state_warm_p50_ms": common.percentile(state_warm, 50),
+ "state_warm_p95_ms": common.percentile(state_warm, 95),
+ "state_wire_sha256": state_digest,
+ "history_wire_sha256": _combined_digest(history_digests),
+ "saver_read_attribution": False,
+ **history_metrics,
+ }
+ )
+ finally:
+ gateway_services.resolve_agent_factory = original_resolve
+ return result
+
+
+def _storage_file_stats(db_path: Path) -> dict[str, Any]:
+ return {
+ "db_bytes": common.file_size(db_path),
+ "wal_bytes": common.file_size(Path(f"{db_path}-wal")),
+ "shm_bytes": common.file_size(Path(f"{db_path}-shm")),
+ }
+
+
+def _run_case(case: ProductionCase, *, work_dir: Path) -> dict:
+ """Run one benchmark case: run phase + read phase in a single event loop."""
+ row = _base_row(case)
+ db_path = work_dir / "production-benchmark.sqlite"
+
+ async def _async_body() -> dict[str, Any]:
+ from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
+
+ async with AsyncSqliteSaver.from_conn_string(str(db_path)) as saver:
+ await saver.setup()
+ timing = _TimingSaver(saver)
+ run_info = await _run_phase(case, saver, timing, work_dir)
+
+ # Seed correctness: materialize through the mode-matched accessor.
+ from deerflow.runtime.checkpoint_state import CheckpointStateAccessor
+
+ accessor = CheckpointStateAccessor.bind(run_info["graph"], saver, mode=case.mode)
+ snapshot = await accessor.aget(run_info["config"])
+ messages = list(snapshot.values.get("messages", []))
+ seed_digest = common.canonical_messages_digest(messages)
+
+ read_metrics = await _read_phase(case, saver, timing)
+ storage_metrics = _storage_file_stats(db_path)
+
+ metrics = {
+ "run_turn_p50_ms": common.percentile(run_info["turn_ms"], 50),
+ "run_turn_p95_ms": common.percentile(run_info["turn_ms"], 95),
+ "checkpoint_write_p50_ms": common.percentile(run_info["write_ms"], 50),
+ "checkpoint_write_p95_ms": common.percentile(run_info["write_ms"], 95),
+ "checkpoint_write_p99_ms": common.percentile(run_info["write_ms"], 99),
+ "message_count": len(messages),
+ "seed_content_sha256": seed_digest,
+ "peak_rss_bytes": common.peak_rss_bytes(),
+ **storage_metrics,
+ **read_metrics,
+ }
+ return metrics
+
+ try:
+ row.update(asyncio.run(_async_body()))
+ except Exception as exc:
+ row["success"] = False
+ row["error"] = common.safe_error(exc, work_dir=work_dir)
+ return row
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/backend/scripts/benchmark/checkpoint/checkpoint_bench_common.py b/backend/scripts/benchmark/checkpoint/checkpoint_bench_common.py
new file mode 100755
index 000000000..4cd92d56e
--- /dev/null
+++ b/backend/scripts/benchmark/checkpoint/checkpoint_bench_common.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""Shared infrastructure for the checkpoint benchmark family.
+
+Pure, stateless helpers plus the child-process controller protocol used by
+both bench_channels.py (storage microbenchmark) and bench_production.py
+(production-shaped benchmark). Scripts in this folder import this module via
+a sibling sys.path insert; it is not a package.
+"""
+
+from __future__ import annotations
+
+import cProfile
+import hashlib
+import json
+import os
+import statistics
+import subprocess
+import sys
+import time
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Literal
+
+from langchain_core.messages import AnyMessage
+
+try:
+ import resource
+except ImportError: # pragma: no cover - Windows only
+ resource = None # type: ignore[assignment]
+
+GIT_SHA_ENV = "DEERFLOW_CHECKPOINT_BENCH_GIT_SHA"
+
+
+def parse_positive_int_csv(value: str, *, option: str) -> list[int]:
+ if not value or value.startswith(",") or value.endswith(",") or ",," in value:
+ raise ValueError(f"{option} must be a comma-separated list of positive integers")
+ result: list[int] = []
+ seen: set[int] = set()
+ duplicates: list[int] = []
+ try:
+ parsed = [int(part.strip()) for part in value.split(",")]
+ except ValueError as exc:
+ raise ValueError(f"{option} must be a comma-separated list of positive integers") from exc
+ if any(item <= 0 for item in parsed):
+ raise ValueError(f"{option} values must be positive integers")
+ for item in parsed:
+ if item not in seen:
+ result.append(item)
+ seen.add(item)
+ elif item not in duplicates:
+ duplicates.append(item)
+ if duplicates:
+ print(
+ f"{option}: ignored duplicate value(s): {', '.join(str(item) for item in duplicates)}; use --repetitions for repeated samples.",
+ file=sys.stderr,
+ )
+ return result
+
+
+def parse_choice_csv(value: str, *, option: str, choices: tuple[str, ...]) -> list[str]:
+ if not value or value.startswith(",") or value.endswith(",") or ",," in value:
+ raise ValueError(f"{option} must contain one or more of: {', '.join(choices)}")
+ result: list[str] = []
+ duplicates: list[str] = []
+ for raw in value.split(","):
+ item = raw.strip()
+ if item not in choices:
+ raise ValueError(f"{option} contains unsupported value {item!r}; expected: {', '.join(choices)}")
+ if item not in result:
+ result.append(item)
+ elif item not in duplicates:
+ duplicates.append(item)
+ if duplicates:
+ print(
+ f"{option}: ignored duplicate value(s): {', '.join(duplicates)}; use --repetitions for repeated samples.",
+ file=sys.stderr,
+ )
+ return result
+
+
+def canonical_messages_digest(messages: list[AnyMessage]) -> str:
+ canonical = [
+ {
+ "id": message.id,
+ "type": message.type,
+ "content": message.content,
+ }
+ for message in messages
+ ]
+ payload = json.dumps(canonical, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
+ return hashlib.sha256(payload).hexdigest()
+
+
+def percentile(values: list[float], percentile: float) -> float:
+ if not values:
+ return 0.0
+ ordered = sorted(values)
+ if len(ordered) == 1:
+ return ordered[0]
+ rank = (len(ordered) - 1) * percentile / 100
+ lower = int(rank)
+ upper = min(lower + 1, len(ordered) - 1)
+ fraction = rank - lower
+ return ordered[lower] * (1 - fraction) + ordered[upper] * fraction
+
+
+def window_median(values: list[float], window: Literal["first", "middle", "last"]) -> float:
+ if not values:
+ return 0.0
+ width = max(1, len(values) // 10)
+ if window == "first":
+ selected = values[:width]
+ elif window == "last":
+ selected = values[-width:]
+ else:
+ center = len(values) // 2
+ start = max(0, center - width // 2)
+ selected = values[start : start + width]
+ return statistics.median(selected)
+
+
+def resolve_git_sha() -> str:
+ try:
+ return subprocess.run(
+ ["git", "rev-parse", "HEAD"],
+ # scripts/benchmark/checkpoint/checkpoint_bench_common.py -> repo root
+ cwd=Path(__file__).resolve().parents[4],
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=5,
+ ).stdout.strip()
+ except (OSError, subprocess.SubprocessError):
+ return "unknown"
+
+
+def safe_error(error: BaseException | str, *, work_dir: Path | None = None) -> str:
+ message = str(error).replace(str(Path.home()), "")
+ if work_dir is not None:
+ message = message.replace(str(work_dir), "")
+ return message[:2000]
+
+
+def file_size(path: Path) -> int:
+ try:
+ return path.stat().st_size
+ except FileNotFoundError:
+ return 0
+
+
+def peak_rss_bytes() -> int | None:
+ if resource is None:
+ return None
+ peak_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+ return int(peak_rss) if sys.platform == "darwin" else int(peak_rss * 1024)
+
+
+def run_profiled(fn: Callable[..., dict[str, Any]], *args: Any, profile_path: Path, **kwargs: Any) -> dict[str, Any]:
+ """Run *fn* under cProfile, mark the row, and dump stats."""
+ profile_path.parent.mkdir(parents=True, exist_ok=True)
+ profiler = cProfile.Profile()
+ row = profiler.runcall(fn, *args, **kwargs)
+ row["profiled"] = True
+ profiler.dump_stats(profile_path)
+ return row
+
+
+def run_child_case(
+ *,
+ script: Path,
+ worker_args: list[str],
+ failure_row: Callable[[str], dict[str, Any]],
+ timeout_seconds: float,
+ git_sha: str,
+) -> dict[str, Any]:
+ """Run one benchmark case in a fresh child process; return its JSONL row.
+
+ The child prints exactly one JSON row on its last stdout line. Any
+ protocol failure is converted into a failure row via *failure_row*.
+ """
+ command = [sys.executable, str(script), *worker_args]
+ started = time.perf_counter()
+ child_env = os.environ.copy()
+ child_env[GIT_SHA_ENV] = git_sha
+ try:
+ completed = subprocess.run(
+ command,
+ check=False,
+ capture_output=True,
+ text=True,
+ timeout=timeout_seconds,
+ env=child_env,
+ )
+ except subprocess.TimeoutExpired:
+ return failure_row(f"child process timed out after {timeout_seconds:g} seconds")
+ child_process_ms = (time.perf_counter() - started) * 1000
+ output_lines = [line for line in completed.stdout.splitlines() if line.strip()]
+ if not output_lines:
+ return failure_row(f"child process returned {completed.returncode} without a result")
+ try:
+ row = json.loads(output_lines[-1])
+ except json.JSONDecodeError:
+ return failure_row(f"child process returned {completed.returncode} with malformed JSON")
+ row["child_process_ms"] = child_process_ms
+ if completed.returncode != 0 and row.get("success"):
+ row["success"] = False
+ row["error"] = f"child process exited with status {completed.returncode}"
+ return row
diff --git a/backend/scripts/benchmark/summarize_checkpoint_channels.py b/backend/scripts/benchmark/checkpoint/summarize_channels.py
similarity index 97%
rename from backend/scripts/benchmark/summarize_checkpoint_channels.py
rename to backend/scripts/benchmark/checkpoint/summarize_channels.py
index 8db2316f6..7fe94ff72 100644
--- a/backend/scripts/benchmark/summarize_checkpoint_channels.py
+++ b/backend/scripts/benchmark/checkpoint/summarize_channels.py
@@ -8,9 +8,9 @@ values below 1 mean delta used less time or storage.
Examples::
- python scripts/benchmark/summarize_checkpoint_channels.py results.jsonl
- python scripts/benchmark/summarize_checkpoint_channels.py results/*.jsonl --json
- python scripts/benchmark/summarize_checkpoint_channels.py results.jsonl \
+ python scripts/benchmark/checkpoint/summarize_channels.py results.jsonl
+ python scripts/benchmark/checkpoint/summarize_channels.py results/*.jsonl --json
+ python scripts/benchmark/checkpoint/summarize_channels.py results.jsonl \
--metrics write_total_ms,cold_read_ms,logical_checkpoint_bytes --csv
"""
diff --git a/backend/scripts/benchmark/checkpoint/summarize_production.py b/backend/scripts/benchmark/checkpoint/summarize_production.py
new file mode 100755
index 000000000..093a503dc
--- /dev/null
+++ b/backend/scripts/benchmark/checkpoint/summarize_production.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""Summarize paired full/delta production-benchmark JSONL results.
+
+Full rows carry snapshot_frequency=None; each delta frequency group pairs
+against the full row at the same (turns, payload_bytes, repetition). Ratios
+are always delta/full. Decision metrics: snapshot_write_spike (delta
+checkpoint_write p99/p50), cache_effect_ms (state cold - warm p50), and
+checkpoint_write_share (checkpoint_write p50 / run_turn p50). Unless
+--metrics is given, history_limit_*_cold_ms / history_limit_*_warm_p50_ms
+fields found in the input rows are appended to the default metric list.
+
+Example::
+
+ python scripts/benchmark/checkpoint/summarize_production.py results.jsonl
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import re
+import statistics
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+DEFAULT_METRICS = [
+ "run_turn_p50_ms",
+ "run_turn_p95_ms",
+ "checkpoint_write_p50_ms",
+ "checkpoint_write_p95_ms",
+ "state_cold_ms",
+ "state_warm_p50_ms",
+ "state_warm_p95_ms",
+ "db_bytes",
+ "peak_rss_bytes",
+]
+
+_GROUP_FIELDS = ("turns", "payload_bytes", "snapshot_frequency")
+_INPUT_INDEX_FIELD = "_benchmark_input_index"
+_HISTORY_METRIC_RE = re.compile(r"history_limit_(\d+)_(?:cold_ms|warm_p50_ms)")
+
+
+def _discover_history_metrics(rows: list[dict[str, Any]]) -> list[str]:
+ """Auto-discover per-limit history metrics present in the input rows."""
+ discovered = {key for row in rows for key in row if _HISTORY_METRIC_RE.fullmatch(key)}
+ return sorted(discovered, key=lambda key: (int(_HISTORY_METRIC_RE.fullmatch(key).group(1)), key)) # type: ignore[union-attr]
+
+
+def _load_jsonl(paths: list[Path]) -> list[dict[str, Any]]:
+ rows: list[dict[str, Any]] = []
+ errors: list[str] = []
+ for input_index, path in enumerate(paths):
+ with path.open("r", encoding="utf-8") as input_file:
+ for line_number, raw_line in enumerate(input_file, start=1):
+ line = raw_line.strip()
+ if not line or line.startswith("#"):
+ continue
+ try:
+ row = json.loads(line)
+ except json.JSONDecodeError as exc:
+ errors.append(f"{path}:{line_number}: {exc.msg}")
+ continue
+ if not isinstance(row, dict):
+ errors.append(f"{path}:{line_number}: expected a JSON object")
+ continue
+ row[_INPUT_INDEX_FIELD] = input_index
+ rows.append(row)
+ if errors:
+ raise ValueError("Malformed JSONL row(s):\n" + "\n".join(errors))
+ return rows
+
+
+def _numeric(row: dict[str, Any], metric: str) -> float | None:
+ value = row.get(metric)
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ return None
+ return float(value)
+
+
+def _medians(rows: list[dict[str, Any]], metric: str) -> float | None:
+ values = [v for row in rows if (v := _numeric(row, metric)) is not None]
+ return statistics.median(values) if values else None
+
+
+def _summarize(rows: list[dict[str, Any]], *, metrics: list[str]) -> list[dict[str, Any]]:
+ # Full rows are frequency-less; index them for broadcast pairing.
+ full_rows: dict[tuple, dict[str, Any]] = {}
+ delta_groups: dict[tuple, dict[tuple, dict[str, Any]]] = defaultdict(dict)
+ failed_rows: list[dict[str, Any]] = []
+ for row in rows:
+ if row.get("profiled"):
+ continue
+ if not row.get("success"):
+ failed_rows.append(row)
+ continue
+ rep_key = (row.get(_INPUT_INDEX_FIELD), row.get("repetition"), row.get("turns"), row.get("payload_bytes"))
+ if row.get("mode") == "full":
+ full_rows[rep_key] = row
+ elif row.get("mode") == "delta":
+ group_key = tuple(row.get(field) for field in _GROUP_FIELDS)
+ delta_groups[group_key][rep_key] = row
+
+ summaries: list[dict[str, Any]] = []
+ for group_key in sorted(delta_groups, key=str):
+ deltas = delta_groups[group_key]
+ pairs = [(full_rows[k], d) for k, d in deltas.items() if k in full_rows]
+ if not pairs:
+ continue
+ summary: dict[str, Any] = {field: group_key[index] for index, field in enumerate(_GROUP_FIELDS)}
+ summary["paired_repetitions"] = len(pairs)
+ # Per-group failures: a failed delta row counts toward the group whose
+ # (turns, payload_bytes, snapshot_frequency) it matches; a failed full
+ # row (frequency-less) counts toward every group at the same
+ # (turns, payload_bytes).
+ summary["failed_rows"] = sum(1 for row in failed_rows if row.get("turns") == group_key[0] and row.get("payload_bytes") == group_key[1] and (row.get("snapshot_frequency") is None or row.get("snapshot_frequency") == group_key[2]))
+ full_pair_rows = [pair[0] for pair in pairs]
+ delta_pair_rows = [pair[1] for pair in pairs]
+ for metric in metrics:
+ full_median = _medians(full_pair_rows, metric)
+ delta_median = _medians(delta_pair_rows, metric)
+ if full_median is None or delta_median is None:
+ continue
+ summary[f"full_{metric}"] = round(full_median, 6)
+ summary[f"delta_{metric}"] = round(delta_median, 6)
+ summary[f"ratio_{metric}"] = round(delta_median / full_median, 6) if full_median != 0 else None
+ for label, group_rows in (("full", full_pair_rows), ("delta", delta_pair_rows)):
+ p50 = _medians(group_rows, "checkpoint_write_p50_ms")
+ p99 = _medians(group_rows, "checkpoint_write_p99_ms")
+ run_p50 = _medians(group_rows, "run_turn_p50_ms")
+ cold = _medians(group_rows, "state_cold_ms")
+ warm = _medians(group_rows, "state_warm_p50_ms")
+ if p50 and p99 is not None:
+ summary[f"{label}_snapshot_write_spike"] = round(p99 / p50, 6)
+ if p50 is not None and run_p50:
+ summary[f"{label}_checkpoint_write_share"] = round(p50 / run_p50, 6)
+ if cold is not None and warm is not None:
+ summary[f"{label}_cache_effect_ms"] = round(cold - warm, 6)
+ summaries.append(summary)
+ return summaries
+
+
+def _columns(rows: list[dict[str, Any]]) -> list[str]:
+ preferred = [*_GROUP_FIELDS, "paired_repetitions", "failed_rows"]
+ extras = sorted({key for row in rows for key in row if key not in preferred})
+ return [field for field in preferred if any(field in row for row in rows)] + extras
+
+
+def _print_table(rows: list[dict[str, Any]]) -> None:
+ if not rows:
+ print("(no comparable full/delta pairs)")
+ return
+ columns = _columns(rows)
+ widths = {column: len(column) for column in columns}
+ for row in rows:
+ for column in columns:
+ widths[column] = max(widths[column], len(str(row.get(column, ""))))
+ print(" ".join(column.rjust(widths[column]) for column in columns))
+ for row in rows:
+ print(" ".join(str(row.get(column, "")).rjust(widths[column]) for column in columns))
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description="Summarize production-shaped checkpoint benchmark results")
+ parser.add_argument("inputs", nargs="+", type=Path)
+ parser.add_argument("--metrics", default=None, help="comma-separated metric fields; defaults to the built-in list plus auto-discovered history_limit_* metrics")
+ output = parser.add_mutually_exclusive_group()
+ output.add_argument("--json", action="store_true")
+ output.add_argument("--csv", action="store_true")
+ args = parser.parse_args(argv)
+
+ try:
+ rows = _load_jsonl(args.inputs)
+ except (OSError, ValueError) as exc:
+ print(str(exc), file=sys.stderr)
+ return 1
+ if args.metrics is None:
+ metrics = [*DEFAULT_METRICS, *_discover_history_metrics(rows)]
+ else:
+ metrics = [metric.strip() for metric in args.metrics.split(",") if metric.strip()]
+ if not metrics:
+ parser.error("--metrics must contain at least one field")
+ summaries = _summarize(rows, metrics=metrics)
+ if args.json:
+ json.dump(summaries, sys.stdout, ensure_ascii=False, indent=2)
+ print()
+ elif args.csv:
+ writer = csv.DictWriter(sys.stdout, fieldnames=_columns(summaries), extrasaction="ignore")
+ writer.writeheader()
+ writer.writerows(summaries)
+ else:
+ _print_table(summaries)
+ return 0 if summaries else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/backend/scripts/benchmark/bench_sandbox_provider.py b/backend/scripts/benchmark/sandbox/bench_provider.py
similarity index 99%
rename from backend/scripts/benchmark/bench_sandbox_provider.py
rename to backend/scripts/benchmark/sandbox/bench_provider.py
index a38ac8c32..9feb23123 100755
--- a/backend/scripts/benchmark/bench_sandbox_provider.py
+++ b/backend/scripts/benchmark/sandbox/bench_provider.py
@@ -6,7 +6,7 @@ workloads, and concurrency levels. Outputs JSONL for aggregation.
Usage::
- python scripts/benchmark/bench_sandbox_provider.py \\
+ python scripts/benchmark/sandbox/bench_provider.py \\
--provider boxlite \\
--scenario warm_same_thread \\
--workload noop \\
@@ -14,7 +14,7 @@ Usage::
--concurrency 4 \\
--output results.jsonl
- python scripts/benchmark/bench_sandbox_provider.py \\
+ python scripts/benchmark/sandbox/bench_provider.py \\
--provider boxlite \\
--scenario cold_unique_thread \\
--no-warmpool \\
diff --git a/backend/scripts/benchmark/summarize_bench.py b/backend/scripts/benchmark/sandbox/summarize.py
similarity index 95%
rename from backend/scripts/benchmark/summarize_bench.py
rename to backend/scripts/benchmark/sandbox/summarize.py
index 0dfe7b879..7099172af 100755
--- a/backend/scripts/benchmark/summarize_bench.py
+++ b/backend/scripts/benchmark/sandbox/summarize.py
@@ -3,9 +3,9 @@
Usage::
- python scripts/benchmark/summarize_bench.py results.jsonl
- python scripts/benchmark/summarize_bench.py results/*.jsonl --group provider,scenario,workload
- python scripts/benchmark/summarize_bench.py results.jsonl --csv > summary.csv
+ python scripts/benchmark/sandbox/summarize.py results.jsonl
+ python scripts/benchmark/sandbox/summarize.py results/*.jsonl --group provider,scenario,workload
+ python scripts/benchmark/sandbox/summarize.py results.jsonl --csv > summary.csv
"""
from __future__ import annotations
@@ -148,7 +148,7 @@ def _print_table(rows: list[dict[str, Any]], fmt: str = "plain") -> None:
def main(argv: list[str] | None = None) -> int:
p = argparse.ArgumentParser(description="Aggregate JSONL benchmark results")
- p.add_argument("inputs", nargs="+", help="JSONL file(s) from bench_sandbox_provider.py")
+ p.add_argument("inputs", nargs="+", help="JSONL file(s) from sandbox/bench_provider.py")
p.add_argument(
"--group",
default="provider,scenario,workload,concurrency",
diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
index 286f03d9c..0def5b958 100644
--- a/backend/tests/conftest.py
+++ b/backend/tests/conftest.py
@@ -109,6 +109,20 @@ def _reset_frozen_checkpoint_channel_mode(monkeypatch):
yield
+@pytest.fixture(autouse=True)
+def _reset_frozen_delta_snapshot_frequency(monkeypatch):
+ """Reset the process-global frozen delta snapshot frequency between tests.
+
+ Same restart-required semantics as the checkpoint channel mode freeze
+ above: ``make_lead_agent`` freezes it from the app config, and the suite
+ builds many agents with different configs in one process.
+ """
+ from deerflow.agents import thread_state
+
+ monkeypatch.setattr(thread_state, "_frozen_delta_snapshot_frequency", None)
+ yield
+
+
@pytest.fixture(autouse=True)
def _restore_title_config_singleton():
"""Reset ``_title_config`` to its pristine default after every test.
diff --git a/backend/tests/test_bench_checkpoint_channels.py b/backend/tests/test_bench_checkpoint_channels.py
index 52f6a2254..d4a1397c5 100644
--- a/backend/tests/test_bench_checkpoint_channels.py
+++ b/backend/tests/test_bench_checkpoint_channels.py
@@ -9,7 +9,7 @@ import pytest
def _load_module():
- path = Path(__file__).resolve().parents[1] / "scripts/benchmark/bench_checkpoint_channels.py"
+ path = Path(__file__).resolve().parents[1] / "scripts/benchmark/checkpoint/bench_channels.py"
spec = importlib.util.spec_from_file_location("bench_checkpoint_channels", path)
assert spec is not None
module = importlib.util.module_from_spec(spec)
diff --git a/backend/tests/test_bench_checkpoint_production.py b/backend/tests/test_bench_checkpoint_production.py
new file mode 100644
index 000000000..2f8330dbe
--- /dev/null
+++ b/backend/tests/test_bench_checkpoint_production.py
@@ -0,0 +1,237 @@
+"""Harness tests for the production-shaped checkpoint benchmark."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+import pytest
+
+
+def _load_module():
+ path = Path(__file__).resolve().parents[1] / "scripts/benchmark/checkpoint/bench_production.py"
+ spec = importlib.util.spec_from_file_location("bench_production", path)
+ assert spec is not None
+ module = importlib.util.module_from_spec(spec)
+ assert spec.loader is not None
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+bench = _load_module()
+
+
+def test_expand_cases_sweeps_frequency_for_delta_only() -> None:
+ cases = bench.expand_cases(
+ modes=["full", "delta"],
+ turn_counts=[10],
+ payload_bytes=[128],
+ snapshot_frequencies=[10, 50],
+ repetitions=1,
+ )
+ delta = [c for c in cases if c.mode == "delta"]
+ full = [c for c in cases if c.mode == "full"]
+ assert sorted(c.snapshot_frequency for c in delta) == [10, 50]
+ assert [c.snapshot_frequency for c in full] == [None]
+ assert all(c.history_limits == (10, 50, 200) for c in cases)
+
+
+def test_expand_cases_alternates_mode_order() -> None:
+ cases = bench.expand_cases(
+ modes=["full", "delta"],
+ turn_counts=[10, 100],
+ payload_bytes=[128],
+ snapshot_frequencies=[10],
+ repetitions=2,
+ seed=1,
+ )
+ grouped_modes: dict[tuple[int, int], list[str]] = {}
+ for case in cases:
+ grouped_modes.setdefault((case.repetition, case.turns), []).append(case.mode)
+
+ mode_orders = list(grouped_modes.values())
+ assert all(set(order) == {"full", "delta"} for order in mode_orders)
+ assert all(current == list(reversed(previous)) for previous, current in zip(mode_orders, mode_orders[1:], strict=False))
+
+
+def test_case_rejects_nonpositive_turns() -> None:
+ with pytest.raises(ValueError, match="turns"):
+ bench.ProductionCase(
+ mode="full",
+ turns=0,
+ payload_bytes=128,
+ snapshot_frequency=None,
+ history_limits=(10,),
+ read_repetitions=5,
+ repetition=0,
+ seed=1,
+ )
+
+
+@pytest.mark.parametrize("turns", [1, 2])
+def test_case_rejects_turns_consumed_by_warmup(turns: int) -> None:
+ with pytest.raises(ValueError, match="warm-up"):
+ bench.ProductionCase(
+ mode="full",
+ turns=turns,
+ payload_bytes=128,
+ snapshot_frequency=None,
+ history_limits=(10,),
+ read_repetitions=5,
+ repetition=0,
+ seed=1,
+ )
+
+
+def test_case_rejects_frequency_on_full_mode() -> None:
+ with pytest.raises(ValueError, match="snapshot_frequency"):
+ bench.ProductionCase(
+ mode="full",
+ turns=10,
+ payload_bytes=128,
+ snapshot_frequency=10,
+ history_limits=(10,),
+ read_repetitions=5,
+ repetition=0,
+ seed=1,
+ )
+
+
+def test_cross_mode_digest_gate_fails_both_rows_on_mismatch() -> None:
+ base = {"turns": 10, "payload_bytes": 128, "repetition": 0, "success": True, "seed_content_sha256": "a", "state_wire_sha256": "b", "history_wire_sha256": "c"}
+ full = {**base, "mode": "full", "snapshot_frequency": None}
+ delta = {**base, "mode": "delta", "snapshot_frequency": 10}
+ rows = [dict(full), dict(delta)]
+ bench.validate_cross_mode_rows(rows)
+ assert all(r["success"] for r in rows)
+
+ delta_bad = {**delta, "seed_content_sha256": "DIFFERENT"}
+ rows = [dict(full), delta_bad]
+ bench.validate_cross_mode_rows(rows)
+ assert not any(r["success"] for r in rows)
+ assert all(r["error"] == "cross-mode materialized state mismatch" for r in rows)
+
+
+def test_cross_mode_gate_tolerates_missing_pair() -> None:
+ row = {"mode": "full", "snapshot_frequency": None, "turns": 10, "payload_bytes": 128, "repetition": 0, "success": True, "seed_content_sha256": "a", "state_wire_sha256": "b", "history_wire_sha256": "c"}
+ rows = [dict(row)]
+ bench.validate_cross_mode_rows(rows)
+ assert rows[0]["success"] is True
+
+
+def test_scripted_model_is_deterministic_across_instances() -> None:
+ from langchain_core.messages import HumanMessage
+
+ first = bench._ScriptedBenchModel(payload_bytes=16)
+ second = bench._ScriptedBenchModel(payload_bytes=16)
+ for _ in range(3):
+ a = first.invoke([HumanMessage(content="hi", id="h0")])
+ b = second.invoke([HumanMessage(content="hi", id="h0")])
+ assert a.id == b.id and a.content == b.content and len(a.content) == 16
+
+
+def test_timing_saver_records_cumulative_ms() -> None:
+ import asyncio
+
+ from langgraph.checkpoint.base import empty_checkpoint
+ from langgraph.checkpoint.memory import InMemorySaver
+
+ timing = bench._TimingSaver(InMemorySaver())
+ config = {"configurable": {"thread_id": "t", "checkpoint_ns": "", "checkpoint_id": "c1"}}
+
+ async def go() -> None:
+ await timing.aput(config, empty_checkpoint(), {"source": "input", "step": 0, "writes": {}}, {})
+ await timing.aput_writes(config, [("ch", "v")], "task")
+
+ asyncio.run(go())
+ assert timing.timings_ms["aput"] >= 0
+ assert timing.timings_ms["aput_writes"] >= 0
+ assert timing._saver is not None
+
+
+def _reset_checkpoint_freezes(monkeypatch: pytest.MonkeyPatch) -> None:
+ from deerflow.agents import thread_state
+ from deerflow.config.app_config import reset_app_config
+ from deerflow.runtime import checkpoint_mode
+
+ reset_app_config()
+ monkeypatch.setattr(checkpoint_mode, "_frozen_checkpoint_channel_mode", None)
+ monkeypatch.setattr(thread_state, "_frozen_delta_snapshot_frequency", None)
+
+
+@pytest.mark.parametrize("mode,frequency", [("full", None), ("delta", 10)])
+def test_run_case_succeeds_end_to_end_small(tmp_path, monkeypatch, mode, frequency) -> None:
+ _reset_checkpoint_freezes(monkeypatch)
+ from app.gateway import services as gateway_services
+ from deerflow.config.app_config import reset_app_config
+
+ gateway_services._state_accessor_graph_cache.clear()
+ case = bench.ProductionCase(
+ mode=mode,
+ turns=3,
+ payload_bytes=32,
+ snapshot_frequency=frequency,
+ history_limits=(2,),
+ read_repetitions=2,
+ repetition=0,
+ seed=1,
+ )
+ row = bench._run_case(case, work_dir=tmp_path)
+ assert row["success"], row.get("error")
+ assert row["message_count"] > 0
+ assert row["seed_content_sha256"]
+ assert row["state_warm_p50_ms"] >= 0
+ # Wire digests must reflect real messages: a duck-typed checkpointer on
+ # app.state silently materializes delta channels as empty (Pregel gates
+ # replay behind isinstance(checkpointer, BaseCheckpointSaver)), which
+ # otherwise passes every assertion above vacuously.
+ empty_wire = bench._wire_messages_digest([])
+ assert row["state_wire_sha256"] != empty_wire
+ assert row["history_wire_sha256"] != bench._combined_digest([empty_wire])
+ assert row["wal_bytes"] > 0
+ assert row["shm_bytes"] > 0
+ # Write cost is merged busy time over concurrent write tasks: it must not
+ # exceed the turn wall clock the way a naive latency sum can.
+ assert row["checkpoint_write_p50_ms"] <= row["run_turn_p50_ms"]
+ gateway_services._state_accessor_graph_cache.clear()
+ reset_app_config()
+
+
+def test_run_case_fails_when_warm_cache_is_not_hit(tmp_path, monkeypatch) -> None:
+ """The cold/warm contract is structural: if the accessor cache never holds,
+ warm samples silently measure the cold path — the case row must fail."""
+ _reset_checkpoint_freezes(monkeypatch)
+ from app.gateway import services as gateway_services
+ from deerflow.config.app_config import reset_app_config
+
+ gateway_services._state_accessor_graph_cache.clear()
+ # Bypass the cache entirely: every accessor resolution rebuilds the graph,
+ # so warm reads trip the contract assertion.
+ monkeypatch.setattr(
+ gateway_services,
+ "_state_accessor_graph",
+ lambda agent_factory, assistant_id, mode, config: agent_factory(config=config),
+ )
+ case = bench.ProductionCase(
+ mode="full",
+ turns=3,
+ payload_bytes=32,
+ snapshot_frequency=None,
+ history_limits=(2,),
+ read_repetitions=2,
+ repetition=0,
+ seed=1,
+ )
+ row = bench._run_case(case, work_dir=tmp_path)
+ assert not row["success"]
+ assert "cache was not hit" in row["error"]
+ gateway_services._state_accessor_graph_cache.clear()
+ reset_app_config()
+
+
+def test_merged_busy_ms_unions_overlapping_intervals() -> None:
+ intervals = [("aput", 0.0, 10.0), ("aput_writes", 5.0, 12.0), ("aput", 20.0, 25.0)]
+ assert bench._merged_busy_ms(iter(intervals)) == 17.0
+ assert bench._merged_busy_ms(iter([])) == 0.0
diff --git a/backend/tests/test_bench_sandbox_provider.py b/backend/tests/test_bench_sandbox_provider.py
index 1217c3be2..133e81ab8 100644
--- a/backend/tests/test_bench_sandbox_provider.py
+++ b/backend/tests/test_bench_sandbox_provider.py
@@ -17,8 +17,8 @@ def _load_module(name: str, relative: str):
return module
-bench = _load_module("bench_sandbox_provider", "scripts/benchmark/bench_sandbox_provider.py")
-summarize = _load_module("summarize_bench", "scripts/benchmark/summarize_bench.py")
+bench = _load_module("bench_provider", "scripts/benchmark/sandbox/bench_provider.py")
+summarize = _load_module("summarize", "scripts/benchmark/sandbox/summarize.py")
class _FakeProvider:
diff --git a/backend/tests/test_checkpoint_mode.py b/backend/tests/test_checkpoint_mode.py
index 66923263d..001d93296 100644
--- a/backend/tests/test_checkpoint_mode.py
+++ b/backend/tests/test_checkpoint_mode.py
@@ -249,6 +249,31 @@ def test_direct_langgraph_request_cannot_select_delta_in_full_process(
assert CHECKPOINT_MODE_METADATA_KEY not in config["metadata"]
+def test_make_lead_agent_freezes_delta_snapshot_frequency_from_app_config(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from deerflow.agents import thread_state
+ from deerflow.agents.lead_agent import agent as lead_agent
+ from deerflow.config.app_config import AppConfig
+
+ app_config = AppConfig.model_validate(
+ {
+ "sandbox": {"use": "deerflow.sandbox.local.provider:LocalSandboxProvider"},
+ "database": {"checkpoint_delta_snapshot_frequency": 7},
+ }
+ )
+ monkeypatch.setattr(lead_agent, "get_app_config", lambda: app_config)
+ monkeypatch.setattr(
+ lead_agent,
+ "_make_lead_agent",
+ lambda config, *, app_config: object(),
+ )
+
+ lead_agent.make_lead_agent({"configurable": {}})
+
+ assert thread_state._frozen_delta_snapshot_frequency == 7
+
+
def test_gateway_runtime_app_config_can_supply_its_frozen_internal_mode(
monkeypatch: pytest.MonkeyPatch,
) -> None:
diff --git a/backend/tests/test_checkpointer.py b/backend/tests/test_checkpointer.py
index 2befb2ad6..dfbad0c6f 100644
--- a/backend/tests/test_checkpointer.py
+++ b/backend/tests/test_checkpointer.py
@@ -1015,6 +1015,7 @@ class TestClientCheckpointerFallback:
model_mock = MagicMock()
config_mock = MagicMock()
config_mock.models = [model_mock]
+ config_mock.database.checkpoint_delta_snapshot_frequency = 1000
config_mock.get_model_config.return_value = MagicMock(supports_vision=False)
config_mock.checkpointer = None
@@ -1054,6 +1055,7 @@ class TestClientCheckpointerFallback:
model_mock = MagicMock()
config_mock = MagicMock()
config_mock.models = [model_mock]
+ config_mock.database.checkpoint_delta_snapshot_frequency = 1000
config_mock.get_model_config.return_value = MagicMock(supports_vision=False)
config_mock.checkpointer = None
diff --git a/backend/tests/test_client.py b/backend/tests/test_client.py
index ea5718bad..2eb0ecfad 100644
--- a/backend/tests/test_client.py
+++ b/backend/tests/test_client.py
@@ -52,6 +52,7 @@ def mock_app_config():
config.skills.container_path = "/mnt/skills"
config.tool_search.enabled = False
config.database.checkpoint_channel_mode = "full"
+ config.database.checkpoint_delta_snapshot_frequency = 1000
config.authorization = AuthorizationConfig(enabled=False)
return config
@@ -145,6 +146,23 @@ class TestClientInit:
):
DeerFlowClient()
+ def test_delta_snapshot_frequency_is_frozen_from_app_config(self, mock_app_config):
+ from typing import get_type_hints
+
+ from langgraph.channels import DeltaChannel
+
+ from deerflow.agents import thread_state
+
+ mock_app_config.database.checkpoint_channel_mode = "delta"
+ mock_app_config.database.checkpoint_delta_snapshot_frequency = 7
+ with patch("deerflow.client.get_app_config", return_value=mock_app_config):
+ DeerFlowClient()
+
+ schema = thread_state.get_thread_state_schema("delta")
+ hint = get_type_hints(schema, include_extras=True)["messages"]
+ channel = next(item for item in hint.__metadata__ if isinstance(item, DeltaChannel))
+ assert channel.snapshot_frequency == 7
+
# ---------------------------------------------------------------------------
# list_models / list_skills / get_memory
diff --git a/backend/tests/test_delta_channel_state.py b/backend/tests/test_delta_channel_state.py
index fdcbfa6e1..7e41ee109 100644
--- a/backend/tests/test_delta_channel_state.py
+++ b/backend/tests/test_delta_channel_state.py
@@ -13,6 +13,7 @@ from langgraph.checkpoint.memory import InMemorySaver
from langgraph.graph import StateGraph
from langgraph.graph.message import REMOVE_ALL_MESSAGES, add_messages
+from deerflow.agents import thread_state
from deerflow.agents.thread_state import (
DeltaThreadState,
ThreadState,
@@ -21,6 +22,7 @@ from deerflow.agents.thread_state import (
merge_message_writes,
normalize_middleware_state_schemas,
)
+from deerflow.runtime.checkpoint_mode import CheckpointModeReconfigurationError
def _fold(state: list, writes: list) -> list:
@@ -287,6 +289,20 @@ def test_delta_adaptation_replaces_agent_state_message_reducer() -> None:
assert any(isinstance(item, DeltaChannel) for item in hint.__metadata__)
+def test_delta_adaptation_cache_varies_by_resolved_snapshot_frequency(monkeypatch) -> None:
+ monkeypatch.setattr(thread_state, "_frozen_delta_snapshot_frequency", None)
+ thread_state._adapt_state_schema_for_mode.cache_clear()
+ default_adapted = adapt_state_schema_for_mode(AgentState, "delta")
+
+ thread_state.freeze_delta_snapshot_frequency(7)
+ configured_adapted = adapt_state_schema_for_mode(AgentState, "delta")
+
+ assert configured_adapted is not default_adapted
+ hint = get_type_hints(configured_adapted, include_extras=True)["messages"]
+ channel = next(item for item in hint.__metadata__ if isinstance(item, DeltaChannel))
+ assert channel.snapshot_frequency == 7
+
+
def test_agents_package_exports_delta_thread_state() -> None:
from deerflow.agents import DeltaThreadState as ExportedDeltaThreadState
@@ -369,3 +385,31 @@ def test_production_message_forms_keep_assigned_ids_across_delta_replay(write: o
assert first[0].id is not None
assert second[0].id == first[0].id
+
+
+def test_delta_snapshot_frequency_defaults_to_1000(monkeypatch) -> None:
+ monkeypatch.setattr(thread_state, "_frozen_delta_snapshot_frequency", None)
+ assert thread_state.resolved_delta_snapshot_frequency() == 1000
+
+
+def test_freeze_delta_snapshot_frequency_is_process_frozen(monkeypatch) -> None:
+ monkeypatch.setattr(thread_state, "_frozen_delta_snapshot_frequency", None)
+ assert thread_state.freeze_delta_snapshot_frequency(50) == 50
+ assert thread_state.freeze_delta_snapshot_frequency(50) == 50
+ with pytest.raises(CheckpointModeReconfigurationError):
+ thread_state.freeze_delta_snapshot_frequency(10)
+
+
+def test_get_thread_state_schema_honors_frozen_frequency(monkeypatch) -> None:
+ monkeypatch.setattr(thread_state, "_frozen_delta_snapshot_frequency", None)
+ thread_state.freeze_delta_snapshot_frequency(7)
+ hint = get_type_hints(thread_state.get_thread_state_schema("delta"), include_extras=True)["messages"]
+ channel = next(item for item in hint.__metadata__ if isinstance(item, DeltaChannel))
+ assert channel.snapshot_frequency == 7
+
+
+def test_database_config_default_snapshot_frequency() -> None:
+ from deerflow.config.database_config import DatabaseConfig
+
+ assert DatabaseConfig().checkpoint_delta_snapshot_frequency == 1000
+ assert DatabaseConfig(checkpoint_delta_snapshot_frequency=25).checkpoint_delta_snapshot_frequency == 25
diff --git a/backend/tests/test_lead_agent_skills.py b/backend/tests/test_lead_agent_skills.py
index 7faec55c1..f40e614f0 100644
--- a/backend/tests/test_lead_agent_skills.py
+++ b/backend/tests/test_lead_agent_skills.py
@@ -273,6 +273,9 @@ def test_make_lead_agent_empty_skills_passed_correctly(monkeypatch):
supports_thinking = False
mock_app_config = MagicMock()
+ # make_lead_agent freezes the delta snapshot frequency from the app config;
+ # a bare MagicMock attribute cannot survive the freeze's positivity check.
+ mock_app_config.database.checkpoint_delta_snapshot_frequency = 1000
mock_app_config.get_model_config.return_value = MockModelConfig()
monkeypatch.setattr(lead_agent_module, "get_app_config", lambda: mock_app_config)
@@ -315,6 +318,9 @@ def test_make_lead_agent_custom_skill_allowlist_does_not_activate_tool_policy(mo
monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: [NamedTool("task"), NamedTool("bash"), NamedTool("read_file"), NamedTool("web_search")])
mock_app_config = MagicMock()
+ # make_lead_agent freezes the delta snapshot frequency from the app config;
+ # a bare MagicMock attribute cannot survive the freeze's positivity check.
+ mock_app_config.database.checkpoint_delta_snapshot_frequency = 1000
mock_app_config.get_model_config.return_value = SimpleNamespace(supports_thinking=False, supports_vision=False)
mock_app_config.tool_search.enabled = True
mock_app_config.skills.container_path = "/mnt/skills"
@@ -357,6 +363,9 @@ def test_make_lead_agent_all_legacy_skills_preserve_all_tools(monkeypatch):
monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: [NamedTool("bash"), NamedTool("read_file")])
mock_app_config = MagicMock()
+ # make_lead_agent freezes the delta snapshot frequency from the app config;
+ # a bare MagicMock attribute cannot survive the freeze's positivity check.
+ mock_app_config.database.checkpoint_delta_snapshot_frequency = 1000
mock_app_config.get_model_config.return_value = SimpleNamespace(supports_thinking=False, supports_vision=False)
monkeypatch.setattr(lead_agent_module, "get_app_config", lambda: mock_app_config)
@@ -410,6 +419,9 @@ def test_make_lead_agent_passive_empty_skill_policy_preserves_mcp_and_other_tool
)
mock_app_config = MagicMock()
+ # make_lead_agent freezes the delta snapshot frequency from the app config;
+ # a bare MagicMock attribute cannot survive the freeze's positivity check.
+ mock_app_config.database.checkpoint_delta_snapshot_frequency = 1000
mock_app_config.get_model_config.return_value = SimpleNamespace(supports_thinking=False, supports_vision=False)
mock_app_config.tool_search.enabled = True
mock_app_config.tool_search.auto_promote_top_k = 3
@@ -456,6 +468,9 @@ def test_default_lead_agent_does_not_apply_installed_skill_allowlists(monkeypatc
)
mock_app_config = MagicMock()
+ # make_lead_agent freezes the delta snapshot frequency from the app config;
+ # a bare MagicMock attribute cannot survive the freeze's positivity check.
+ mock_app_config.database.checkpoint_delta_snapshot_frequency = 1000
mock_app_config.get_model_config.return_value = SimpleNamespace(supports_thinking=False, supports_vision=False)
mock_app_config.tool_search.enabled = True
mock_app_config.skills.container_path = "/mnt/skills"
@@ -485,6 +500,9 @@ def test_make_lead_agent_fails_closed_when_skill_policy_load_fails(monkeypatch):
monkeypatch.setattr(lead_agent_module, "load_agent_config", lambda x: AgentConfig(name="test", skills=["restricted"]))
mock_app_config = MagicMock()
+ # make_lead_agent freezes the delta snapshot frequency from the app config;
+ # a bare MagicMock attribute cannot survive the freeze's positivity check.
+ mock_app_config.database.checkpoint_delta_snapshot_frequency = 1000
mock_app_config.get_model_config.return_value = SimpleNamespace(supports_thinking=False, supports_vision=False)
def fail_storage(*args, **kwargs):
@@ -530,6 +548,9 @@ def test_make_lead_agent_drops_update_agent_on_github_channel(monkeypatch):
monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: [NamedTool("bash"), NamedTool("read_file")])
mock_app_config = MagicMock()
+ # make_lead_agent freezes the delta snapshot frequency from the app config;
+ # a bare MagicMock attribute cannot survive the freeze's positivity check.
+ mock_app_config.database.checkpoint_delta_snapshot_frequency = 1000
mock_app_config.get_model_config.return_value = SimpleNamespace(supports_thinking=False, supports_vision=False)
monkeypatch.setattr(lead_agent_module, "get_app_config", lambda: mock_app_config)
@@ -566,6 +587,9 @@ def test_make_lead_agent_keeps_update_agent_on_non_webhook_channels(monkeypatch)
monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: [NamedTool("bash")])
mock_app_config = MagicMock()
+ # make_lead_agent freezes the delta snapshot frequency from the app config;
+ # a bare MagicMock attribute cannot survive the freeze's positivity check.
+ mock_app_config.database.checkpoint_delta_snapshot_frequency = 1000
mock_app_config.get_model_config.return_value = SimpleNamespace(supports_thinking=False, supports_vision=False)
monkeypatch.setattr(lead_agent_module, "get_app_config", lambda: mock_app_config)
diff --git a/backend/tests/test_summarize_checkpoint_channels.py b/backend/tests/test_summarize_checkpoint_channels.py
index d5bb76341..5ec7ee7dc 100644
--- a/backend/tests/test_summarize_checkpoint_channels.py
+++ b/backend/tests/test_summarize_checkpoint_channels.py
@@ -9,7 +9,7 @@ import pytest
def _load_module():
- path = Path(__file__).resolve().parents[1] / "scripts/benchmark/summarize_checkpoint_channels.py"
+ path = Path(__file__).resolve().parents[1] / "scripts/benchmark/checkpoint/summarize_channels.py"
spec = importlib.util.spec_from_file_location("summarize_checkpoint_channels", path)
assert spec is not None
module = importlib.util.module_from_spec(spec)
diff --git a/backend/tests/test_summarize_checkpoint_production.py b/backend/tests/test_summarize_checkpoint_production.py
new file mode 100644
index 000000000..19aa42dfd
--- /dev/null
+++ b/backend/tests/test_summarize_checkpoint_production.py
@@ -0,0 +1,128 @@
+"""Tests for the production benchmark summarizer."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+
+def _load_module():
+ path = Path(__file__).resolve().parents[1] / "scripts/benchmark/checkpoint/summarize_production.py"
+ spec = importlib.util.spec_from_file_location("summarize_production", path)
+ assert spec is not None
+ module = importlib.util.module_from_spec(spec)
+ assert spec.loader is not None
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+summ = _load_module()
+
+
+def _row(mode, rep, *, freq=None, turns=100, payload=128, run_p50=100.0, write_p50=10.0, write_p99=20.0, state_cold=50.0, state_warm=5.0, success=True):
+ return {
+ "mode": mode,
+ "repetition": rep,
+ "snapshot_frequency": freq,
+ "turns": turns,
+ "payload_bytes": payload,
+ "success": success,
+ "profiled": False,
+ "run_turn_p50_ms": run_p50,
+ "checkpoint_write_p50_ms": write_p50,
+ "checkpoint_write_p99_ms": write_p99,
+ "state_cold_ms": state_cold,
+ "state_warm_p50_ms": state_warm,
+ }
+
+
+def test_pairs_full_row_against_every_delta_frequency(tmp_path) -> None:
+ path = tmp_path / "rows.jsonl"
+ rows = [
+ _row("full", 0),
+ _row("delta", 0, freq=10, run_p50=80.0),
+ _row("delta", 0, freq=100, run_p50=90.0),
+ ]
+ path.write_text("\n".join(json.dumps(r) for r in rows) + "\n")
+ summaries = summ._summarize(summ._load_jsonl([path]), metrics=["run_turn_p50_ms"])
+ assert len(summaries) == 2
+ by_freq = {s["snapshot_frequency"]: s for s in summaries}
+ assert by_freq[10]["ratio_run_turn_p50_ms"] == 0.8
+ assert by_freq[100]["ratio_run_turn_p50_ms"] == 0.9
+
+
+def test_decision_metrics_emitted(tmp_path) -> None:
+ path = tmp_path / "rows.jsonl"
+ rows = [_row("full", 0), _row("delta", 0, freq=10, write_p50=10.0, write_p99=50.0)]
+ path.write_text("\n".join(json.dumps(r) for r in rows) + "\n")
+ summaries = summ._summarize(summ._load_jsonl([path]), metrics=["run_turn_p50_ms"])
+ assert summaries[0]["delta_snapshot_write_spike"] == 5.0
+ assert summaries[0]["delta_cache_effect_ms"] == 45.0
+ assert summaries[0]["full_cache_effect_ms"] == 45.0
+
+
+def test_failed_rows_counted_per_group(tmp_path) -> None:
+ path = tmp_path / "rows.jsonl"
+ rows = [
+ _row("full", 0),
+ _row("delta", 0, freq=10),
+ _row("delta", 0, freq=100),
+ _row("delta", 1, freq=10, success=False),
+ _row("delta", 2, freq=10, success=False),
+ _row("delta", 1, freq=100, success=False),
+ # Different turns: matches no group below.
+ _row("delta", 0, freq=10, turns=200, success=False),
+ # A failed full row counts toward every group at the same (turns, payload_bytes).
+ _row("full", 1, success=False),
+ ]
+ path.write_text("\n".join(json.dumps(r) for r in rows) + "\n")
+ summaries = summ._summarize(summ._load_jsonl([path]), metrics=["run_turn_p50_ms"])
+ by_freq = {s["snapshot_frequency"]: s for s in summaries}
+ assert by_freq[10]["failed_rows"] == 3
+ assert by_freq[100]["failed_rows"] == 2
+
+
+def test_history_limit_metrics_auto_discovered(tmp_path, capsys) -> None:
+ path = tmp_path / "rows.jsonl"
+ full = _row("full", 0)
+ delta = _row("delta", 0, freq=10)
+ for row, cold, warm in ((full, 30.0, 12.0), (delta, 60.0, 24.0)):
+ row["history_limit_64_cold_ms"] = cold
+ row["history_limit_64_warm_p50_ms"] = warm
+ path.write_text("\n".join(json.dumps(r) for r in (full, delta)) + "\n")
+ assert summ.main([str(path), "--json"]) == 0
+ out = json.loads(capsys.readouterr().out)
+ assert out[0]["ratio_history_limit_64_cold_ms"] == 2.0
+ assert out[0]["delta_history_limit_64_warm_p50_ms"] == 24.0
+
+
+def test_explicit_metrics_override_skips_discovery(tmp_path, capsys) -> None:
+ path = tmp_path / "rows.jsonl"
+ full = _row("full", 0)
+ delta = _row("delta", 0, freq=10)
+ for row in (full, delta):
+ row["history_limit_64_cold_ms"] = 30.0
+ path.write_text("\n".join(json.dumps(r) for r in (full, delta)) + "\n")
+ assert summ.main([str(path), "--json", "--metrics", "run_turn_p50_ms"]) == 0
+ out = json.loads(capsys.readouterr().out)
+ assert "delta_history_limit_64_cold_ms" not in out[0]
+
+
+def test_checkpoint_write_share_emitted_and_zero_guarded(tmp_path) -> None:
+ path = tmp_path / "rows.jsonl"
+ rows = [
+ _row("full", 0, run_p50=100.0, write_p50=10.0),
+ _row("delta", 0, freq=10, run_p50=80.0, write_p50=20.0),
+ _row("full", 0, turns=200, run_p50=0.0, write_p50=10.0),
+ _row("delta", 0, freq=10, turns=200, run_p50=0.0, write_p50=20.0),
+ ]
+ path.write_text("\n".join(json.dumps(r) for r in rows) + "\n")
+ summaries = summ._summarize(summ._load_jsonl([path]), metrics=["run_turn_p50_ms"])
+ by_turns = {s["turns"]: s for s in summaries}
+ assert by_turns[100]["full_checkpoint_write_share"] == 0.1
+ assert by_turns[100]["delta_checkpoint_write_share"] == 0.25
+ assert "full_checkpoint_write_share" not in by_turns[200]
+ assert "delta_checkpoint_write_share" not in by_turns[200]
diff --git a/backend/tests/test_token_usage.py b/backend/tests/test_token_usage.py
index bec9e9ac3..fd679329f 100644
--- a/backend/tests/test_token_usage.py
+++ b/backend/tests/test_token_usage.py
@@ -147,6 +147,7 @@ def _mock_app_config():
model.model_dump.return_value = {"name": "test-model", "use": "langchain_openai:ChatOpenAI"}
config = MagicMock()
config.models = [model]
+ config.database.checkpoint_delta_snapshot_frequency = 1000
return config
From 1baa8ad69609530b95057b581f69ced487a5ed9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=98=BF=E6=B3=BD?=
Date: Mon, 27 Jul 2026 14:05:31 +0800
Subject: [PATCH 03/15] feat(clarification): structured form fields for
human-input cards (#4400 Phase 1) (#4406)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat(clarification): structured form fields for human-input cards
Add a request-side v2 `form` mode to the ask_clarification protocol so
business flows (e.g. expense reimbursement) can collect several values
in one card instead of sequential free-text questions:
- `ask_clarification` gains a restricted `fields` parameter (text /
textarea / number / select / multi_select / checkbox / date)
- ClarificationMiddleware validates and normalizes fields explicitly
(whitelisted types, unknown -> text, select-likes without options ->
text, duplicate/invalid entries dropped, all-invalid falls back to
the legacy modes) since the middleware short-circuits before tool
execution; the plain-text fallback lists fields for IM channels
- Form payloads carry `version: 2` so older frontends degrade to the
text fallback; replies stay on the v1 response protocol — the card
submits a readable summary as `response_kind: "text"`, so journal
persistence and answered-card recovery are unchanged
- Frontend renders typed field controls with required-field validation
and compact multi-select chips
Part of #4400 (scope narrowed per maintainer feedback: request-side
only, no new response kinds, no top-level multi_choice).
Co-Authored-By: Claude Fable 5
* fix(clarification): harden form protocol per review feedback
Address the five review points on #4406:
- Reject field names colliding with JS Object.prototype members on both
sides; frontend reads form values via own-property access only, so
`constructor`/`toString`-style names can no longer leak inherited
members into required validation or the submitted summary
- Close open requests answered through the legacy text fallback: a
visible plain human reply (no response metadata) now marks every
previously-opened request as answered, so upgrading to a v2-aware
frontend cannot leave the composer locked on an already-answered card
- Give checkbox fields deterministic boolean semantics: values are
seeded to an explicit false ("no" in the summary) and `required` means
must-agree/consent; documented in the tool schema
- Make middleware field validation atomic: structurally broken entries
(bad/duplicate/reserved names, over-cap field/option counts or text
lengths) degrade the whole form instead of silently dropping fields;
options are trimmed/deduped with blanks removed so the backend never
emits payloads the frontend parser rejects
- Associate form labels/controls (htmlFor/id), aria-required,
aria-invalid, and error descriptions for accessibility
Co-Authored-By: Claude Fable 5
* refactor(clarification): type the fields item schema via TypedDict
Replace `fields: list[dict[str, Any]]` with `list[ClarificationFormField]`
(a TypedDict with `name` required and the type whitelist as a Literal) so
the provider-facing tool schema documents the item shape instead of an
opaque object relying on the docstring. Runtime validation is unchanged
and stays in ClarificationMiddleware, which intercepts the call before
tool execution. Addresses the non-blocking review suggestion on #4406.
Co-Authored-By: Claude Fable 5
* fix(frontend): drop unsupported aria-invalid from multi-select group
jsx-a11y: role=group does not support aria-invalid; the error linkage
stays via aria-describedby.
Co-Authored-By: Claude Fable 5
* fix(clarification): coerce numeric required flags and normalize fields once
- `_normalize_bool` now coerces 1/0 (some providers serialize booleans
as integers), so `required: 1` no longer silently flips to optional
- `_handle_clarification` normalizes `fields` once and passes the result
to both the text fallback and the payload builder
Addresses the non-blocking review nits on #4406.
Co-Authored-By: Claude Fable 5
* fix(clarification): harden form protocol per contract review round 2
Backend:
- Guard unhashable JSON in the intercept path: `type: []`/`{}` degrades
the field to text and `clarification_type: []` coerces to str instead
of raising TypeError (which, with return_direct, ended the turn with
an error and no card or fallback)
- Add a total budget over the serialized normalized fields (16KB UTF-8
bytes): per-item caps alone admitted forms whose IM text fallback
exceeded channel delivery limits (Slack 40k chars, Feishu ~30KB card),
silently truncating trailing fields; a boundary test proves any
accepted form's fallback stays deliverable
Frontend:
- Submission value now appends a JSON block keyed by stable field names
(readable summary alone is delimiter-ambiguous), with a collision
regression test
- Parser boundary tightened to match backend constraints: empty option
values (Radix SelectItem crash), duplicate option ids/values,
duplicate field names, and the form<->version-2 binding are rejected
- Keep the error node mounted while any field is still invalid so
aria-describedby never points at a removed element (happy-dom
interaction test)
- Required semantics are now accessible: native checkbox control (no
HTML required attribute — it would intercept the custom submit path),
visually-hidden localized "required" markers next to the aria-hidden
asterisks
- Legacy-fallback closure narrowed to the latest unanswered request:
nothing guarantees a single outstanding clarification across runs, and
closing all would silently swallow older decisions; an older request
left open becomes the active card again
Co-Authored-By: Claude Fable 5
* fix(frontend): keep clarification selects controlled
---------
Co-authored-by: Claude Fable 5
---
backend/AGENTS.md | 4 +-
.../middlewares/clarification_middleware.py | 211 +++++++++-
.../tools/builtins/clarification_tool.py | 41 +-
.../tests/test_clarification_middleware.py | 390 ++++++++++++++++++
backend/tests/test_human_input.py | 7 +
frontend/AGENTS.md | 2 +-
.../workspace/messages/human-input-card.tsx | 348 +++++++++++++++-
frontend/src/core/i18n/locales/en-US.ts | 3 +
frontend/src/core/i18n/locales/types.ts | 3 +
frontend/src/core/i18n/locales/zh-CN.ts | 3 +
frontend/src/core/messages/human-input.ts | 280 ++++++++++++-
.../messages/human-input-card.dom.test.tsx | 103 +++++
.../messages/human-input-card.test.ts | 111 +++++
.../unit/core/messages/human-input.test.ts | 348 ++++++++++++++++
14 files changed, 1826 insertions(+), 28 deletions(-)
create mode 100644 frontend/tests/unit/components/workspace/messages/human-input-card.dom.test.tsx
diff --git a/backend/AGENTS.md b/backend/AGENTS.md
index 79948b0f3..7960776ed 100644
--- a/backend/AGENTS.md
+++ b/backend/AGENTS.md
@@ -365,7 +365,7 @@ Before changing a later authorization phase, read the [authorization RFC](../doc
32. **TerminalResponseMiddleware** - When a provider returns an empty terminal `AIMessage` after tool execution, injects a hidden recovery prompt and retries the model once; a second empty response is replaced in checkpoint state by a visible error fallback marked for the run worker, so the run finishes as an error instead of a silent success
33. **ModelLengthFinishReasonMiddleware** - Records `stop_reason=model_length_capped` when provider-specific length detectors match a terminal `AIMessage` without tool-call intent (`finish_reason=length` / `MAX_TOKENS`, or `stop_reason=max_tokens`), preserving the original assistant content and never reparsing textual tool-call-like envelopes
34. **SafetyFinishReasonMiddleware** - *(optional, if `safety_finish_reason.enabled`)* Suppresses tool execution when the provider safety-terminated the response (e.g. `finish_reason=content_filter`); registered after terminal-response/custom/configured middlewares so LangChain's reverse-order `after_model` dispatch runs it first
-35. **ClarificationMiddleware** - Intercepts `ask_clarification` tool calls, writes a readable `ToolMessage.content` fallback plus structured `ToolMessage.artifact.human_input` request payload, and interrupts via `Command(goto=END)` (must be last). Because this middleware can short-circuit tool execution before LangChain emits `on_tool_end`, `RunJournal` performs a root-run final reconciliation for allowlisted clarification `ToolMessage`s whose `tool_call_id` was produced by the current run, so human-input request cards remain recoverable from `run_events` after checkpoint compaction. Human Input Card replies are submitted as `hide_from_ui` `HumanMessage`s with `additional_kwargs.human_input_response`; `RunJournal` persists only allowlisted hidden response sources (currently `ask_clarification`) as `llm.human.input`, which preserves answered-card state after compaction without exposing generic internal hidden context.
+35. **ClarificationMiddleware** - Intercepts `ask_clarification` tool calls, writes a readable `ToolMessage.content` fallback plus structured `ToolMessage.artifact.human_input` request payload, and interrupts via `Command(goto=END)` (must be last). Payloads are versioned: legacy modes (`free_text` / `choice_with_other`) keep `version: 1` unchanged, while the v2 `form` mode (from `fields`) carries `version: 2` so older frontends reject the payload and degrade to the plain-text fallback. Field normalization is deterministic and lives in the middleware, not the tool schema — the middleware short-circuits before tool execution, so tool-arg typing alone provides no runtime validation. Validation is atomic: any structurally broken entry (non-dict, bad/duplicate name, a name colliding with a JS `Object.prototype` member like `__proto__`/`constructor`, exceeding the caps of 16 fields / 24 options per field / 200 chars per text, or the whole normalized definition exceeding `MAX_FORM_SERIALIZED_BYTES` = 16KB UTF-8 — the per-item caps alone admit forms whose IM text fallback would blow channel delivery limits and truncate away trailing fields) degrades the whole form to the legacy option/free-text modes, so a card can never render "complete" while silently missing a business field; benign issues keep local degradation (unknown types — including unhashable JSON like `type: []`, which must never raise from the membership probe — and option-less selects become `text`), and options are trimmed/deduped with blanks dropped (both form-level and top-level) because the frontend parser rejects blank option labels. Checkbox fields are booleans that default to an explicit "no"; `required` on a checkbox means must-agree/consent semantics. The response protocol is deliberately unchanged (v1 `text`/`option` only): form cards submit a readable text summary as `response_kind: "text"`, so journal persistence and answered-card recovery need no new allowlist entries. Because this middleware can short-circuit tool execution before LangChain emits `on_tool_end`, `RunJournal` performs a root-run final reconciliation for allowlisted clarification `ToolMessage`s whose `tool_call_id` was produced by the current run, so human-input request cards remain recoverable from `run_events` after checkpoint compaction. Human Input Card replies are submitted as `hide_from_ui` `HumanMessage`s with `additional_kwargs.human_input_response`; `RunJournal` persists only allowlisted hidden response sources (currently `ask_clarification`) as `llm.human.input`, which preserves answered-card state after compaction without exposing generic internal hidden context.
### Configuration System
@@ -613,7 +613,7 @@ copying a raw checkpoint because delta state is not self-contained in one tuple.
2. **MCP tools** - From enabled MCP servers (lazy initialized, cached with resolved-path + content-signature invalidation)
3. **Built-in tools**:
- `present_files` - Make output files visible to user (only `/mnt/user-data/outputs`)
- - `ask_clarification` - Request clarification (intercepted by ClarificationMiddleware, which preserves text fallback and adds `artifact.human_input` for Web UI Human Input Cards)
+ - `ask_clarification` - Request clarification (intercepted by ClarificationMiddleware, which preserves text fallback and adds `artifact.human_input` for Web UI Human Input Cards). Beyond free text and single choice, the request-side v2 protocol supports `fields` (structured form card collecting several values at once; field types: text/textarea/number/select/multi_select/checkbox/date, validated and normalized server-side in the middleware — invalid entries are dropped, unknown types degrade to `text`; a standalone multi-select question is a one-field form). Replies stay on the v1 response protocol (`text`/`option`): the form card submits a readable text summary
- `view_image` - Read image as base64 (added only if model supports vision)
- `setup_agent` - Bootstrap-only: persist a brand-new custom agent's `SOUL.md` and `config.yaml`. Bound only when `is_bootstrap=True`.
- `update_agent` - Custom-agent-only: persist self-updates to the current agent's `SOUL.md` / `config.yaml` from inside a normal chat (partial update + atomic write). Bound when `agent_name` is set and `is_bootstrap=False`.
diff --git a/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py
index 3adf0d9c4..488740331 100644
--- a/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py
+++ b/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py
@@ -15,6 +15,47 @@ from langgraph.types import Command
logger = logging.getLogger(__name__)
+# Whitelisted form field types; anything else degrades to "text" so a bad
+# model-provided type can never produce an unrenderable card.
+FORM_FIELD_TYPES = frozenset({"text", "textarea", "number", "select", "multi_select", "checkbox", "date"})
+_OPTION_FIELD_TYPES = frozenset({"select", "multi_select"})
+
+# Field names that collide with JavaScript Object.prototype properties. The
+# frontend stores form values in a plain object keyed by field name, so these
+# would read inherited prototype members instead of user input.
+_RESERVED_FIELD_NAMES = frozenset(
+ {
+ "__proto__",
+ "constructor",
+ "prototype",
+ "toString",
+ "toLocaleString",
+ "valueOf",
+ "hasOwnProperty",
+ "isPrototypeOf",
+ "propertyIsEnumerable",
+ "__defineGetter__",
+ "__defineSetter__",
+ "__lookupGetter__",
+ "__lookupSetter__",
+ }
+)
+
+# Hard caps so a runaway model cannot publish an unbounded form. Exceeding a
+# cap is a structural error: the whole form degrades to the legacy modes
+# instead of silently truncating business fields.
+MAX_FORM_FIELDS = 16
+MAX_FIELD_OPTIONS = 24
+MAX_FIELD_TEXT_CHARS = 200
+# Total budget over the serialized normalized fields, in UTF-8 bytes. The
+# per-item caps alone still admit forms whose plain-text IM fallback exceeds
+# channel delivery limits (Slack truncates at 40k chars per message; Feishu
+# guides ~30KB per card), which would silently drop trailing fields — the very
+# thing atomic validation exists to prevent. 16KB keeps the fallback text of
+# any accepted form comfortably inside the strictest supported channel while
+# leaving headroom for question/context.
+MAX_FORM_SERIALIZED_BYTES = 16_384
+
class ClarificationMiddlewareState(AgentState):
"""Compatible with the `ThreadState` schema."""
@@ -62,21 +103,141 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]):
if not isinstance(options, list):
options = [options]
- return [str(option) for option in options]
+ # Trim, drop blanks, and dedupe (order-preserving): the frontend parser
+ # rejects the whole payload on blank option labels, so they must never
+ # be emitted.
+ normalized: list[str] = []
+ seen: set[str] = set()
+ for option in options:
+ text = str(option).strip()
+ if not text or text in seen:
+ continue
+ seen.add(text)
+ normalized.append(text)
+ return normalized
- def _build_human_input_payload(self, args: dict[str, Any], *, tool_call_id: str, request_id: str) -> dict[str, Any]:
- """Build the structured UI payload while keeping ToolMessage.content as fallback."""
+ @staticmethod
+ def _normalize_bool(raw: Any) -> bool:
+ """Coerce a model-provided boolean; some models serialize booleans as strings or 1/0."""
+ if isinstance(raw, bool):
+ return raw
+ if isinstance(raw, int | float):
+ return bool(raw)
+ if isinstance(raw, str):
+ return raw.strip().lower() == "true"
+ return False
+
+ def _normalize_fields(self, raw_fields: Any) -> list[dict[str, Any]]:
+ """Normalize tool-provided form fields into the validated v2 field schema.
+
+ Validation is atomic: any structurally broken entry (non-dict, bad or
+ reserved or duplicate name, over-cap counts/lengths) invalidates the
+ whole form so the card can never render "complete" while silently
+ missing a required business field. Benign issues keep their local
+ degradation: unknown types and option-less selects become ``text``.
+ """
+ fields = raw_fields
+ if isinstance(fields, str):
+ try:
+ fields = json.loads(fields)
+ except (json.JSONDecodeError, TypeError):
+ return []
+ if not isinstance(fields, list):
+ return []
+ if len(fields) > MAX_FORM_FIELDS:
+ return []
+
+ normalized: list[dict[str, Any]] = []
+ seen_names: set[str] = set()
+ for entry in fields:
+ if not isinstance(entry, dict):
+ return []
+ raw_name = entry.get("name")
+ if not isinstance(raw_name, str) or not raw_name.strip():
+ return []
+ name = raw_name.strip()
+ if name in _RESERVED_FIELD_NAMES or name in seen_names or len(name) > MAX_FIELD_TEXT_CHARS:
+ return []
+ seen_names.add(name)
+
+ raw_label = entry.get("label")
+ label = raw_label.strip() if isinstance(raw_label, str) and raw_label.strip() else name
+ if len(label) > MAX_FIELD_TEXT_CHARS:
+ return []
+
+ field_type = entry.get("type")
+ # isinstance guard first: `type: []` / `type: {}` are legal JSON
+ # from a model, and an unhashable membership probe would raise
+ # TypeError instead of degrading.
+ if not isinstance(field_type, str) or field_type not in FORM_FIELD_TYPES:
+ field_type = "text"
+
+ options = self._normalize_options(entry.get("options")) if field_type in _OPTION_FIELD_TYPES else []
+ if len(options) > MAX_FIELD_OPTIONS or any(len(option) > MAX_FIELD_TEXT_CHARS for option in options):
+ return []
+ if field_type in _OPTION_FIELD_TYPES and not options:
+ field_type = "text"
+
+ field: dict[str, Any] = {
+ "name": name,
+ "label": label,
+ "type": field_type,
+ "required": self._normalize_bool(entry.get("required")),
+ }
+ if field_type in _OPTION_FIELD_TYPES:
+ field["options"] = [
+ {
+ "id": f"{name}-option-{index}",
+ "label": option,
+ "value": option,
+ }
+ for index, option in enumerate(options, 1)
+ ]
+ placeholder = entry.get("placeholder")
+ if isinstance(placeholder, str) and placeholder.strip():
+ if len(placeholder.strip()) > MAX_FIELD_TEXT_CHARS:
+ return []
+ field["placeholder"] = placeholder.strip()
+ normalized.append(field)
+
+ if len(json.dumps(normalized, ensure_ascii=False).encode("utf-8")) > MAX_FORM_SERIALIZED_BYTES:
+ return []
+
+ return normalized
+
+ def _build_human_input_payload(self, args: dict[str, Any], *, tool_call_id: str, request_id: str, fields: list[dict[str, Any]] | None = None) -> dict[str, Any]:
+ """Build the structured UI payload while keeping ToolMessage.content as fallback.
+
+ Protocol versioning: legacy modes (``free_text`` / ``choice_with_other``)
+ keep ``version: 1`` so their wire format is unchanged; the v2 ``form``
+ mode carries ``version: 2`` so older frontends reject the payload and
+ degrade to the plain-text ToolMessage content. Replies stay on the v1
+ response protocol (``text`` / ``option``) — the form card submits a
+ readable ``value`` summary, so no new response kind is introduced.
+
+ ``fields`` accepts an already-normalized list so callers rendering both
+ the payload and the text fallback normalize only once.
+ """
+ if fields is None:
+ fields = self._normalize_fields(args.get("fields"))
options = self._normalize_options(args.get("options", []))
clarification_type = str(args.get("clarification_type", "missing_info"))
+ if fields:
+ version, input_mode = 2, "form"
+ elif options:
+ version, input_mode = 1, "choice_with_other"
+ else:
+ version, input_mode = 1, "free_text"
+
payload: dict[str, Any] = {
- "version": 1,
+ "version": version,
"kind": "human_input_request",
"source": "ask_clarification",
"request_id": request_id,
"clarification_type": clarification_type,
"question": str(args.get("question") or ""),
- "input_mode": "choice_with_other" if options else "free_text",
+ "input_mode": input_mode,
}
if tool_call_id:
@@ -86,7 +247,9 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]):
context = args.get("context")
payload["context"] = None if context is None else str(context)
- if options:
+ if input_mode == "form":
+ payload["fields"] = fields
+ elif options:
payload["options"] = [
{
"id": f"option-{index}",
@@ -109,18 +272,24 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]):
"""
return any("\u4e00" <= char <= "\u9fff" for char in text)
- def _format_clarification_message(self, args: dict) -> str:
+ def _format_clarification_message(self, args: dict, fields: list[dict[str, Any]] | None = None) -> str:
"""Format the clarification arguments into a user-friendly message.
Args:
args: The tool call arguments containing clarification details
+ fields: Already-normalized form fields, so callers rendering both
+ the payload and this fallback normalize only once
Returns:
Formatted message string
"""
question = args.get("question", "")
- clarification_type = args.get("clarification_type", "missing_info")
+ # str() coercion keeps the icon lookup hashable — `clarification_type:
+ # []` is legal JSON from a model and would raise TypeError as a dict key.
+ clarification_type = str(args.get("clarification_type", "missing_info"))
context = args.get("context")
+ if fields is None:
+ fields = self._normalize_fields(args.get("fields"))
options = self._normalize_options(args.get("options", []))
# Type-specific icons
@@ -146,8 +315,22 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]):
# Just the question with icon
message_parts.append(f"{icon} {question}")
- # Add options in a cleaner format
- if options and len(options) > 0:
+ # Form fields take precedence over options, mirroring the payload logic.
+ if fields:
+ message_parts.append("") # blank line for spacing
+ for i, field in enumerate(fields, 1):
+ line = f" {i}. {field['label']}"
+ if field["required"]:
+ line += " (required)"
+ field_options = field.get("options")
+ if field_options:
+ line += " — options: " + " / ".join(option["label"] for option in field_options)
+ if field["type"] == "multi_select":
+ line += " (multiple allowed)"
+ message_parts.append(line)
+ message_parts.append("")
+ message_parts.append("Please reply with a value for each field.")
+ elif options and len(options) > 0:
message_parts.append("") # blank line for spacing
for i, option in enumerate(options, 1):
message_parts.append(f" {i}. {option}")
@@ -208,14 +391,18 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]):
logger.info("Intercepted clarification request")
logger.debug("Clarification question: %s", question)
+ # Normalize form fields once; both the text fallback and the payload
+ # consume the same result.
+ fields = self._normalize_fields(args.get("fields"))
+
# Format the clarification message
- formatted_message = self._format_clarification_message(args)
+ formatted_message = self._format_clarification_message(args, fields=fields)
# Get the tool call ID
tool_call_id = request.tool_call.get("id", "")
request_id = self._stable_message_id(tool_call_id, formatted_message)
- human_input_payload = self._build_human_input_payload(args, tool_call_id=tool_call_id, request_id=request_id)
+ human_input_payload = self._build_human_input_payload(args, tool_call_id=tool_call_id, request_id=request_id, fields=fields)
# Create a ToolMessage with the formatted question
# This will be added to the message history
diff --git a/backend/packages/harness/deerflow/tools/builtins/clarification_tool.py b/backend/packages/harness/deerflow/tools/builtins/clarification_tool.py
index 49c3db102..8bf0de9b4 100644
--- a/backend/packages/harness/deerflow/tools/builtins/clarification_tool.py
+++ b/backend/packages/harness/deerflow/tools/builtins/clarification_tool.py
@@ -1,8 +1,24 @@
-from typing import Literal
+from typing import Literal, Required, TypedDict
from langchain.tools import tool
+class ClarificationFormField(TypedDict, total=False):
+ """One form field definition for a structured clarification card.
+
+ The model-visible schema documents the item shape; runtime validation
+ still happens defensively in ``ClarificationMiddleware`` because the
+ middleware intercepts the call before tool execution.
+ """
+
+ name: Required[str]
+ label: str
+ type: Literal["text", "textarea", "number", "select", "multi_select", "checkbox", "date"]
+ required: bool
+ options: list[str]
+ placeholder: str
+
+
@tool("ask_clarification", parse_docstring=True, return_direct=True)
def ask_clarification_tool(
question: str,
@@ -15,6 +31,7 @@ def ask_clarification_tool(
],
context: str | None = None,
options: list[str] | None = None,
+ fields: list[ClarificationFormField] | None = None,
) -> str:
"""Ask the user for clarification when you need more information to proceed.
@@ -36,11 +53,22 @@ def ask_clarification_tool(
- You're about to perform a potentially dangerous operation
- You have a recommendation but need user approval
+ Choosing the interaction shape:
+ - One open question -> just `question` (free text input)
+ - Pick exactly one option -> `options`
+ - Pick several options -> a single `fields` entry of type `multi_select`
+ - Collect several values at once (e.g. a set of parameters for one action) ->
+ `fields`, which renders a single structured form instead of several
+ sequential questions. Prefer one form over asking field-by-field.
+
Best practices:
- - Ask ONE clarification at a time for clarity
+ - Ask ONE clarification at a time for clarity; a form with several fields
+ still counts as one clarification
- Be specific and clear in your question
- Don't make assumptions when clarification is needed
- For risky operations, ALWAYS ask for confirmation
+ - If a skill provides a predefined field template, pass it through `fields`
+ unchanged instead of redesigning it
- After calling this tool, execution will be interrupted automatically
Args:
@@ -48,6 +76,15 @@ def ask_clarification_tool(
clarification_type: The type of clarification needed (missing_info, ambiguous_requirement, approach_choice, risk_confirmation, suggestion).
context: Optional context explaining why clarification is needed. Helps the user understand the situation.
options: Optional list of choices (for approach_choice or suggestion types). Present clear options for the user to choose from.
+ fields: Optional form field definitions for collecting multiple values in one card; takes precedence over `options`.
+ Each field is an object with `name` (unique identifier, required; avoid JavaScript prototype names like
+ `constructor` or `toString`), `label` (display text, defaults to name), `type` (one of: text, textarea,
+ number, select, multi_select, checkbox, date; defaults to text), `required` (boolean, defaults to false),
+ `options` (list of strings, required for select/multi_select types), and `placeholder` (optional hint text).
+ A `checkbox` field is a boolean that defaults to "no"; set `required` on a checkbox only for
+ must-agree/consent semantics (the user has to tick it to submit). Keep forms bounded: at most 16 fields,
+ 24 options per field, and 200 characters per name/label/option/placeholder — exceeding a limit degrades
+ the whole request to a plain-text question.
"""
# This is a placeholder implementation
# The actual logic is handled by ClarificationMiddleware which intercepts this tool call
diff --git a/backend/tests/test_clarification_middleware.py b/backend/tests/test_clarification_middleware.py
index 040081e12..8155443f5 100644
--- a/backend/tests/test_clarification_middleware.py
+++ b/backend/tests/test_clarification_middleware.py
@@ -215,6 +215,396 @@ class TestHumanInputPayload:
assert "options" not in payload
+class TestFormPayload:
+ """v2 protocol: fields render a structured form card."""
+
+ def _fields(self):
+ return [
+ {"name": "amount", "label": "Amount", "type": "number", "required": True},
+ {"name": "category", "label": "Category", "type": "select", "options": ["travel", "meals"], "required": True},
+ {"name": "receipts", "label": "Receipts", "type": "multi_select", "options": ["A-1", "A-2"]},
+ {"name": "note", "type": "textarea"},
+ ]
+
+ def test_form_payload_with_native_fields(self, middleware):
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Please provide the expense details.",
+ "clarification_type": "missing_info",
+ "fields": self._fields(),
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["version"] == 2
+ assert payload["input_mode"] == "form"
+ assert payload["fields"] == [
+ {"name": "amount", "label": "Amount", "type": "number", "required": True},
+ {
+ "name": "category",
+ "label": "Category",
+ "type": "select",
+ "required": True,
+ "options": [
+ {"id": "category-option-1", "label": "travel", "value": "travel"},
+ {"id": "category-option-2", "label": "meals", "value": "meals"},
+ ],
+ },
+ {
+ "name": "receipts",
+ "label": "Receipts",
+ "type": "multi_select",
+ "required": False,
+ "options": [
+ {"id": "receipts-option-1", "label": "A-1", "value": "A-1"},
+ {"id": "receipts-option-2", "label": "A-2", "value": "A-2"},
+ ],
+ },
+ {"name": "note", "label": "note", "type": "textarea", "required": False},
+ ]
+
+ def test_form_payload_with_json_string_fields(self, middleware):
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": json.dumps([{"name": "amount", "type": "number"}]),
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["input_mode"] == "form"
+ assert payload["fields"] == [{"name": "amount", "label": "amount", "type": "number", "required": False}]
+
+ def test_form_takes_precedence_over_options(self, middleware):
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": [{"name": "amount"}],
+ "options": ["dev", "prod"],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["input_mode"] == "form"
+ assert "options" not in payload
+
+ def test_unknown_field_type_degrades_to_text(self, middleware):
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": [{"name": "amount", "type": "slider"}],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["fields"][0]["type"] == "text"
+
+ def test_select_without_options_degrades_to_text(self, middleware):
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": [{"name": "category", "type": "select"}],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["fields"][0]["type"] == "text"
+ assert "options" not in payload["fields"][0]
+
+ def test_any_invalid_field_entry_degrades_whole_form(self, middleware):
+ """Atomic validation: a structurally broken entry must not silently
+ vanish from an otherwise-rendered form — the whole form degrades."""
+ for bad_entry in ["not-a-dict", {"label": "no name"}, {"name": " "}]:
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": [bad_entry, {"name": "amount", "type": "number"}],
+ "options": ["dev", "prod"],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["version"] == 1
+ assert payload["input_mode"] == "choice_with_other"
+ assert "fields" not in payload
+
+ def test_duplicate_field_names_degrade_whole_form(self, middleware):
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": [
+ {"name": "amount", "type": "number"},
+ {"name": "amount", "type": "text"},
+ ],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["version"] == 1
+ assert payload["input_mode"] == "free_text"
+ assert "fields" not in payload
+
+ def test_reserved_prototype_field_names_degrade_whole_form(self, middleware):
+ """`__proto__`/`constructor`/... collide with JS Object.prototype in the
+ frontend form-value store and must be rejected server-side."""
+ for reserved in ["__proto__", "constructor", "toString", "hasOwnProperty"]:
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": [{"name": reserved, "type": "text"}],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["input_mode"] == "free_text", reserved
+ assert "fields" not in payload
+
+ def test_field_options_are_trimmed_deduped_and_blank_dropped(self, middleware):
+ """Backend must never emit blank option labels — the frontend parser
+ rejects the whole payload on them."""
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": [{"name": "category", "type": "select", "options": [" travel ", "", " ", "travel", "meals"]}],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert [option["label"] for option in payload["fields"][0]["options"]] == ["travel", "meals"]
+
+ def test_field_count_over_cap_degrades_whole_form(self, middleware):
+ fields = [{"name": f"field_{i}", "type": "text"} for i in range(17)]
+ payload = middleware._build_human_input_payload(
+ {"question": "Details please", "clarification_type": "missing_info", "fields": fields},
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["input_mode"] == "free_text"
+ assert "fields" not in payload
+
+ def test_option_count_over_cap_degrades_whole_form(self, middleware):
+ options = [f"option-{i}" for i in range(25)]
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": [{"name": "category", "type": "select", "options": options}],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["input_mode"] == "free_text"
+ assert "fields" not in payload
+
+ def test_overlong_field_text_degrades_whole_form(self, middleware):
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": [{"name": "amount", "label": "x" * 201, "type": "number"}],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["input_mode"] == "free_text"
+ assert "fields" not in payload
+
+ def test_top_level_options_are_trimmed_and_blank_dropped(self, middleware):
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Which env?",
+ "clarification_type": "approach_choice",
+ "options": [" dev ", "", "prod", "dev"],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["options"] == [
+ {"id": "option-1", "label": "dev", "value": "dev"},
+ {"id": "option-2", "label": "prod", "value": "prod"},
+ ]
+
+ def test_unhashable_field_type_degrades_to_text_not_crash(self, middleware):
+ """`type: []` / `type: {}` are legal JSON from a model; an unhashable
+ membership check would raise TypeError and (with return_direct=True)
+ end the turn with an error instead of any card or fallback."""
+ for bad_type in [[], {}, ["select"], {"t": "select"}]:
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": [{"name": "amount", "type": bad_type}],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["fields"][0]["type"] == "text"
+
+ def test_unhashable_clarification_type_does_not_crash(self, middleware):
+ """Same crash class as unhashable field types: `clarification_type: []`
+ must not raise in the icon lookup or payload builder."""
+ from langgraph.types import Command
+
+ request = SimpleNamespace(
+ tool_call={
+ "name": "ask_clarification",
+ "id": "call-clarify-1",
+ "args": {"question": "q?", "clarification_type": [], "fields": [{"name": "amount", "type": []}]},
+ },
+ runtime=None,
+ )
+
+ result = middleware.wrap_tool_call(request, lambda _req: pytest.fail("handler should not be called"))
+
+ assert isinstance(result, Command)
+ message = result.update["messages"][0]
+ assert message.artifact["human_input"]["input_mode"] == "form"
+
+ def test_serialized_fields_over_byte_budget_degrade_whole_form(self, middleware):
+ """Per-item caps alone allow a form whose IM text fallback exceeds
+ channel limits (Slack 40k chars, Feishu ~30KB card); a total serialized
+ byte budget must bound the whole definition."""
+ fields = [
+ {
+ "name": f"field_{i}",
+ "label": "标" * 190,
+ "type": "select",
+ "options": ["选" * 190 + str(j) for j in range(20)],
+ }
+ for i in range(16)
+ ]
+ payload = middleware._build_human_input_payload(
+ {"question": "Details please", "clarification_type": "missing_info", "fields": fields},
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["input_mode"] == "free_text"
+ assert "fields" not in payload
+
+ def test_accepted_forms_keep_text_fallback_under_channel_limits(self, middleware):
+ """Boundary: any form the byte budget accepts must produce an IM text
+ fallback deliverable to the strictest supported channel (~30KB)."""
+ # Grow a form until the budget rejects it; the largest accepted form's
+ # fallback must stay under the channel bound.
+ largest_accepted_fallback = None
+ for count in range(1, 17):
+ fields = [
+ {
+ "name": f"field_{i}",
+ "label": "标" * 150,
+ "type": "select",
+ "options": ["选" * 100 + str(j) for j in range(10)],
+ }
+ for i in range(count)
+ ]
+ args = {"question": "Q", "clarification_type": "missing_info", "fields": fields}
+ payload = middleware._build_human_input_payload(args, tool_call_id="c", request_id="clarification:c")
+ if payload["input_mode"] != "form":
+ break
+ largest_accepted_fallback = middleware._format_clarification_message(args)
+
+ assert largest_accepted_fallback is not None
+ assert len(largest_accepted_fallback.encode("utf-8")) < 30_000
+
+ def test_required_accepts_integer_serialization(self, middleware):
+ """Some providers emit 1/0 for booleans; `required: 1` must not
+ silently flip a model-intended required field to optional."""
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": [
+ {"name": "amount", "type": "number", "required": 1},
+ {"name": "note", "type": "text", "required": 0},
+ ],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["fields"][0]["required"] is True
+ assert payload["fields"][1]["required"] is False
+
+ def test_field_placeholder_is_preserved(self, middleware):
+ payload = middleware._build_human_input_payload(
+ {
+ "question": "Details please",
+ "clarification_type": "missing_info",
+ "fields": [{"name": "note", "placeholder": "Optional remarks"}],
+ },
+ tool_call_id="call-abc",
+ request_id="clarification:call-abc",
+ )
+
+ assert payload["fields"][0]["placeholder"] == "Optional remarks"
+
+ def test_form_fallback_text_lists_fields(self, middleware):
+ result = middleware._format_clarification_message(
+ {
+ "question": "Please provide the expense details.",
+ "clarification_type": "missing_info",
+ "fields": self._fields(),
+ }
+ )
+
+ assert "1. Amount (required)" in result
+ assert "2. Category (required) — options: travel / meals" in result
+ assert "3. Receipts — options: A-1 / A-2 (multiple allowed)" in result
+ assert "4. note" in result
+ assert "Please reply with a value for each field." in result
+
+
+class TestClarificationToolSchema:
+ """The tool schema must expose the v2 form parameter (request-side only)."""
+
+ def test_tool_exposes_fields_argument(self):
+ from deerflow.tools.builtins.clarification_tool import ask_clarification_tool
+
+ schema = ask_clarification_tool.args
+ assert "fields" in schema
+ # The reply protocol stays v1 (text/option): no top-level multi_select
+ # mode — a standalone multi-select question is a one-field form.
+ assert "multi_select" not in schema
+
+ def test_fields_item_schema_is_typed(self):
+ """The provider-facing schema must expose the field item shape (typed
+ via ClarificationFormField), not an opaque object relying on the
+ docstring alone."""
+ from langchain_core.utils.function_calling import convert_to_openai_tool
+
+ from deerflow.tools.builtins.clarification_tool import ask_clarification_tool
+
+ parameters = convert_to_openai_tool(ask_clarification_tool)["function"]["parameters"]
+ items = parameters["properties"]["fields"]["anyOf"][0]["items"]
+
+ assert items["required"] == ["name"]
+ assert sorted(items["properties"].keys()) == ["label", "name", "options", "placeholder", "required", "type"]
+ assert items["properties"]["type"]["enum"] == ["text", "textarea", "number", "select", "multi_select", "checkbox", "date"]
+
+
class TestClarificationCommandIdempotency:
"""Clarification tool-call retries should not duplicate messages in state."""
diff --git a/backend/tests/test_human_input.py b/backend/tests/test_human_input.py
index 999e8cd5c..d8626cb63 100644
--- a/backend/tests/test_human_input.py
+++ b/backend/tests/test_human_input.py
@@ -22,3 +22,10 @@ def test_read_human_input_response_preserves_non_empty_value():
assert response is not None
assert response["value"] == " staging "
+
+
+def test_read_human_input_response_rejects_unknown_kind():
+ raw = _text_response("staging")
+ raw["response_kind"] = "unknown"
+
+ assert read_human_input_response({"human_input_response": raw}) is None
diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md
index 9906415b1..0b28fcf94 100644
--- a/frontend/AGENTS.md
+++ b/frontend/AGENTS.md
@@ -81,7 +81,7 @@ Auth UI note: the login page's "keep me signed in" option submits only `remember
`/goal` and `/compact` are built-in composer commands, not skill activations. `src/components/workspace/input-box.tsx` intercepts `/goal`, `/goal clear`, and `/goal ` before normal chat submission, calling Gateway `GET/PUT/DELETE /api/threads/{thread_id}/goal`. Setting `/goal ` also submits the condition text as the next user task so the agent starts running immediately; status and clear do not start a run. Goal and compact requests are tied to the current `threadId` with an `AbortController`, so switching threads or unmounting the composer aborts in-flight requests and stale responses cannot update the new thread's composer state. The chat pages render `GoalStatus` above the composer from `AgentThreadState.goal`, with local optimistic state until the next stream `values` update arrives. `/compact` calls `POST /api/threads/{thread_id}/compact` to summarize older active context while leaving the full visible chat history intact; it is skipped on new/empty threads and blocked server-side while a run is in flight. Thread rename uses the same serialized state-write route; the rename dialog stays open and surfaces the server error when an active run returns 409.
-Human input requests are a structured message protocol layered on normal chat history. The backend writes request payloads to `ToolMessage.artifact.human_input`, `src/core/messages/human-input.ts` owns the runtime validators/types, and `src/components/workspace/messages/human-input-card.tsx` renders the reusable card. `MessageList` owns answered/latest/pending state for visible cards, but derives answered responses from raw `thread.messages` because replies are hidden; pending cards clear when the hidden reply appears, when dispatch is dropped, or when a new `thread.error` reports an async stream failure. Page-level submit callbacks must send a normal human message and put `hide_from_ui: true` plus the response payload in the fourth `sendMessage(..., options)` argument as `options.additionalKwargs`; the third argument remains run context such as `{ agent_name }`. Composer entry points should disable normal bottom input while `hasOpenHumanInputRequest(...)` is true so users answer through the card and preserve response metadata.
+Human input requests are a structured message protocol layered on normal chat history. The backend writes request payloads to `ToolMessage.artifact.human_input`, `src/core/messages/human-input.ts` owns the runtime validators/types, and `src/components/workspace/messages/human-input-card.tsx` renders the reusable card. The protocol is versioned on the request side only: v1 covers `free_text` / `choice_with_other`, and v2 adds `form` (typed fields — text/textarea/number/select/multi_select/checkbox/date — with required-field validation in the card). Replies deliberately stay on the v1 response protocol: the form card submits a `response_kind: "text"` reply whose value is the human-readable summary plus one JSON block keyed by stable field names (`buildHumanInputFormSubmissionValue` — the readable part alone is ambiguous because labels/values may contain the separators), so the model can reconstruct the submitted mapping without a structured response kind. The validators reject unknown versions/modes (and field names colliding with JS `Object.prototype` members) so future protocol bumps degrade to the plain-text ToolMessage fallback rather than rendering a broken card. Form values are read through own-property access only (`readHumanInputFormValue`); select fields stay controlled from their empty-string placeholder state through selection; checkbox fields are native `` controls seeded to an explicit `false` (`buildInitialHumanInputFormValues`) so an untouched checkbox submits as "no" while a `required` checkbox keeps must-agree semantics (no HTML `required` attribute — native constraint validation would intercept the custom submit path), and form controls carry label/`htmlFor`, `aria-required` plus a visually-hidden localized "required" marker, and `aria-invalid`/error associations whose error node stays mounted while any field is still invalid. Legacy-fallback closure: `deriveHumanInputThreadState` treats a visible plain human message as answering the latest unanswered request opened before it (only the latest — nothing guarantees a single outstanding request across runs, and closing all would silently swallow older decisions; an older request left open simply becomes the active card again) — an old v1-only frontend degrades a v2 request to text and the user replies through the normal composer without response metadata, and without this rule an upgraded frontend would see that request as still open and lock the composer. `MessageList` owns answered/latest/pending state for visible cards, but derives answered responses from raw `thread.messages` because replies are hidden; pending cards clear when the hidden reply appears, when dispatch is dropped, or when a new `thread.error` reports an async stream failure. Page-level submit callbacks must send a normal human message and put `hide_from_ui: true` plus the response payload in the fourth `sendMessage(..., options)` argument as `options.additionalKwargs`; the third argument remains run context such as `{ agent_name }`. Composer entry points should disable normal bottom input while `hasOpenHumanInputRequest(...)` is true so users answer through the card and preserve response metadata.
Tool-calling AI messages can contain user-visible text as well as `tool_calls`. `core/messages/utils.ts` keeps these turns in an `assistant:processing` group, and `components/workspace/messages/message-group.tsx` must render the visible text as a processing step instead of treating the message as only tool metadata. This preserves provider text such as error explanations or "trying another approach" notes during tool-heavy runs.
diff --git a/frontend/src/components/workspace/messages/human-input-card.tsx b/frontend/src/components/workspace/messages/human-input-card.tsx
index e4d268752..aaad8e264 100644
--- a/frontend/src/components/workspace/messages/human-input-card.tsx
+++ b/frontend/src/components/workspace/messages/human-input-card.tsx
@@ -2,6 +2,7 @@
import {
CheckCircle2Icon,
+ CheckIcon,
Loader2Icon,
MessageCircleQuestionMarkIcon,
} from "lucide-react";
@@ -9,11 +10,25 @@ import { useId, useState, type KeyboardEvent } from "react";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
import { Textarea } from "@/components/ui/textarea";
import { useI18n } from "@/core/i18n/hooks";
import {
+ buildHumanInputFormSubmissionValue,
+ buildHumanInputFormSummary,
+ buildInitialHumanInputFormValues,
createHumanInputOptionResponse,
createHumanInputTextResponse,
+ readHumanInputFormValue,
+ type HumanInputField,
+ type HumanInputFormValue,
type HumanInputOption,
type HumanInputRequest,
type HumanInputResponse,
@@ -36,6 +51,169 @@ export function shouldSubmitHumanInputTextOnKeyDown(
);
}
+function isEmptyFieldValue(value: HumanInputFormValue | undefined) {
+ if (value === undefined) {
+ return true;
+ }
+ if (typeof value === "string") {
+ return value.trim().length === 0;
+ }
+ if (Array.isArray(value)) {
+ return value.length === 0;
+ }
+ return value === false;
+}
+
+export function findMissingRequiredFields(
+ fields: HumanInputField[],
+ values: Record,
+) {
+ // Own-property reads only: field names like "toString" must never resolve
+ // to inherited Object.prototype members and satisfy required validation.
+ return fields.filter(
+ (field) =>
+ field.required &&
+ isEmptyFieldValue(readHumanInputFormValue(values, field.name)),
+ );
+}
+
+function FormFieldInput({
+ field,
+ value,
+ disabled,
+ selectPlaceholder,
+ controlId,
+ labelId,
+ invalid,
+ errorId,
+ onChange,
+}: {
+ field: HumanInputField;
+ value: HumanInputFormValue | undefined;
+ disabled: boolean;
+ selectPlaceholder: string;
+ controlId: string;
+ labelId: string;
+ invalid: boolean;
+ errorId?: string;
+ onChange: (value: HumanInputFormValue) => void;
+}) {
+ const stringValue = typeof value === "string" ? value : "";
+ const ariaProps = {
+ "aria-required": field.required || undefined,
+ "aria-invalid": invalid || undefined,
+ "aria-describedby": invalid ? errorId : undefined,
+ };
+
+ if (field.type === "textarea") {
+ return (
+