Merge branch 'main' into rayhpeng/hexagonal-feedback-slice

Resolves the alembic head fork: main introduced 0010_run_cancel_request
on 0009 while this branch carried 0010_feedback_tags -> 0011_..._drop.
The unmerged branch revisions are renumbered and rechained after main's
published one (0010_run_cancel_request -> 0011_feedback_tags ->
0012_feedback_drop_message_id), and every test head pin moves to 0012.

.env.example

@@ -17,6 +17,7 @@ INFOQUEST_API_KEY=your-infoquest-api-key
 # FIRECRAWL_API_KEY=your-firecrawl-api-key
 # VOLCENGINE_API_KEY=your-volcengine-api-key
 # OPENAI_API_KEY=your-openai-api-key
+# OPENVIKING_API_KEY=your-openviking-trusted-root-api-key
 # GEMINI_API_KEY=your-gemini-api-key
 # DEEPSEEK_API_KEY=your-deepseek-api-key
 # NOVITA_API_KEY=your-novita-api-key  # OpenAI-compatible, see https://novita.ai

.github/workflows/lark-cli-images.yaml

@@ -0,0 +1,102 @@
+name: Publish lark-cli Images
+
+# Publishes the two Lark sandbox runtime images (Pattern A init container and
+# Pattern B broker sidecar). These track the upstream `larksuite/cli` release,
+# not the DeerFlow `v*` release, so they publish on their own trigger and are
+# tagged by the lark-cli version rather than being wired into container.yaml.
+on:
+  workflow_dispatch:
+    inputs:
+      lark_cli_version:
+        description: "larksuite/cli release to build (e.g. v1.0.65)"
+        required: true
+        default: v1.0.65
+  push:
+    tags:
+      - "lark-cli-v*"
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  build-images:
+    # Only publish from the upstream repo; a fork dispatch/tag skips the job.
+    if: github.repository == 'bytedance/deer-flow'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Pattern A init image: relative `COPY build-runtime.sh entrypoint.sh`,
+          # so the build context is its own dir.
+          - component: lark-cli-init
+            context: docker/lark-cli-init
+            file: docker/lark-cli-init/Dockerfile
+          # Pattern B broker image: copies the shared build-runtime.sh and the
+          # harness module, so the build context is the repo root.
+          - component: lark-cli-broker
+            context: .
+            file: docker/lark-cli-broker/Dockerfile
+    env:
+      IMAGE_NAME: ${{ github.repository }}-${{ matrix.component }}
+    steps:
+      - name: Resolve lark-cli version
+        id: version
+        # workflow_dispatch supplies it as an input; a `lark-cli-v*` tag encodes
+        # it after the prefix (lark-cli-v1.0.65 -> v1.0.65). The input is passed
+        # via env (not interpolated into the script) to avoid shell expression
+        # injection if the repo gate on dispatch ever widens.
+        env:
+          LARK_CLI_VERSION_INPUT: ${{ inputs.lark_cli_version }}
+        run: |
+          if [ -n "$LARK_CLI_VERSION_INPUT" ]; then
+            version="$LARK_CLI_VERSION_INPUT"
+          else
+            version="${GITHUB_REF_NAME#lark-cli-}"
+          fi
+          echo "value=${version}" >> "$GITHUB_OUTPUT"
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 #v6.0.3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 #v3.6.0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 #v3.11.1
+      - name: Log in to the Container registry
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #v3.4.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 #v5.7.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          # Tag by the lark-cli version (not the DeerFlow release). No `latest`.
+          tags: |
+            type=raw,value=${{ steps.version.outputs.value }}
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 #v6.18.0
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.file }}
+          push: true
+          # The sidecar/init image ships a real arch-dispatched lark-cli binary,
+          # so publish both arches the sandbox may run on.
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            LARK_CLI_VERSION=${{ steps.version.outputs.value }}
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be #v2.4.0
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

AGENTS.md

@@ -113,6 +113,8 @@ cd frontend && pnpm test      # Unit tests
 Rule of thumb: **root `make` = the full application**; **`backend/Makefile` and `frontend/`
 (`pnpm`) = per-module work.**
 
+Host-side pnpm consumers, including the root/frontend Makefiles and local diagnostic scripts, must run through `scripts/pnpm.py`. The runner preserves direct `pnpm`/`pnpm.cmd` priority, falls back to `corepack pnpm`, and is invoked from `frontend/` so Corepack honors the package-manager version pinned by that project.
+
 ## Where to Go Next
 
 - Backend work → **[backend/AGENTS.md](backend/AGENTS.md)**

CHANGELOG.md

@@ -62,6 +62,13 @@ This section accumulates work toward the **2.1.0** milestone
   intentional -- silent empties are worse than a loud startup error. Fix: switch
   to `mode: middleware` or override `search()` (and set `supports_search=True`).
   ([#4324])
+- **config:** `database.checkpoint_delta_snapshot_frequency` moved to
+  `database.checkpoint_delta.snapshot_frequency` and its default changed from
+  `1000` to `10`. A legacy top-level value is still honored with a deprecation
+  warning and mapped onto the nested key (an explicitly set nested key wins).
+  Deployments that relied on the old default now snapshot 100x more often in
+  delta mode -- set `database.checkpoint_delta.snapshot_frequency: 1000`
+  explicitly to keep the previous cadence. ([#4516])
 
 ### Added
 
@@ -1343,3 +1350,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag.
 [#4468]: https://github.com/bytedance/deer-flow/pull/4468
 [#4469]: https://github.com/bytedance/deer-flow/pull/4469
 [#4471]: https://github.com/bytedance/deer-flow/pull/4471
+[#4516]: https://github.com/bytedance/deer-flow/pull/4516

Makefile

@@ -16,6 +16,8 @@ else
     RUN_WITH_GIT_BASH =
 endif
 
+FRONTEND_PNPM = $(PYTHON) ../scripts/pnpm.py
+
 help:
 	@echo "DeerFlow Development Commands:"
 	@echo "  make setup           - Interactive setup wizard (recommended for new users)"
@@ -80,7 +82,7 @@ install:
 	@echo "Installing backend dependencies..."
 	@cd backend && uv sync
 	@echo "Installing frontend dependencies..."
-	@cd frontend && pnpm install
+	@cd frontend && $(FRONTEND_PNPM) install
 	@echo "Installing pre-commit hooks..."
 	@uv tool install pre-commit
 	@pre-commit install --overwrite

README.md

@@ -181,7 +181,7 @@ That prompt is intended for coding agents. It tells the agent to clone the repo
 
    To route OpenAI models through `/v1/responses`, keep using `langchain_openai:ChatOpenAI` and set `use_responses_api: true` with `output_version: responses/v1`.
 
-   For vLLM 0.19.0, use `deerflow.models.vllm_provider:VllmChatModel`. For Qwen-style reasoning models, DeerFlow toggles reasoning with `extra_body.chat_template_kwargs.enable_thinking` and preserves vLLM's non-standard `reasoning` field across multi-turn tool-call conversations. Legacy `thinking` configs are normalized automatically for backward compatibility. Reasoning models may also require the server to be started with `--reasoning-parser ...`. If your local vLLM deployment accepts any non-empty API key, you can still set `VLLM_API_KEY` to a placeholder value.
+   For vLLM 0.19.0, use `deerflow.models.vllm_provider:VllmChatModel`. For Qwen-style reasoning models, DeerFlow toggles reasoning with `extra_body.chat_template_kwargs.enable_thinking` and preserves vLLM's non-standard `reasoning` field across multi-turn tool-call conversations. Legacy `thinking` configs are normalized automatically for backward compatibility. If the endpoint reports a cumulative usage snapshot on every streaming chunk, set `cumulative_stream_usage: true` so DeerFlow converts those snapshots into per-chunk deltas; the option is disabled by default and leaves usage unchanged when a stable completion id is unavailable. Reasoning models may also require the server to be started with `--reasoning-parser ...`. If your local vLLM deployment accepts any non-empty API key, you can still set `VLLM_API_KEY` to a placeholder value.
 
    CLI-backed provider examples:
 
@@ -256,9 +256,9 @@ honor environment proxy settings.
 
 Backend processes automatically pick up `config.yaml` changes on the next config access, so model metadata updates do not require a manual restart during development.
 The checkpoint storage settings `database.checkpoint_channel_mode` and
-`database.checkpoint_delta_snapshot_frequency` are exceptions: both are frozen
-when the process first builds an agent (including through `DeerFlowClient`) and
-require a process restart to change safely.
+`database.checkpoint_delta.snapshot_frequency` (default `10`) are exceptions:
+both are frozen when the process first builds an agent (including through
+`DeerFlowClient`) and require a process restart to change safely.
 
 > [!TIP]
 > On Linux, if Docker-based commands fail with `permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock`, add your user to the `docker` group and re-login before retrying. See [CONTRIBUTING.md](CONTRIBUTING.md#linux-docker-daemon-permission-denied) for the full fix.
@@ -286,6 +286,8 @@ DeerFlow still uses `Forwarded` / `X-Forwarded-*` headers to recover the browser
 > [!IMPORTANT]
 > The Gateway still owns active run tasks in process, so production defaults to a single Gateway worker (`GATEWAY_WORKERS=1`). Multi-worker deployments require Postgres, the Redis stream bridge (`stream_bridge.type: redis`), `run_ownership.heartbeat_enabled: true`, and `run_events.backend: db`; process-local memory/JSONL event stores cannot enforce singleton delivery receipts across workers. The bridge shares SSE delivery and bounded `Last-Event-ID` replay across workers. When a valid reconnect cursor has been trimmed, or a subscriber that already established an empty-stream wait falls behind before its first delivery, Memory and Redis emit a machine-readable SSE `gap` event instead of silently returning a partial replay; the Web UI reloads durable thread/event state and resumes from the retained tail. Lease reconciliation marks runs from dead workers as errors, persists their delivery receipts, publishes the terminal stream marker, schedules retained-stream cleanup, and updates the affected thread status. SSE and `/wait` consumers also refresh durable status on heartbeats as a fallback if terminal publication fails. Malformed Redis reconnect IDs live-tail new events instead of replaying the retained buffer, and the rolling retained-buffer TTL (`stream_ttl_seconds`) remains a cleanup safety net rather than a run timeout. IM channel state and other process-local services still need their own multi-worker coordination.
 >
+> Run cancellation may land on any Gateway worker. A non-owning worker now persists the interrupt or rollback request for the live owner, which observes it during lease renewal and performs the normal cancellation flow; load-balancer routing alone no longer produces a 409. The first accepted action wins even if a retry lands on the owner, and accepted cancellation competes atomically with owner completion. Dead owners still follow lease takeover and orphan recovery. Cancellation latency is therefore bounded by the lease heartbeat interval.
+>
 > With lease heartbeat enabled, a transient RunStore renewal error is retried only until the last confirmed lease expires; the stale worker then cancels local execution and suppresses checkpoint, completion-hook, delivery-receipt, and thread-status finalization. A remote tool side effect already in flight may still be outside local cancellation.
 >
 > Reconciliation uses an atomic takeover claim that re-checks the lease after candidate selection, so a successful owner renewal wins over orphan recovery and only one reconciler can report a run as recovered. When multiple Gateway workers share the Docker/AIO or E2B sandbox backend, also configure `sandbox.ownership.type: redis`; E2B uses the leases during background startup and periodic reconciliation so duplicate/orphan cleanup cannot terminate a live peer's sandbox.
@@ -304,6 +306,8 @@ On Windows, run the local development flow from Git Bash. Native `cmd.exe` and P
    make check  # Verifies Node.js 22+, pnpm, uv, nginx
    ```
 
+   The local `make check`, `make install`, `make dev`, and `make start` entry points use a direct `pnpm`/`pnpm.cmd` executable when available and otherwise fall back to `corepack pnpm`. Corepack runs from `frontend/`, so it honors the `packageManager` version pinned in `frontend/package.json`; enabling a global pnpm shim is not required.
+
 2. **Install dependencies**:
    ```bash
    make install  # Install backend + frontend dependencies + pre-commit hooks
@@ -345,6 +349,17 @@ DeerFlow runs the agent runtime inside the Gateway API. Development mode enables
 
 Gateway owns `/api/langgraph/*` and translates those public LangGraph-compatible paths to its native `/api/*` routers behind nginx.
 
+For workflows that invoke `backend/langgraph.json` through LangGraph Studio or
+a direct LangGraph Server, DeerFlow consumes the authenticated identity
+published by that runtime and uses it for custom-agent configuration/SOUL, user
+skills and skill policy, uploads, thread data, and memory reads/writes. This
+keeps authenticated runs out of the shared `default` filesystem bucket, and the
+server-owned identity takes precedence over ordinary client-supplied `user_id`
+values. External identities such as email addresses are mapped to stable,
+collision-resistant directory-safe user IDs before accessing DeerFlow storage.
+The default DeerFlow service topology remains the Gateway-embedded runtime
+described above.
+
 Gateway runs automatically enforce native delivery for artifacts created or modified under `/mnt/user-data/outputs`: `present_files` must present at least one output produced by the current run, and the terminal `run.delivery` receipt must be durably recorded. Runs that do not produce output artifacts keep ordinary conversational behavior.
 
 DeerFlow's built-in custom events are available through both LangGraph streaming interfaces: native clients can continue subscribing to `stream_mode="custom"`, while callback-based integrations can consume the same payloads as `on_custom_event` records from `astream_events(version="v2")`. The callback event name matches the payload's `type` field.
@@ -385,6 +400,12 @@ For stdio MCP servers, per-tool call timeouts can be configured with `tool_call_
 MCP routing hints can also prefer a specific MCP tool for matching requests without forbidding other tools. When `tool_search` defers MCP schemas, matching routing metadata can auto-promote up to `tool_search.auto_promote_top_k` deferred schemas before the model call.
 See the [MCP Server Guide](backend/docs/MCP_SERVER.md) for detailed instructions.
 
+Security: pass per-request MCP credentials only through `config.context.secrets`;
+credentials must never be placed in either run metadata surface
+(`metadata.auth_token` or `config.metadata.auth_token`). See [MCP credential migration and cleanup](backend/docs/MCP_SERVER.md#migrating-legacy-mcp-credentials)
+for the supported interceptor flow and the required rotation and retained-copy
+cleanup when migrating from legacy metadata credentials.
+
 #### IM Channels
 
 DeerFlow supports receiving tasks from messaging apps. Channels auto-start when configured — no public IP required for any of them.
@@ -686,7 +707,7 @@ A skill directory is a package boundary: once DeerFlow finds its `SKILL.md`, nes
 
 Users can explicitly activate an enabled skill for a single turn by starting the request with `/skill-name`, for example `/data-analysis analyze uploads/foo.csv`. DeerFlow loads that skill's `SKILL.md` as hidden current-turn context while leaving the base prompt limited to skill metadata. Slash activation respects disabled skills, custom-agent skill whitelists, and existing channel commands such as `/new` and `/help`.
 
-An enabled skill's `allowed-tools` policy applies only after that skill is explicitly slash-activated or captured in the thread's active skill context after a `read_file` load. Merely enabling, advertising, or listing a skill in a custom agent's `skills` allowlist does not reduce the lead agent's normal toolset. During a slash-activated run, that explicit skill's policy is authoritative: reading another `SKILL.md` may provide instructions but cannot widen the slash skill's tools. Without slash activation, policies from skills actually loaded into active context retain their union semantics. Once active, the policy filters both model-visible tool schemas and tool execution. Framework discovery tools (`tool_search` and `describe_skill`) remain available so an allowed deferred tool or installed skill can still be discovered, but discovery and promotion never grant permission to execute a business tool omitted from `allowed-tools`. `task` is not framework-exempt; a restrictive skill must list it explicitly to delegate to a subagent. Per-step policy decisions are internal runtime context and are removed from observable or persisted context copies. Registry failures and an active set with no remaining valid skill fail closed to framework-safe tools; individual stale paths are ignored only when another valid active skill remains. This is best-effort behavioral scoping, not a hard security boundary: loading skill instructions through another tool is not captured, and active-skill entries can be evicted from bounded context.
+An enabled skill's `allowed-tools` policy applies only after that skill is explicitly slash-activated or captured in the agent's active skill context after a `read_file` load. Merely enabling, advertising, or listing a skill in a custom agent or subagent `skills` allowlist does not reduce that agent's normal toolset; subagents use the same progressive discovery and activation policy as the lead agent. During a slash-activated run, that explicit skill's policy is authoritative: reading another `SKILL.md` may provide instructions but cannot widen the slash skill's tools. Without slash activation, policies from skills actually loaded into active context retain their union semantics. Once active, the policy filters both model-visible tool schemas and tool execution. Framework discovery tools (`tool_search` and `describe_skill`) remain available so an allowed deferred tool or installed skill can still be discovered, but discovery and promotion never grant permission to execute a business tool omitted from `allowed-tools`. `task` is not framework-exempt; a restrictive skill must list it explicitly to delegate to a subagent. Per-step policy decisions are internal runtime context and are removed from observable or persisted context copies. Registry failures and an active set with no remaining valid skill fail closed to framework-safe tools; individual stale paths are ignored only when another valid active skill remains. This is best-effort behavioral scoping, not a hard security boundary: loading skill instructions through another tool is not captured, and active-skill entries can be evicted from bounded context.
 
 When you install `.skill` archives through the Gateway, DeerFlow accepts standard optional frontmatter metadata such as `version`, `author`, and `compatibility` instead of rejecting otherwise valid external skills.
 
@@ -775,6 +796,8 @@ Gateway-generated follow-up suggestions now normalize both plain-string model ou
 
 The Web UI composer can polish draft input before sending. The rewrite runs as a short Gateway LLM request using the `input_polish` model configuration, keeps slash skill prefixes such as `/data-analysis`, and only replaces the local draft after the user clicks the polish button; it does not create a thread run or persist a message.
 
+When the agent asks for clarification, the Web UI shows the structured response card but keeps the normal composer available. Users can complete the card or send a free-form chat message to bypass it; that message closes the latest pending clarification and becomes the agent's next input.
+
 Unsent Web UI composer drafts survive page reloads and switching between conversations within the same browser tab. Drafts are isolated by user, agent, and conversation, include a selected slash skill when present, and are cleared once a send is accepted. Attachments and quoted conversation context are intentionally not persisted.
 
 The Web UI composer also supports browser-based voice dictation when the browser exposes the Web Speech API. The microphone button transcribes speech into the local draft only; DeerFlow receives only the transcribed text, while audio handling is delegated to the browser or operating system speech-recognition service according to that environment's policy. Users can review or edit the text before sending.
@@ -786,7 +809,7 @@ Interrupted first-turn runs still persist a fallback conversation title, so stop
 
 Streaming Markdown responses animate only newly arrived words; text that is already visible is not faded out and replayed when the next chunk extends the same block.
 
-In the Web UI, completed assistant turns can be branched into a new main conversation. The new thread starts from that turn's checkpoint and keeps the preceding replay checkpoint, so the branched response can be regenerated immediately. Regenerating the latest response preserves the thread's current title, including a title you renamed manually after the original response. Legacy or imported histories without checkpoint parent links use a bounded chronological fallback; if no earlier replay checkpoint exists, branching still succeeds with the legacy single-checkpoint shape, while regeneration remains unavailable for that inherited response. Existing single-checkpoint branches are left unchanged rather than attempting an unsafe checkpoint copy. Because workspace files are not checkpointed, the branch only receives a best-effort copy of the current workspace when you branch from the latest turn; branching from an older turn keeps just the restored message history so the branch never inherits files that were created in a later part of the conversation.
+In the Web UI, completed assistant turns can be branched into a new main conversation. The new thread starts from that turn's checkpoint and keeps the preceding replay checkpoint, so the branched response can be regenerated immediately. The latest response can also be regenerated after an interruption, even when its streamed partial text never reached a checkpoint. Regenerating the latest response preserves the thread's current title, including a title you renamed manually after the original response. Legacy or imported histories without checkpoint parent links use a bounded chronological fallback; if no earlier replay checkpoint exists, branching still succeeds with the legacy single-checkpoint shape, while regeneration remains unavailable for that inherited response. Existing single-checkpoint branches are left unchanged rather than attempting an unsafe checkpoint copy. Because workspace files are not checkpointed, the branch only receives a best-effort copy of the current workspace when you branch from the latest turn; branching from an older turn keeps just the restored message history so the branch never inherits files that were created in a later part of the conversation.
 
 The Web UI reports completed task time once per run. This is total wall-clock time—including model reasoning, tool calls, and waiting—not a per-step or model-only thinking duration. Reasoning content remains available through its own separate disclosure.
 
@@ -901,6 +924,14 @@ After each run, DeerFlow records a workspace change summary for the run-owned `w
 
 With `AioSandboxProvider`, shell execution runs inside isolated containers. With `LocalSandboxProvider`, file tools still map to per-thread directories on the host, but host `bash` is disabled by default because it is not a secure isolation boundary. Re-enable host bash only for fully trusted local workflows. Host bash commands have a wall-clock timeout, and long-lived processes should be started in the background with output redirected to a workspace log.
 
+`AioSandboxProvider` normally detects thread-data mounts from its backend: local
+containers use the mounted gateway directories, while remote/provisioner
+sandboxes receive uploaded files through explicit synchronization. Deployments
+where both sides are guaranteed to share the same thread user-data directories
+can set `sandbox.thread_data_mounts: true` to skip that per-upload sandbox
+acquire and sync. Leave the field unset for automatic detection; setting it
+incorrectly can make uploaded files unavailable inside the sandbox.
+
 This is the difference between a chatbot with tool access and an agent with an actual execution environment.
 
 ```
@@ -939,12 +970,31 @@ Then uncomment the `group: browser` tool entries in `config.yaml` (`browser_navi
 
 Most agents forget everything the moment a conversation ends. DeerFlow remembers.
 
+DeerFlow also includes an optional `openviking` memory backend. It connects to
+an independent OpenViking server over HTTP, submits completed turns through
+OpenViking Sessions, and recalls remote memories for prompt injection while
+leaving DeerMem as the default. The initial integration supports
+`memory.mode: middleware`. Bounded submitted-message watermarks cover long and
+compacted histories and prevent a failed Session commit from duplicating
+already accepted messages on retry; the shared HTTP client also has explicit
+connection limits and jittered retries. See
+[OpenViking memory backend](docs/OPENVIKING.md) for configuration and Docker
+startup.
+
 Across sessions, DeerFlow builds a persistent memory of your profile, preferences, and accumulated knowledge. The more you use it, the better it knows you — your writing style, your technical stack, your recurring workflows. Memory is stored locally and stays under your control.
 
+DeerMem remains the default local backend. An opt-in `mem0` backend is also
+available for the hosted mem0 Platform API or API-compatible self-hosted
+servers. Its token-bearing `base_url` must use HTTPS by default; plaintext HTTP
+requires an explicit local-development opt-in. See the
+[mem0 backend guide](backend/packages/harness/deerflow/agents/memory/backends/mem0/README.md).
+
 Memory updates now skip duplicate fact entries at apply time, so repeated preferences and context do not accumulate endlessly across sessions.
 
 File-backed memory now separates global user context from agent facts. Each user has one `memory.json` containing only the project-independent `user` and `history` summaries; every fact is a canonical Markdown file below `agents/{agent_name}/facts/`. Existing lead-agent middleware, API, Settings, import/export, and embedded-client calls that omit `agent_name` resolve inside DeerMem to the reserved `__default__` bucket. That bucket is outside the valid custom-agent name grammar, so a real custom agent named `lead-agent` has a separate fact repository and deleting a custom agent cannot delete a memory-only directory without `config.yaml`. Public agent identifiers are case-insensitive and canonicalized to lowercase. Runtime/API readers still receive a compatibility `facts` array for the selected/default agent, so the frontend does not read agent facts from `memory.json`; structured Markdown `source` metadata is projected to the historical string field at the MemoryManager boundary. An unscoped Clear All first migrates facts from unread legacy per-agent JSON without adopting its soon-to-be-cleared summaries, then removes shared summaries and facts from every agent bucket while preserving agent configuration files, so a later read cannot resurrect skipped legacy facts; an explicitly agent-scoped clear removes only that agent's facts. On first normal read, old facts embedded in the user JSON are migrated automatically to `__default__`; facts written to the earlier implicit `lead-agent` bucket are also moved when that directory is not a real custom agent. Migration and normal writes notify the configured retrieval adapter only after durable storage locks are released. DeerMem uses a scope-aware SQLite FTS5/BM25 adapter by default, stores only rebuildable derived index data under `.retrieval/`, and rebuilds it in the background during Gateway startup or lazily on the first scoped search. A corrupt derived index is recreated automatically. Set `memory.backend_config.retrieval_adapter` to an empty string to disable it and use the local substring fallback. Chinese tokenization is optional; install the backend `memory-zh` extra (`uv sync --extra memory-zh`) for jieba-assisted sub-phrase search. Journaled writes, a shared user lock, and optimistic user-memory revisions prevent silent lost updates.
 
+Memory injection follows the configured operation mode. In `middleware` mode, DeerMem injects the user-global summaries and the selected agent's facts. In `tool` mode, the automatic `<memory>` block contains only the global `user` and `history` summaries; agent facts are retrieved explicitly through `memory_search`, avoiding duplicate automatic and tool-returned fact context. Setting `memory.injection_enabled: false` still disables the entire block in either mode.
+
 Single-fact repository operations are genuinely incremental: an upsert/delete reads, journals, writes, and re-indexes only the addressed fact files, and returns an explicit incomplete delta rather than a cache-dependent fake full document. Summary change sets merge the supplied `user`/`history` child keys over the persisted sections so a partial update cannot erase omitted siblings; full imports normalize both sections to the complete compatibility schema before applying replacement values. Manager/API compatibility methods materialize a fresh full document only when their public response contract requires one. Fact-level point operations use separate expected user-memory and fact revisions and may explicitly rebase when every addressed fact precondition still holds. Snapshot-derived operations such as scoped clear, capped create, consolidation, and trimming never replay stale delete/trim sets: a manifest conflict reloads the complete document and recomputes the operation, with a bounded retry. Fact paths use the first two hexadecimal characters of `SHA-256(fact_id)` so generated `fact_*` IDs distribute across shards. The cache token combines the shared JSON's nanosecond mtime, size, and persisted revision; this prevents coarse-mtime same-size writes from returning stale data without scanning fact files. Direct out-of-band Markdown edits require an explicit reload. Storage-specific conflicts and corruption are translated at the MemoryManager boundary; the Gateway returns conflict as HTTP 409 and a stable, non-sensitive corruption error as HTTP 500. Full-document `save()` remains a compatibility API and computes a diff before writing; malformed or missing `facts` can no longer silently erase an agent's Markdown files. Legacy migration preserves non-empty `user`/`history` before deleting an agent `memory.json`; conflicting summaries keep the legacy file and fail loudly instead of choosing a winner.
 
 Legacy facts in `memory.json` migrate automatically into the reserved `__default__` Markdown bucket on the user's first normal memory read. Operators who prefer to audit or complete the migration before serving traffic can run the optional idempotent CLI from `backend/`:

RELEASING.md

@@ -111,6 +111,24 @@ Artifacts (under the running repo's owner, where `<date>` is `YYYYMMDD`):
 The chart version is patched in-workflow only - `Chart.yaml` and `values.yaml`
 in the repo are never modified.
 
+## lark-cli sandbox images
+
+The two optional Lark sandbox runtime images — `lark-cli-init` (Pattern A) and
+`lark-cli-broker` (Pattern B) — are **not** part of the `v*` release. They track
+the upstream `larksuite/cli` version, so they publish independently via
+`.github/workflows/lark-cli-images.yaml`:
+
+- Trigger with `workflow_dispatch` (a `lark_cli_version` input, e.g. `v1.0.65`)
+  or by pushing a `lark-cli-v*` tag (the version is read from after the prefix).
+- Builds multi-arch (`linux/amd64,linux/arm64`) and pushes
+  `ghcr.io/<owner>/deer-flow-{lark-cli-init,lark-cli-broker}:<lark-cli-version>`.
+- Gated on `github.repository == 'bytedance/deer-flow'`; not tied to the
+  `verify-versions` gate (its version is the lark-cli release, not the DeerFlow
+  release), and it never touches `latest`.
+
+Both features stay opt-in: the provisioner ignores them until
+`LARK_CLI_INIT_IMAGE` / `LARK_CLI_BROKER_IMAGE` point at a published tag.
+
 ## Version gate
 
 Both publishing workflows call `.github/workflows/verify-versions.yml` as their

backend/AGENTS.md

@@ -229,7 +229,10 @@ Blocking-IO runtime gate (`tests/blocking_io/`):
   offloading the uploads-directory scan off the event loop);
   `test_uploads_router.py` (locks Gateway upload/list/delete endpoints
   offloading upload directory creation, staged writes, chmod/cleanup,
-  directory scans/deletes, and remote sandbox sync off the event loop); and
+  directory scans/deletes, and remote sandbox sync off the event loop);
+  `test_openviking_memory_backend.py` (locks the OpenViking backend's async
+  add/context/search entrypoints offloading synchronous HTTP and watermark
+  filesystem IO); and
   `test_workspace_changes_recorder.py` (locks the offload around the snapshot
   text cache lifecycle — roots resolution, `mkdtemp`, and the `shutil.rmtree`
   on both the capture-failure branch and `record_workspace_changes`' `finally`).
@@ -245,6 +248,23 @@ Blocking-IO runtime gate (`tests/blocking_io/`):
 Boundary check (harness → app import firewall):
 - `tests/test_harness_boundary.py` — ensures `packages/harness/deerflow/` never imports from `app.*`
 
+Memory backend async boundary:
+- `MemoryMiddleware.aafter_agent` calls `MemoryManager.aadd`; network-backed
+  managers must override their `a*` methods to offload or use native async I/O.
+- The mem0 backend requires an HTTPS `base_url` by default because requests
+  carry an API token. Plain HTTP requires the explicit
+  `backend_config.allow_insecure_http: true` local-development opt-in.
+- Gateway memory routes offload the synchronous management contract with
+  `asyncio.to_thread`, so backend file or HTTP I/O does not run on the ASGI
+  event loop. Gateway startup and shutdown also resolve the manager off-loop,
+  because a backend's `from_config` may perform a fail-fast connectivity check.
+- A backend may set `requires_passive_writes_in_tool_mode = True` when tool-mode
+  search is supported but durable writes still depend on conversation-level
+  extraction. Such backends receive memory tools and retain `MemoryMiddleware`.
+- Prompt recall rethrows `MemoryManagerError` only when backend config declares
+  `failure_policy.read: fail_closed`; other recall errors preserve the existing
+  log-and-empty-context behavior.
+
 CI runs these regression tests for every pull request via [.github/workflows/backend-unit-tests.yml](../.github/workflows/backend-unit-tests.yml).
 
 Agentic browser sessions are process-local. The Gateway startup safety gate rejects
@@ -327,8 +347,8 @@ Lead-agent middlewares are assembled in strict order across three functions: the
 1. **InputSanitizationMiddleware** - First, so it is the outermost `wrap_model_call` wrapper; every inner middleware (including LLM retries) sees sanitized messages. `additional_kwargs.original_user_content` is server-owned provenance: Gateway strips caller-supplied values for non-internal run requests, trusted IM calls may carry the string they captured before adding transport/file context, and the middleware replaces any non-string value before wrapping. Uploads and sanitization retain first-writer-wins only for validated strings.
 2. **ToolOutputBudgetMiddleware** - Caps tool output size (per app config) before it re-enters the model context
 3. **ToolResultSanitizationMiddleware** - Neutralizes framework/injection tags (e.g. `<system-reminder>`) and boundary markers in *remote-content* tool results (`web_fetch`/`web_search`/`image_search`/`web_capture`) so attacker-controlled fetched pages cannot forge trusted framework context. Mirrors `InputSanitizationMiddleware`'s user-input guardrail for the other untrusted-content entry point; sits inner of `ToolOutputBudgetMiddleware` (neutralizes the raw output, then the budget truncates). Local tool output (bash/read_file) is left untouched. Scope is a name-based allowlist, so MCP remote-content tools registered under other names (e.g. `fetch_url`) are not yet covered — a metadata-tagging follow-up is tracked in the middleware source
-4. **ThreadDataMiddleware** - Creates per-thread directories under the user's isolation scope (`backend/.deer-flow/users/{user_id}/threads/{thread_id}/user-data/{workspace,uploads,outputs}`); resolves `user_id` via `get_effective_user_id()` (falls back to `"default"` in no-auth mode)
-5. **UploadsMiddleware** - Tracks and injects newly uploaded files into conversation (lead agent only)
+4. **ThreadDataMiddleware** - Creates per-thread directories under the user's isolation scope (`backend/.deer-flow/users/{user_id}/threads/{thread_id}/user-data/{workspace,uploads,outputs}`); resolves identity via `resolve_runtime_user_id(runtime)`, including Gateway runtime context and standalone LangGraph Server auth, then falls back to the request ContextVar / `"default"`
+5. **UploadsMiddleware** - Tracks and injects newly uploaded files into conversation (lead agent only); upload existence checks use the same runtime-resolved user bucket as thread-data creation
 6. **SandboxMiddleware** - Acquires sandbox, stores `sandbox_id` in state
 7. **DanglingToolCallMiddleware** - Injects placeholder ToolMessages for AIMessage tool_calls that lack responses (e.g., user interruption), preserving raw provider tool-call payloads in `additional_kwargs["tool_calls"]`; malformed tool-call names and arguments are sanitized in the model-bound request so strict OpenAI-compatible providers do not reject the next request
 8. **LLMErrorHandlingMiddleware** - Normalizes provider/model invocation failures into recoverable assistant-facing errors before later stages run
@@ -354,7 +374,7 @@ Before changing a later authorization phase, read the [authorization RFC](../doc
 19. **TodoListMiddleware** - *(optional, if `is_plan_mode`)* Task tracking with the `write_todos` tool
 20. **TokenUsageMiddleware** - *(optional, if `token_usage.enabled`)* Records token usage metrics; subagent usage is merged back into the dispatching AIMessage by message position
 21. **TitleMiddleware** - Auto-generates the thread title after the first complete exchange and normalizes structured message content before prompting the title model. If a first-turn run is interrupted before this middleware can write a title, `runtime/runs/worker.py` keeps the run in a finalizing state, persists a local fallback title from the latest checkpoint or original run input, and then syncs it to `threads_meta.display_name`. Replacement runs admitted by `multitask_strategy="interrupt"` / `"rollback"` wait for older same-thread finalization before entering the graph; the interrupted run only skips the fallback title write once a later run has started and may have advanced the checkpoint.
-22. **MemoryMiddleware** - Queues conversations for async memory update (filters to user + final AI responses)
+22. **MemoryMiddleware** - Queues conversations for async memory update (filters to user + final AI responses); captures the runtime-resolved user so standalone LangGraph Server reads and writes stay in the same bucket
 23. **ViewImageMiddleware** - *(optional, if the model supports vision)* Injects a hidden HumanMessage with base64 image data, identified by a reserved ID prefix plus a server-owned metadata marker, before the LLM call. Because `before_model`, `model`, and `after_model` are separate graph nodes, the `before_model` and `model` node checkpoints for that call still contain the payload; `after_model` / `aafter_model` then emits `RemoveMessage`, so subsequent checkpoints do not retain it
 24. **McpRoutingMiddleware** - *(optional, if `tool_search.enabled` and PR1 MCP routing metadata produce a routing index)* Auto-promotes matching deferred MCP tool schemas before the model call by writing a minimal `promoted` state update. It matches only the latest real `HumanMessage`, uses the global `tool_search.auto_promote_top_k` limit (default 3, clamped to 1..5), never executes tools, and must be installed before `DeferredToolFilterMiddleware`
 25. **DeferredToolFilterMiddleware** - *(optional, if `tool_search.enabled`)* Hides deferred (MCP) tool schemas from the bound model until `tool_search` or `McpRoutingMiddleware` promotes them (reads per-thread promotions from `ThreadState.promoted`, hash-scoped)
@@ -367,7 +387,7 @@ Before changing a later authorization phase, read the [authorization RFC](../doc
 32. **TerminalResponseMiddleware** - When a provider returns an empty terminal `AIMessage` after tool execution, injects a hidden recovery prompt and retries the model once; a second empty response is replaced in checkpoint state by a visible error fallback marked for the run worker, so the run finishes as an error instead of a silent success
 33. **ModelLengthFinishReasonMiddleware** - Records `stop_reason=model_length_capped` when provider-specific length detectors match a terminal `AIMessage` without tool-call intent (`finish_reason=length` / `MAX_TOKENS`, or `stop_reason=max_tokens`), preserving the original assistant content and never reparsing textual tool-call-like envelopes
 34. **SafetyFinishReasonMiddleware** - *(optional, if `safety_finish_reason.enabled`)* Suppresses tool execution when the provider safety-terminated the response (e.g. `finish_reason=content_filter`); registered after terminal-response/custom/configured middlewares so LangChain's reverse-order `after_model` dispatch runs it first
-35. **ClarificationMiddleware** - Intercepts `ask_clarification` tool calls, writes a readable `ToolMessage.content` fallback plus structured `ToolMessage.artifact.human_input` request payload, and interrupts via `Command(goto=END)` (must be last). Payloads are versioned: legacy modes (`free_text` / `choice_with_other`) keep `version: 1` unchanged, while the v2 `form` mode (from `fields`) carries `version: 2` so older frontends reject the payload and degrade to the plain-text fallback. Field normalization is deterministic and lives in the middleware, not the tool schema — the middleware short-circuits before tool execution, so tool-arg typing alone provides no runtime validation. Validation is atomic: any structurally broken entry (non-dict, bad/duplicate name, a name colliding with a JS `Object.prototype` member like `__proto__`/`constructor`, exceeding the caps of 16 fields / 24 options per field / 200 chars per text, or the whole normalized definition exceeding `MAX_FORM_SERIALIZED_BYTES` = 16KB UTF-8 — the per-item caps alone admit forms whose IM text fallback would blow channel delivery limits and truncate away trailing fields) degrades the whole form to the legacy option/free-text modes, so a card can never render "complete" while silently missing a business field; benign issues keep local degradation (unknown types — including unhashable JSON like `type: []`, which must never raise from the membership probe — and option-less selects become `text`), and options are trimmed/deduped with blanks dropped (both form-level and top-level) because the frontend parser rejects blank option labels. Checkbox fields are booleans that default to an explicit "no"; `required` on a checkbox means must-agree/consent semantics. The response protocol is deliberately unchanged (v1 `text`/`option` only): form cards submit a readable text summary as `response_kind: "text"`, so journal persistence and answered-card recovery need no new allowlist entries. Because this middleware can short-circuit tool execution before LangChain emits `on_tool_end`, `RunJournal` performs a root-run final reconciliation for allowlisted clarification `ToolMessage`s whose `tool_call_id` was produced by the current run, so human-input request cards remain recoverable from `run_events` after checkpoint compaction. Human Input Card replies are submitted as `hide_from_ui` `HumanMessage`s with `additional_kwargs.human_input_response`; `RunJournal` persists only allowlisted hidden response sources (currently `ask_clarification`) as `llm.human.input`, which preserves answered-card state after compaction without exposing generic internal hidden context.
+35. **ClarificationMiddleware** - Intercepts `ask_clarification` tool calls, writes a readable `ToolMessage.content` fallback plus structured `ToolMessage.artifact.human_input` request payload, and interrupts via `Command(goto=END)` (must be last). Payloads are versioned: legacy modes (`free_text` / `choice_with_other`) keep `version: 1` unchanged, while the v2 `form` mode (from `fields`) carries `version: 2` so older frontends reject the payload and degrade to the plain-text fallback. Field normalization is deterministic and lives in the middleware, not the tool schema — the middleware short-circuits before tool execution, so tool-arg typing alone provides no runtime validation. Validation is atomic: any structurally broken entry (non-dict, bad/duplicate name, a name colliding with a JS `Object.prototype` member like `__proto__`/`constructor`, exceeding the caps of 16 fields / 24 options per field / 200 chars per text, or the whole normalized definition exceeding `MAX_FORM_SERIALIZED_BYTES` = 16KB UTF-8 — the per-item caps alone admit forms whose IM text fallback would blow channel delivery limits and truncate away trailing fields) degrades the whole form to the legacy option/free-text modes, so a card can never render "complete" while silently missing a business field; benign issues keep local degradation (unknown types — including unhashable JSON like `type: []`, which must never raise from the membership probe — and option-less selects become `text`), and options are trimmed/deduped with blanks dropped (both form-level and top-level) because the frontend parser rejects blank option labels. Model-produced XML-to-dict option payloads are recursively flattened from dict/list containers in source order, scalar string/number leaves are retained, and residual XML tags are removed before the same trimming and deduplication. Checkbox fields are booleans that default to an explicit "no"; `required` on a checkbox means must-agree/consent semantics. The response protocol is deliberately unchanged (v1 `text`/`option` only): form cards submit a readable text summary as `response_kind: "text"`, so journal persistence and answered-card recovery need no new allowlist entries. Because this middleware can short-circuit tool execution before LangChain emits `on_tool_end`, `RunJournal` performs a root-run final reconciliation for allowlisted clarification `ToolMessage`s whose `tool_call_id` was produced by the current run, so human-input request cards remain recoverable from `run_events` after checkpoint compaction. Human Input Card replies are submitted as `hide_from_ui` `HumanMessage`s with `additional_kwargs.human_input_response`; `RunJournal` persists only allowlisted hidden response sources (currently `ask_clarification`) as `llm.human.input`, which preserves answered-card state after compaction without exposing generic internal hidden context.
 
 ### Configuration System
 
@@ -419,7 +439,7 @@ Extensions are optional only in the fallback *search* mode (priority 3-4 above):
 
 FastAPI application on port 8001 with health check at `GET /health`. Set `GATEWAY_ENABLE_DOCS=false` to disable `/docs`, `/redoc`, and `/openapi.json` in production (default: enabled).
 
-CORS is same-origin by default when requests enter through nginx on port 2026. Split-origin or port-forwarded browser clients must opt in with `GATEWAY_CORS_ORIGINS` (comma-separated exact origins); Gateway `CORSMiddleware` and `CSRFMiddleware` both read that variable so browser CORS and auth-origin checks stay aligned.
+CORS is same-origin by default when requests enter through nginx on port 2026. Split-origin or port-forwarded browser clients must opt in with `GATEWAY_CORS_ORIGINS` (comma-separated exact origins); Gateway `CORSMiddleware` and `CSRFMiddleware` both read that variable so browser CORS and auth-origin checks stay aligned. Those clients also need `CORS_EXPOSED_HEADERS` (`csrf_middleware.py`): run-creating routes return the run's id in `Content-Location`, which is not CORS-safelisted, so JS cannot read it unless it is exposed. The LangGraph SDK resolves run metadata from that header alone — withhold it and `useStream`'s `onCreated` never fires, a new thread keeps its placeholder route, and every action gated on an established thread (edit, regenerate, branch) stays hidden until the page is reloaded. Same-origin nginx deployments never hit this because CORS does not apply.
 
 Browser auth sessions are owned by `app.gateway.auth.session_cookie`. Login accepts a `remember_me` form flag, but the Gateway never stores passwords. `SessionCookiePolicy` persists the `HttpOnly access_token` cookie only for HTTPS/trusted-forwarded HTTPS, direct-host localhost HTTP, or explicit operator opt-in for insecure persistence; public HTTP sandbox URLs degrade to session cookies. Session-creating handlers stamp the final `max_age` on `request.state`, and CSRF cookie creation mirrors that value so the double-submit cookie pair expires together, including explicit re-issue after password changes and OIDC callbacks. A small `HttpOnly` preference cookie preserves the user's remember choice across token re-issue paths. Logout clears all auth cookies and suppresses CSRF re-issue on the logout response.
 
@@ -441,7 +461,7 @@ Localhost persistence deliberately reads the direct request `Host` and ignores `
 | **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types |
 | **Suggestions** (`/api/suggestions`) | `GET /config` - returns global suggestions config boolean; `POST /threads/{id}/suggestions` - generate follow-up questions; rich list/block model content is normalized and inline reasoning (`<think>...</think>`, including unclosed/truncated blocks from reasoning models like MiniMax-M3) is stripped before JSON parsing |
 | **Input Polish** (`/api/input-polish`) | `POST /` - rewrite a composer draft before it is sent. This is a short authenticated `runs:create` LLM request using `input_polish` config; it does not create a LangGraph run, persist a message, or modify thread state. Shares the non-graph one-shot LLM path (`deerflow.utils.oneshot_llm.run_oneshot_llm`) with the suggestions route so model build + Langfuse metadata + invoke stay in one place; validates the same stripped view of the draft it sends to the model, and preserves literal `<think>` substrings in the rewrite (`strip_think_blocks(truncate_unclosed=False)`) |
-| **Thread Runs** (`/api/threads/{id}/runs`) | `POST /` - create background run; `POST /stream` - create + SSE stream; `POST /wait` - create + block; `POST /regenerate/prepare` - prepare clean input + checkpoint metadata for regenerating the latest assistant answer; carrying the latest non-empty thread title in graph input so resuming an older checkpoint cannot roll back a later manual rename (#4457); `POST /edit-regenerate/prepare` - prepare a checkpoint replay from the latest editable human turn with a replacement user message and edit replay metadata; `GET /` - list runs; `GET /{rid}` - run details; `POST /{rid}/cancel` - cancel; `GET /{rid}/join` - join SSE; `GET /{rid}/messages` - paginated per-run messages `{data, has_more}`; `GET /{rid}/events` - full event stream; `GET /{rid}/workspace-changes` - workspace/output file change summary and optional diffs; `GET /../messages` - legacy thread message array; `GET /../messages/page` - backward thread-global `seq` history page with middleware/subagent-AI/successful-regenerate/edit-replay filtering and page-run-scoped feedback enrichment; subagent AI callbacks remain available through run events while parent `task` ToolMessages stay visible for card restoration; `GET /../token-usage` - aggregate tokens |
+| **Thread Runs** (`/api/threads/{id}/runs`) | `POST /` - create background run; `POST /stream` - create + SSE stream; `POST /wait` - create + block; `POST /regenerate/prepare` - prepare clean input + checkpoint metadata for regenerating the latest completed or interrupted assistant answer, carrying the latest non-empty thread title in graph input so resuming an older checkpoint cannot roll back a later manual rename (#4457); `POST /edit-regenerate/prepare` - prepare a checkpoint replay from the latest editable human turn with a replacement user message and edit replay metadata; it carries the current thread title the same way, but only when the replay base already has one — an untitled base belongs to a thread the title middleware has not named yet, so pinning the current title there would keep a name generated from the prompt the edit just replaced; `GET /` - list runs; `GET /{rid}` - run details; `POST /{rid}/cancel` - cancel; `GET /{rid}/join` - join SSE; `GET /{rid}/messages` - paginated per-run messages `{data, has_more}`; `GET /{rid}/events` - full event stream; `GET /{rid}/workspace-changes` - workspace/output file change summary and optional diffs; `GET /../messages` - legacy thread message array; `GET /../messages/page` - backward thread-global `seq` history page with middleware/subagent-AI/successful-regenerate/edit-replay filtering and page-run-scoped feedback enrichment; subagent AI callbacks remain available through run events while parent `task` ToolMessages stay visible for card restoration; `GET /../token-usage` - aggregate tokens |
 | **Feedback** (`/api/threads/{id}/runs/{rid}/feedback`) | `PUT /` - upsert feedback; `DELETE /` - delete user feedback; `POST /` - create feedback; `GET /` - list feedback; `GET /stats` - aggregate stats; `DELETE /{fid}` - delete specific |
 | **Runs** (`/api/runs`) | `POST /stream` - stateless run + SSE; `POST /wait` - stateless run + block; `GET /{rid}/messages` - paginated messages by run_id `{data, has_more}` (cursor: `after_seq`/`before_seq`); `GET /{rid}/feedback` - list feedback by run_id |
 | **GitHub Webhooks** (`/api/webhooks/github`) | `POST /` - receive GitHub App / repo webhook deliveries. Verifies `X-Hub-Signature-256` against `GITHUB_WEBHOOK_SECRET`; exempt from auth + CSRF because authenticity is enforced by HMAC. The route is fail-closed: mounted only when `GITHUB_WEBHOOK_SECRET` is set, or when explicit dev opt-in `DEER_FLOW_ALLOW_UNVERIFIED_GITHUB_WEBHOOKS=1` is set. Recognized events include `ping`, `issues`, `issue_comment`, `pull_request`, `pull_request_review`, and `pull_request_review_comment`; unknown events return 200 with `handled=false`. Fan-out runtime failures return 503, keeping the delivery recorded as failed for manual/API/scripted redelivery (GitHub does not automatically retry any failed delivery, 5xx included); permanent/non-retryable conditions such as `channels.github.enabled: false`, unknown events, malformed payloads, or unavailable channel service return 200 with a skipped/handled response. |
@@ -508,10 +528,10 @@ JSONL event stores when `GATEWAY_WORKERS > 1`.
 - Edit-and-rerun visibility is derived from edit replay runs (`metadata.replay_kind="edit"` plus `regenerate_from_run_id`) by `RunManager.list_edit_replay_visibility()`: the newest attempt for each source run is authoritative. Pending/running/success attempts hide the original source run; failed, timed-out, or interrupted attempts hide only the failed attempt so the original conversation reappears.
 - When a persistent `RunStore` is configured, `get()` and `list_by_thread()` hydrate historical runs from the store. In-memory records win for the same `run_id` so task, abort, and stream-control state stays attached to active local runs.
 - Thread metadata status switches to `running` only after `RunManager.try_start()` succeeds. Pending-cancelled runs therefore skip the old `running` projection, while clients may observe the prior thread status during the short worker-startup window.
-- `cancel()` returns a :class:`~deerflow.runtime.CancelOutcome` enum: `cancelled` (local cancel), `taken_over` (non-owning worker claimed the run because the owner's lease expired — marks it as `error`), `lease_valid_elsewhere` (owner's lease is still alive — caller should return 409 + `Retry-After`), `not_active_locally` (heartbeat disabled, preserving the old 409 path), `not_cancellable` (terminal state), or `unknown` (not found in memory or store). `create_or_reject(..., multitask_strategy="interrupt"|"rollback")` persists interrupted status through `RunStore.update_status()`, matching normal `set_status()` transitions.
+- `cancel()` returns a :class:`~deerflow.runtime.CancelOutcome` enum: `cancelled` (local cancel), `requested` (the non-owning worker durably recorded the first cancellation action for the live owner), `taken_over` (non-owning worker claimed the run because the owner's lease expired — marks it as `error`), `lease_valid_elsewhere` (legacy/custom store lacks the durable request primitive — caller retains the safe 409 + `Retry-After` fallback), `not_active_locally` (heartbeat disabled, preserving the old 409 path), `not_cancellable` (terminal state), or `unknown` (not found in memory or store). `create_or_reject(..., multitask_strategy="interrupt"|"rollback")` persists interrupted status through `RunStore.update_status()`, matching normal `set_status()` transitions.
 - Interrupt/rollback admission registers the replacement before its best-effort persistence of locally interrupted predecessors. If the admitting caller is cancelled during that post-registration await, `RunManager` drains a shielded replacement cleanup before propagating `CancelledError`, including across repeated cancellation. The cleanup normally persists `interrupted`; if that best-effort transition fails, it retries the active-to-`interrupted` store transition strictly and verifies the result with the replacement's captured owner identity. A concurrent peer terminal transition wins and is synchronized back into the local record rather than being overwritten or deleted.
-- Store-only hydrated runs are readable history. In multi-worker mode with heartbeat enabled, cancel on a store-only run can take over (mark `error`) when the owner's lease has expired past the grace window; otherwise it fails with 409 + `Retry-After`. In single-worker mode (heartbeat off), store-only runs still return 409.
-- A local worker's `RunRecord.lease_expires_at` is the last durably confirmed ownership deadline. `_renew_leases()` bounds each renewal attempt by that deadline: transient store exceptions remain retryable while it is valid, but an exception or blocked call that reaches expiry sets the process-local `ownership_lost` fence, raises `abort_event`, and cancels the run task. Fenced workers do not perform subsequent journal/delivery-receipt, progress/completion/status, checkpoint/thread-metadata, or `on_run_completed` writes; the peer recovery path owns the terminal receipt. `RunStore.update_run_completion()` also refuses to replace a different terminal status, closing the peer-takeover/late-finalization race. `grace_seconds` delays peer reclamation for clock skew but is not extra execution time for an owner that can no longer confirm its lease. Already-committed remote tool side effects remain outside this local cancellation boundary.
+- Store-only hydrated runs are readable history. In multi-worker mode with heartbeat enabled, cancel on a store-only run records `runs.cancel_action` / `cancel_requested_at` while the owner's lease is live; the first action wins even if a retry later lands on the owner. `RunStore.request_cancel()` and owner completion through `finalize_if_not_cancelled()` are competing active-row CAS operations, so an accepted cancel cannot be overwritten by a later success. `RunStore.renew_lease()` renews and observes the request atomically in the SQL implementation. The owner then executes the normal process-local interrupt/rollback and terminal stream path without transferring the lease. An expired owner is still taken over and marked `error`. `wait=true` and cancel-then-stream use the shared bridge to observe owner finalization; a non-standard process-local bridge returns accepted 202 instead of subscribing to an unreachable stream. In single-worker mode (heartbeat off), store-only runs still return 409.
+- A local worker's `RunRecord.lease_expires_at` is the last durably confirmed ownership deadline. `_renew_leases()` bounds each renewal attempt by that deadline: transient store exceptions remain retryable while it is valid, but an exception or blocked call that reaches expiry sets the process-local `ownership_lost` fence, raises `abort_event`, and cancels the run task. Successful renewals collect durable cancellation actions; after all local renewals have been attempted, heartbeat only signals the corresponding process-local tasks, leaving status writes and rollback cleanup to the worker finalization path. Fenced workers do not perform subsequent journal/delivery-receipt, progress/completion/status, checkpoint/thread-metadata, or `on_run_completed` writes; the peer recovery path owns the terminal receipt. `RunStore.update_run_completion()` also refuses to replace a different terminal status, closing the peer-takeover/late-finalization race. `grace_seconds` delays peer reclamation for clock skew but is not extra execution time for an owner that can no longer confirm its lease. Already-committed remote tool side effects remain outside this local cancellation boundary.
 - Startup/orphan reconciliation must claim stale active rows with `RunStore.claim_for_takeover()`, not a plain `update_status()`. The final claim re-checks `status` and lease expiry atomically, so a heartbeat renewal between the candidate scan and the recovery write keeps the run active.
 - Run admission and independent checkpoint writes are first-class thread operations. `runs.operation_kind` distinguishes user-visible `run` rows from internal `checkpoint_write` reservations, while every active kind shares the existing durable active-thread uniqueness constraint. New operation kinds must go through `RunStore.create_thread_operation_atomic()` and `RunManager.reserve_thread_operation()` rather than adding another lock or metadata marker. Live and lease-less reservations are non-interruptible; an expired leased reservation can be reclaimed immediately by interrupt/rollback admission without waiting for orphan reconciliation. Lease-less rows stay fail-closed because the store cannot distinguish a stale row from a live checkpoint writer in another heartbeat-disabled worker; a rare failed delete therefore requires startup reconciliation, and heartbeat-disabled multi-worker deployment remains unsupported. Reservation bodies are attached to their caller task so loss detected by lease renewal cancels the checkpoint writer before it can continue after takeover; the context manager translates that lease-loss cancellation to `ConflictError` after cleanup so Gateway mutation routes return a retryable 409 instead of dropping the HTTP request. The cleanup scope begins immediately after durable admission, including the await that attaches the caller task, so cancellation cannot strand a locally renewed pending reservation. A failed renewal is revalidated under the manager lock before cancellation; if the reservation completed and unregistered while the store update was in flight, its request task must not be cancelled after the checkpoint write. Reservations are excluded from run history/reporting and from run-only helpers such as `list_by_thread()` and `has_inflight()`, release uses the captured owner rather than ambient user context, and local cleanup still runs when the best-effort store delete fails. `RunStore.create_run_atomic()` remains a deprecated compatibility shim for external stores that only admit normal runs; new stores must implement `create_thread_operation_atomic()` to support internal operation kinds.
 - Gateway checkpoint mutations outside run execution must use `services.reserve_checkpoint_write()`, which composes the process-local thread lock with the durable `checkpoint_write` reservation. Manual compaction, `POST /threads/{id}/state`, and both goal mutation routes (`PUT` / `DELETE /threads/{id}/goal`, including creation of a missing goal checkpoint) use this boundary, so an existing run blocks the write and the reservation blocks new reject/interrupt/rollback runs across workers.
@@ -533,6 +553,20 @@ full and delta checkpoint modes. Only an explicitly absent legacy parent link ma
 use chronological compatibility lookup; cycles, dangling links, and depth-limit
 exhaustion fail closed. Existing single-checkpoint branches are never repaired by
 copying a raw checkpoint because delta state is not self-contained in one tuple.
+Both lookups additionally require the replay base to be a **settled** checkpoint
+(`has_pending_tasks` — no scheduled `next` tasks). A checkpoint with pending tasks
+is a mid-run snapshot: resuming from it replays the writes of the node that was
+about to run. Message ids alone cannot exclude those, because middleware may
+rewrite a message's id inside the run that produced it — `DynamicContextMiddleware`
+moves the first user turn to `{id}__user` and gives `{id}` to the injected
+reminder, so every checkpoint written before it holds the same prompt under an
+unmatched id. Selecting one of those re-added the original prompt *after* the
+edited one, and the model answered the question the edit was replacing (#4531).
+`next` is not derivable on the degraded raw-checkpoint read path, which reports no
+tasks; absence of evidence stays permissive there rather than failing closed.
+Edit replay resolves its base through the same lineage-first path as regenerate;
+it must pass `head_checkpoint` or it silently degrades to the chronological scan
+that cannot tell sibling branches apart.
 
 ### Sandbox System (`packages/harness/deerflow/sandbox/`)
 
@@ -541,7 +575,7 @@ copying a raw checkpoint because delta state is not self-contained in one tuple.
 **Environment policy** (`sandbox/env_policy.py`): `execute_command` no longer inherits the full `os.environ`. `build_sandbox_env()` scrubs secret-looking names (`*KEY*`/`*SECRET*`/`*TOKEN*`/`*PASS*`/`*CREDENTIAL*`) from the inherited environment before layering injected request secrets on top, so platform credentials (e.g. `OPENAI_API_KEY`) never leak into skill subprocesses. Benign vars (`PATH`, `HOME`, `LANG`, `VIRTUAL_ENV`, ...) are preserved.
 **Implementations**:
 - `LocalSandboxProvider` - Local filesystem execution. `acquire(thread_id)` returns a per-thread `LocalSandbox` (id `local:{thread_id}`) whose `path_mappings` resolve `/mnt/user-data/{workspace,uploads,outputs}` and `/mnt/acp-workspace` to that thread's host directories, so the public `Sandbox` API honours the `/mnt/user-data` contract uniformly with AIO. `acquire()` / `acquire(None)` keeps the legacy generic singleton (id `local`) for callers without a thread context. Per-thread sandboxes are held in an LRU cache (default 256 entries) guarded by a `threading.Lock`. Legacy global-custom mounts are gated by the same user-scoped skill discovery rule used for prompt/list visibility; providers must not infer visibility from raw directory presence alone.
-- `AioSandboxProvider` (`packages/harness/deerflow/community/`) - Docker-based isolation. Active-cache and warm-pool entries are checked with the backend during acquire/reuse; definitively dead containers are dropped from all in-process maps so the thread can discover or create a fresh sandbox instead of reusing a stale client. Backend health-check failures are treated as unknown, not dead; local discovery likewise treats an unverifiable container as not adoptable and falls through to create rather than failing acquire. `get()` remains an in-memory lookup for event-loop-safe tool paths — it never touches the ownership store (that would be blocking IO on the event loop); ownership is published on acquire/reclaim and refreshed off the event loop by the dedicated renewal thread (`_renew_owned_leases`). Legacy global-custom mounts follow the same shared visibility helper as local and remote providers. Readiness probes and `agent_sandbox` clients classify loopback/private IPs, single-label cluster hosts, and Docker/Podman internal hostnames as direct control-plane destinations and set `trust_env=False`; external FQDNs and public IPs retain environment proxy support.
+- `AioSandboxProvider` (`packages/harness/deerflow/community/`) - Docker-based isolation. Active-cache and warm-pool entries are checked with the backend during acquire/reuse; definitively dead containers are dropped from all in-process maps so the thread can discover or create a fresh sandbox instead of reusing a stale client. Backend health-check failures are treated as unknown, not dead; local discovery likewise treats an unverifiable container as not adoptable and falls through to create rather than failing acquire. `get()` remains an in-memory lookup for event-loop-safe tool paths — it never touches the ownership store (that would be blocking IO on the event loop); ownership is published on acquire/reclaim and refreshed off the event loop by the dedicated renewal thread (`_renew_owned_leases`). `uses_thread_data_mounts` defaults to backend detection (`LocalContainerBackend=True`, remote/provisioner backends=False), while the optional `sandbox.thread_data_mounts` boolean takes precedence for deployments that guarantee the Gateway and sandbox share the same thread user-data directories. Setting it `true` skips upload-time sandbox acquire/sync; a false positive leaves uploads unavailable to the sandbox. Legacy global-custom mounts follow the same shared visibility helper as local and remote providers. Readiness probes and `agent_sandbox` clients classify loopback/private IPs, single-label cluster hosts, and Docker/Podman internal hostnames as direct control-plane destinations and set `trust_env=False`; external FQDNs and public IPs retain environment proxy support.
 - `E2BSandboxProvider` (`packages/harness/deerflow/community/e2b_sandbox/`) provides E2B remote isolation.
   Acquire and release share a per-user and thread lock. The provider lock does
   not cover remote IO. `burst_limit` adds capacity only for the `burst` policy.
@@ -624,10 +658,12 @@ copying a raw checkpoint because delta state is not self-contained in one tuple.
 **Guardrail caps & `stop_reason` (#3875 Phase 2)**: three independent axes can end a subagent run early, and all now surface *why* through one additive field rather than a new status enum. **Turn axis**: `recursion_limit` on the subagent `run_config` equals `max_turns`, so exhausting the turn budget raises `GraphRecursionError` from `agent.astream`; `executor.py::_aexecute` catches it specifically (before the generic `except Exception`). **Token axis**: `TokenBudgetMiddleware` is attached per-agent via `build_subagent_runtime_middlewares` from `subagents.token_budget` (default `max_tokens` **coupled to `summarization.enabled`** — 1,000,000 when subagent summarization is on, 2,000,000 when off, warn at 0.7, hard-stop at 1.0; a user-set budget always wins regardless of the switch — #3875 Phase 3; a backstop against a subagent that burns tokens on trivial work). It does *not* raise: at the hard-stop threshold it strips the in-flight turn's tool calls, forces `finish_reason="stop"`, and lets the run complete naturally with a final answer. **Loop axis**: `LoopDetectionMiddleware` (attached at the same point) catches repeated identical tool-call sets — or one tool *type* called many times with varying args — and its hard-stop likewise strips `tool_calls` and forces a final answer without raising, recording `loop_capped`. Each guard exposes its cap on a per-`run_id` `consume_stop_reason(run_id)` accessor; `_aexecute` collects **every** middleware with that method (duck-typed via `hasattr`, so the executor has no import coupling to the guard classes) and surfaces the first non-`None` reason — adding a future guard needs no executor change. **Surfacing**: whichever axis fired, `_aexecute` stamps a normal status plus an additive reason — `completed` + `stop_reason=token_capped|turn_capped|loop_capped` when a usable final answer (or partial recovered from the last streamed chunk via `_extract_final_result` → `utils/messages.py::message_content_to_text`, returning a `"No response Generated"` sentinel when no text survived) was produced; `failed` + `stop_reason=turn_capped` when nothing usable survived. `SubagentResult.stop_reason` flows through `task_tool.py::_task_result_command` → `format_subagent_result_message` (renders `Task Succeeded (capped: ...)` / `Task failed (capped: ...)`) and `make_subagent_additional_kwargs`, which stamps the additive `subagent_stop_reason` key alongside the normal `subagent_status`. **Why additive, not an enum**: a new status value would break v1 consumers; an optional field is ignored by older frontends and ledger readers, so the cross-language contract (`contracts/subagent_status_contract.json` v2 + `subagents/status_contract.py` + `frontend/.../subtask-result.ts`, pinned by `test_status_values_match_contract` / `test_stop_reason_values_match_contract`) stays backward-compatible. The durable delegation ledger captures `stop_reason` onto the entry and renders model-facing guidance ("hit a guardrail cap with a partial result; reuse it, retry tighter, or raise the per-agent budget (`max_turns` / `token_budget`)") so the lead reuses a capped completion knowingly instead of mistaking it for a clean one. (Phase 1 shipped this surfacing as a `MAX_TURNS_REACHED` status enum in #3949; Phase 2 replaced that enum with the additive `stop_reason` field per the agreed design — the `max_turns_reached` status value and `SubagentStatus.MAX_TURNS_REACHED` are gone.)
 **Context compaction (#3875 Phase 3, #4039)**: subagents inherit `DeerFlowSummarizationMiddleware` via `build_subagent_runtime_middlewares`, gated on the **same** `summarization.enabled` switch the lead reads (one config covers both chains; trigger/keep/model/prompt come from the shared `summarization` config so they cannot drift). The subagent builder attaches `DurableContextMiddleware` immediately before summarization, using the same skills path/read-tool settings as the lead chain. Compaction stores the generated summary in `ThreadState.summary_text` rather than as a `messages` item; the durable-context wrapper therefore projects it into the next model request as guarded hidden human data. This is required when a message-count keep policy preserves only an assistant tool-call plus its tool results: without the injected summary the next request begins with assistant/tool history and strict OpenAI-compatible providers can reject it. Because `DurableContextMiddleware` inserts a second `SystemMessage(authority_contract)` after the subagent's leading system prompt, the builder also appends `SystemMessageCoalescingMiddleware` innermost (mirroring the lead chain, appended after the optional summarization middleware so it is unconditionally last) to merge every `SystemMessage` into one leading `system_message` — otherwise the durable fix would trade #4039's assistant-first HTTP 400 for a duplicate-system 400 on the same strict backends (#4040). The factory is called with `skip_memory_flush=True` on the subagent path: the lead's `memory_flush_hook` (attached when `memory.enabled`) flushes pre-compaction messages into durable memory keyed by `thread_id`, and subagents share the parent's `thread_id`, so without skipping the hook a subagent's internal turns would pollute the **parent** thread's durable memory. Placement differs from the lead chain (lead appends summarization *before* the guard trio; subagent appends it *after*) — benign because the middleware implements only `before_model` (compaction) with no `after_model`/`consume_stop_reason`, so it cannot disturb the Phase 2 guard-cap stop-reason channel. Compaction rewrites the messages channel via `RemoveMessage(id=REMOVE_ALL_MESSAGES)`, which shrinks `len(messages)` below the step-capture cursor mid-run; `capture_new_step_messages` (see Step capture below) resets the cursor to the new tail on contraction so steps appended after the compaction point are not silently dropped.
 **Step capture & persistence (#3779)**: `executor.py` captures both assistant turns (`AIMessage`) **and** tool outputs (`ToolMessage`) via `subagents/step_events.py::capture_new_step_messages`, which walks the *newly-appended tail* of each `stream_mode="values"` chunk (not just `messages[-1]`) so a multi-tool-call turn — where LangGraph's `ToolNode` appends several `ToolMessage`s in one super-step — keeps every tool output instead of dropping all but the last. `runtime/runs/worker.py::_SubagentEventBuffer` additionally persists these `task_*` custom events to the `RunEventStore` as `subagent.start`/`subagent.step`/`subagent.end` (`category="subagent"`, `task_id` in `metadata`). It **batches** writes via `put_batch` (flushing on a terminal `subagent.end`, at `FLUSH_THRESHOLD` events, and in the worker's `finally`) rather than one `put()` per step, since `put()` is a documented low-frequency path (per-thread advisory lock per call) and a deep subagent (`max_turns=150`) emits hundreds of steps on the hot stream loop. `subagent_run_event` rejects malformed chunks that lack a non-empty `task_id`; running chunks additionally require a non-negative integer `message_index` and a message object, so persisted records always satisfy the required lifecycle envelope. `build_subagent_step` caps both the per-step `text` and each tool call's serialized `args` at `SUBAGENT_STEP_MAX_CHARS` (flagged `truncated` / `args_truncated`) so a large `write_file`/`bash` payload can't produce an unbounded row. The dedicated category keeps them out of `list_messages` (the thread feed) while `list_events` returns them for the frontend's fetch-on-expand backfill. `list_events` accepts `task_id` (filters on `metadata["task_id"]` — SQL-side in `DbRunEventStore` via `event_metadata["task_id"].as_string()`, in-memory in the JSONL/memory stores) plus an `after_seq` forward cursor, so the card pages through one subagent's steps without the run-wide `limit` truncating the tail (no schema migration: the filter rides the existing run-scoped index). `step_events.py` is a pure, unit-tested layer (`build_subagent_step` / `subagent_run_event`). **History contraction (#3875 Phase 3)**: `capture_new_step_messages` assumes append-only growth, but `DeerFlowSummarizationMiddleware` rewrites the messages channel via `RemoveMessage(id=REMOVE_ALL_MESSAGES)`, shrinking `len(messages)` below the cursor mid-run. On contraction (`total < processed_count`) the cursor resets to the new tail; `capture_step_message`'s id/content dedup prevents re-emitting pre-compaction steps, so steps appended after the compaction point are still captured instead of being dropped until `total` overtakes the stale cursor.
-**Deferred MCP tools** (if `tool_search.enabled`): `SubagentExecutor._build_initial_state` assembles deferral after policy filtering via the shared `assemble_deferred_tools` (fail-closed), appends the `tool_search` tool, injects the `<available-deferred-tools>` section into the subagent's `SystemMessage`, and threads the setup to `_create_agent`, which attaches `McpRoutingMiddleware` (when PR1 routing metadata matches deferred tools) before `DeferredToolFilterMiddleware` through `build_subagent_runtime_middlewares(...)`. Subagents thus withhold full MCP schemas until promotion, same as the lead agent; each task run gets a fresh `ThreadState` so promotion is isolated per run
+**Deferred MCP tools** (if `tool_search.enabled`): `SubagentExecutor._build_initial_state` applies the subagent name allow/deny list and assembly-time authorization before calling the shared `assemble_deferred_tools`, appends the `tool_search` tool, injects the `<available-deferred-tools>` section into the subagent's `SystemMessage`, and threads the setup to `_create_agent`, which attaches `McpRoutingMiddleware` (when PR1 routing metadata matches deferred tools) before `DeferredToolFilterMiddleware` through `build_subagent_runtime_middlewares(...)`. Runtime skill policy is intentionally later and dynamic: `tool_search` may disclose/promote catalog metadata, but `SkillToolPolicyMiddleware` still removes or blocks any promoted business tool omitted by the active skill. Subagents thus withhold full MCP schemas until promotion, same as the lead agent; each task run gets a fresh `ThreadState` so promotion is isolated per run
 **Checkpointer isolation**: Subagent graphs are compiled with `checkpointer=False` to avoid inheriting the parent run's checkpointer, since subagents are one-shot and never resume.
 **Checkpoint lineage / stream isolation**: `_aexecute` deliberately omits checkpoint-coordinate keys (`thread_id`, `checkpoint_ns`, `checkpoint_id`, `checkpoint_map`) from the child `RunnableConfig`. LangGraph must inherit those coordinates from the copied parent ContextVar so the delegated graph retains a non-root subgraph namespace; explicitly re-supplying even the same parent `thread_id` starts a new root lineage on LangGraph 1.2.6+ and can route child AI/tool frames into the parent `messages` stream. DeerFlow business components still receive the parent `thread_id` through `runtime.context`, which is the preferred lookup path for sandbox, middleware, and attribution code. Regression coverage in `tests/test_subagent_executor.py::TestSubagentCheckpointLineage` keeps the invocation-contract assertion active on every supported version and version-gates the production-shaped parent-stream test to LangGraph 1.2.6+, where the leak exists.
 
+**Isolated-loop callback boundary**: sync delegation from an active event loop and `execute_async()` copy the ambient ContextVars into the persistent subagent loop so checkpoint lineage, user identity, tracing context, tags, metadata, and LangGraph's namespaced message-stream handler survive. Before submission, `_copy_isolated_subagent_context()` copies the callback manager/list and removes only handlers marked `deerflow_loop_bound`; `RunJournal` carries that marker because it owns parent-loop tasks and a SQL store/pool. LangGraph merges inherited callbacks with the child run's explicit `SubagentTokenCollector`/tracing callbacks, so letting `RunJournal` cross loops causes duplicate accounting and `Future attached to a different loop` failures, while dropping the whole callback chain silently removes child token frames. Do not replace the boundary with a blank `Context`; the inherited checkpoint namespace and framework stream callback are required by the stream-isolation contract above.
+
 ### Tool System (`packages/harness/deerflow/tools/`)
 
 `get_available_tools(groups, include_mcp, model_name, subagent_enabled)` assembles:
@@ -689,14 +725,14 @@ E2B output sync records remote file versions and actual host file metadata in a
 - **Format**: Directory with `SKILL.md` (YAML frontmatter: name, description, license, allowed-tools, required-secrets)
 - **Loading**: `load_skills()` recursively scans public, per-user custom, global integration, and legacy custom locations for `SKILL.md`, parses metadata, and reads enabled state from extensions_config.json plus per-user skill state for non-public categories; that directory is a package boundary, so no nested `SKILL.md` is registered as a runtime skill. SkillScan has a deliberately narrower packaging rule: known eval fixtures are permitted as support data, while other nested `SKILL.md` files are reported as package defects. It parses runtime metadata and reads enabled state from extensions_config.json.
 - **External reload**: `POST /api/skills/reload` is an admin-only, process-local invalidation hook for trusted MinIO/NFS/CSI writes. `SkillStorage` instances do not cache a catalog — `load_skills()` scans on every call — so the route clears all `(app_config, user_id)` entries and the rendered prompt-section LRU, then waits up to the shared refresh timeout for the existing off-loop single-flight refresh. Each invalidation receives a generation-bound result handle; a successful scan atomically replaces the global enabled-skills cache, while a loader-level failure propagates to the HTTP waiter and preserves the last-known-good global cache. Per-user/config scans capture the refresh version and cannot repopulate shared caches if invalidation occurs while they are loading. A timed-out HTTP wait fails generically while the daemon refresh worker continues. Subsequent runs rescan after a successful reload; active runs keep their existing snapshot. Each Uvicorn worker/Kubernetes Pod must be targeted separately. Direct mount writes bypass install/edit validation, SkillScan, and history, so mounted roots are an operator-controlled trust boundary.
-- **Tool policy**: Lead-agent `allowed-tools` declarations apply dynamically only to slash-activated skills and skills captured in `ThreadState.skill_context` through configured `read_file` loads; passive enabled skills and custom-agent skill allowlists remain discoverable without clamping the global toolset. Slash policy is dominant for its run, preventing subsequently read skills from widening explicit authority; autonomous captured skills use the existing union only when no slash source exists. `tool_search` and `describe_skill` stay available as framework discovery infrastructure, while every discovered or promoted business tool still requires active-policy permission for schema visibility and execution; `task` likewise requires an explicit declaration. Each active model call intentionally reloads the full live registry so enable/disable changes, frontmatter edits, and custom/public name-shadow winners take effect without a stale TTL or unsafe direct-path cache; all tool calls produced by that model step reuse the resulting source-and-path-signed decision. Registry failures and all-invalid active sets fail closed, while stale individual paths are skipped when another valid skill remains. This is best-effort behavioral scoping, not a hard security boundary: alternate loading paths are not captured and bounded autonomous context may evict entries. Subagents still filter statically because their configured skills are all loaded into the session at startup.
+- **Tool policy**: Agent `allowed-tools` declarations apply dynamically only to slash-activated skills and skills captured in `ThreadState.skill_context` through configured `read_file` loads; passive enabled skills and custom-agent/subagent skill allowlists remain discoverable without clamping the baseline toolset. Subagents render only skill discovery metadata at startup and reuse the same adjacent `SkillActivationMiddleware` + `SkillToolPolicyMiddleware` pair as the lead; their configured `skills` field limits discovery and activation instead of eagerly loading bodies or unioning policies. Slash policy is dominant for its run, preventing subsequently read skills from widening explicit authority; autonomous captured skills use the existing union only when no slash source exists. `tool_search` and `describe_skill` stay available as framework discovery infrastructure, while every discovered or promoted business tool still requires active-policy permission for schema visibility and execution; `task` likewise requires an explicit declaration. Each active model call intentionally reloads the full live registry so enable/disable changes, frontmatter edits, and custom/public name-shadow winners take effect without a stale TTL or unsafe direct-path cache; all tool calls produced by that model step reuse the resulting source-and-path-signed decision. Registry failures and all-invalid active sets fail closed, while stale individual paths are skipped when another valid skill remains. This is best-effort behavioral scoping, not a hard security boundary: alternate loading paths are not captured and bounded autonomous context may evict entries.
 - **Injection (legacy / default)**: Enabled skills are listed in the agent system prompt with full metadata and container paths (`<available_skills>` block). Controlled by `skills.deferred_discovery: false` (default).
 - **Deferred discovery** (`skills.deferred_discovery: true`): Skills are listed by name only in a compact `<skill_index>` block, keeping the system prompt prefix-cache friendly. The agent calls the `describe_skill` tool at runtime to fetch full metadata for skills it wants to use, then loads the SKILL.md via `read_file`. Two new modules support this path:
   - `skills/catalog.py` — `SkillCatalog` (immutable, searchable; query forms: `select:a,b`, `+prefix`, free-text regex); `select:` returns all requested skills without a result cap; other modes cap at `MAX_RESULTS=5`.
   - `skills/describe.py` — `build_describe_skill_tool(catalog)` builds the `describe_skill` tool as a closure; `build_skill_search_setup(skills, enabled, ...)` produces a `SkillSearchSetup(describe_skill_tool, skill_names)` that is wired into both the LangGraph agent factory (`agent.py`) and the embedded client (`client.py`).
 - **Slash activation**: `/skill-name task` loads that enabled skill's `SKILL.md` for the current model call only. The resolver rejects leading whitespace, missing separators, reserved channel commands (`/new`, `/help`, `/bootstrap`, `/status`, `/models`, `/memory`, `/goal`), disabled skills, and skills outside a custom agent's whitelist.
 - **Installation**: `POST /api/skills/install` extracts .skill ZIP archive to custom/ directory
-- **Managed integrations**: Lark/Feishu CLI support installs one global official `lark-*` pack as read-only `SkillCategory.INTEGRATION` entries under `/mnt/skills/integrations/lark-cli/...`; enabled flags, app configuration, and OAuth data remain per-user. Install resolves the newest `larksuite/cli` release from GitHub (`releases/latest`) at install time (falling back to a bottom-line pinned version if the lookup fails) rather than hard-coding the pack version; integrity relies on the official host + structural archive guards + a recorded hash of the effective installed tree after shared guidance injection (not a pinned archive-byte SHA, which GitHub does not keep stable). The Gateway image still installs a pinned `@larksuite/cli` binary, so `get_lark_integration_status` surfaces `latest_available_version` and `runtime_version_mismatch` for the UI. AIO installs additionally verify and publish official Linux amd64/arm64 binaries under `{DEER_FLOW_HOME}/integrations/lark-cli/sandbox-cli`, mounted read-only at `/mnt/integrations/lark-cli/runtime`; `/mnt/integrations/lark-cli/config` (app credentials, incl. the long-lived `appSecret`) is mounted **read-only** into the sandbox and `/mnt/integrations/lark-cli/data` (refreshable OAuth tokens) stays writable, both mapping to owner-only per-user credential directories. **Sandbox trust boundary:** those two dirs are still *readable* by arbitrary sandbox processes (the agent's `bash` tool, or code reached via prompt-injection in a tool result), so the app secret and tokens are exposed to sandbox-side code even though they never reach the browser — the read-only config mount only prevents in-sandbox tampering, not read/exfiltration. The sidecar credential-broker follow-up (issue #4338) is the planned fix that removes these plaintext mounts from sandbox execution. `DEER_FLOW_LARK_CLI_SANDBOX_RUNTIME_DIR` supplies a validated, symlink-free pre-staged runtime for air-gapped deployments. For the remote provisioner (K8s), the runtime is instead provisioned by an optional init container + shared `emptyDir` (Pattern A): set `LARK_CLI_INIT_IMAGE` on the provisioner (see `docker/lark-cli-init/`) and the Gateway sends `provision_lark_cli_runtime` on sandbox create once the pack is installed, so remote installs skip the Gateway-side GitHub download entirely. `get_lark_integration_status(check_runtime=True)` surfaces `sandbox_runtime_mode` (`none` / `gateway-download` / `init-container`) and `sandbox_runtime_ready` (init-container mode reads the provisioner `GET /api/capabilities`) so a green UI can't hide a chat-time `lark-cli: command not found`. Cheap status probes are explicitly not live-verified; users authorize or reconnect through the browser device-flow endpoints instead of running terminal commands.
+- **Managed integrations**: Lark/Feishu CLI support installs one global official `lark-*` pack as read-only `SkillCategory.INTEGRATION` entries under `/mnt/skills/integrations/lark-cli/...`; enabled flags, app configuration, and OAuth data remain per-user. Install resolves the newest `larksuite/cli` release from GitHub (`releases/latest`) at install time (falling back to a bottom-line pinned version if the lookup fails) rather than hard-coding the pack version; integrity relies on the official host + structural archive guards + a recorded hash of the effective installed tree after shared guidance injection (not a pinned archive-byte SHA, which GitHub does not keep stable). The Gateway image still installs a pinned `@larksuite/cli` binary, so `get_lark_integration_status` surfaces `latest_available_version` and `runtime_version_mismatch` for the UI. AIO installs additionally verify and publish official Linux amd64/arm64 binaries under `{DEER_FLOW_HOME}/integrations/lark-cli/sandbox-cli`, mounted read-only at `/mnt/integrations/lark-cli/runtime`; `/mnt/integrations/lark-cli/config` (app credentials, incl. the long-lived `appSecret`) is mounted **read-only** into the sandbox and `/mnt/integrations/lark-cli/data` (refreshable OAuth tokens) stays writable, both mapping to owner-only per-user credential directories. **Sandbox trust boundary:** those two dirs are still *readable* by arbitrary sandbox processes (the agent's `bash` tool, or code reached via prompt-injection in a tool result), so the app secret and tokens are exposed to sandbox-side code even though they never reach the browser — the read-only config mount only prevents in-sandbox tampering, not read/exfiltration. The sidecar credential-broker (Pattern B, issue #4338) is the fix that removes these plaintext mounts from sandbox execution: set `LARK_CLI_BROKER_IMAGE` on the provisioner (see `docker/lark-cli-broker/`) and the Gateway sends `provision_lark_cli_broker` on sandbox create. The provisioner then runs a `lark-cli-broker` sidecar that owns the per-user `config`/`data` (mounted into the **sidecar only**, at `/var/lark/{config,data}`) and serves the `lark-cli` command surface on Pod loopback (`http://127.0.0.1:8788`); a shim init container (`install-shim`) writes a forwarding `lark-cli` into the shared runtime `emptyDir`, so the sandbox gets `DEERFLOW_LARK_BROKER_URL` + a shim on PATH but **no** credential files. The on-PATH `bin/lark-cli` is a `/bin/sh` launcher that resolves a Python 3 interpreter and execs the Python shim body (`bin/lark-cli-shim.py`) beside it, so broker mode does not silently ENOEXEC on a sandbox image without a `#!/usr/bin/env python3`-resolvable interpreter — it fails loudly (exit 127, actionable message) and can be pinned with `DEERFLOW_LARK_BROKER_PYTHON`. The broker runs `lark-cli` in the sidecar's cwd and cannot see the sandbox filesystem, so cwd is intentionally **not** forwarded and file-I/O subcommands relative to the sandbox cwd are unsupported (command surface only). An optional `DEERFLOW_LARK_BROKER_DENY_SUBCOMMANDS` denylist (comma-separated command prefixes, forwarded from the provisioner) lets the broker refuse secret-dumping subcommands before spawning the binary. `lark_cli_env_overlay(broker=True)` therefore omits `LARKSUITE_CLI_CONFIG_DIR`/`DATA_DIR`; `sandbox_lark_broker_active()` (TTL-cached provisioner `/api/capabilities` probe, tight timeout + longer negative caching on the bash hot path) selects broker vs. binary mode for both the bash env overlay and status. `DEER_FLOW_LARK_CLI_SANDBOX_RUNTIME_DIR` supplies a validated, symlink-free pre-staged runtime for air-gapped deployments. For the remote provisioner (K8s), the runtime binary is otherwise provisioned by an optional init container + shared `emptyDir` (Pattern A): set `LARK_CLI_INIT_IMAGE` on the provisioner (see `docker/lark-cli-init/`) and the Gateway sends `provision_lark_cli_runtime` on sandbox create once the pack is installed, so remote installs skip the Gateway-side GitHub download entirely. Broker (Pattern B) supersedes the init-container binary (Pattern A) when both images are configured. `get_lark_integration_status(check_runtime=True)` surfaces `sandbox_runtime_mode` (`none` / `gateway-download` / `init-container` / `broker`) and `sandbox_runtime_ready` (remote modes read the provisioner `GET /api/capabilities`: `lark_cli_init_image` / `lark_cli_broker_image`) so a green UI can't hide a chat-time `lark-cli: command not found`. Cheap status probes are explicitly not live-verified; users authorize or reconnect through the browser device-flow endpoints instead of running terminal commands.
 - **SkillScan**: `packages/harness/deerflow/skills/skillscan/` is the native deterministic scanner for `.skill` archives and agent-managed skill writes. It runs offline before the LLM scanner, emits structured findings (`rule_id`, `severity`, `file`, `line`, `message`, `remediation`, redacted `evidence` — category/analyzer are encoded in the `rule_id` prefix), blocks `CRITICAL`, and passes warning findings into `scan_skill_content()`. `scan_archive_preflight()` / `scan_skill_dir()` are pure sync functions (dispatch off the event loop); `enforce_static_scan()` applies the blocking policy and the `skill_scan.enabled` kill switch. The Python instance-client signal deliberately follows only a one-level, same-scope evidence chain (PR #4265 review): a proven imported constructor bound to a simple name, optional name-to-name alias propagation, rebinding invalidation, and a constructor-supported outbound method or context-manager use; bare canonical-looking names never fall back to module identity. Nested scopes never inherit client handles and inherit only constructor aliases proven stable by a binding-only enclosing-scope prepass. Comprehensions, walrus-bearing statements, annotations, executable expressions inside complex binding targets, unsupported operations, and ambiguous flows produce no finding from this signal; skipped constructs invalidate all names they may bind, while representative false negatives are pinned by `test_python_declared_false_negatives_stay_unreported`. Compound bodies are walked from isolated copies so wrapping code in `if True:` is not a bypass, while copied scope entries, binding-only prepasses, and AST visits consume a deterministic work budget and the walk stops after its first sink. Budget or recursion exhaustion skips only this best-effort signal and retains deterministic findings already collected for the file. Do not add Semgrep/OpenGrep or YAML rule-engine dependencies to the core path; Phase 1 rule specs live in Python constants next to their analyzers in `skillscan/orchestrator.py`.
 - **Skill Review Core**: `packages/harness/deerflow/skills/review/` provides read-only package snapshots, deterministic facts, resource/eval analysis, report rendering, and the CLI (`python -m deerflow.skills.review.cli`). It reuses the shared frontmatter helper and SkillScan; it must not import `app.*`, execute target scripts, install dependencies, or call networks. JSON contracts live in `contracts/skill_review/`. The `review_skill_package` built-in tool labels results with `review_subject_entry` and never `skill_context_entry`, so reviewing a target does not activate it, bind its `required-secrets`, or apply its `allowed-tools`. Its model-visible `ToolMessage.content` is a compact JSON payload with untrusted control tags neutralized; the full raw review payload, including Markdown renders, stays in `ToolMessage.artifact`. CI should run the CLI with `--fail-on error --fail-on-incomplete` so blocker/error findings and truncated/not-assessed packages fail the gate. The public `skills/public/skill-reviewer` skill owns semantic readiness review and suggestions only; mutation and runtime experiments remain owned by `skill-creator`.
 
@@ -705,13 +741,15 @@ E2B output sync records remote file versions and actual host file metadata in a
 Lets a caller pass per-request, short-lived end-user credentials (e.g. an ERP token) to a skill's sandbox scripts without the value entering the prompt, tool arguments, the executed command string, or traces (issue #3861).
 
 - **Declare**: a skill lists the secrets it needs in `SKILL.md` frontmatter — `required-secrets:` as a string list or `{name, optional}` mappings. `name` is both the lookup key and the env var name exposed to scripts. Parsed by `skills/parser.py::parse_required_secrets` into `Skill.required_secrets` (`SecretRequirement`); malformed entries are dropped with a warning.
-- **Carry**: the caller sends values out-of-band in the run request's `context.secrets` mapping (never a message). `runtime/secret_context.py` owns the contract (`SECRETS_CONTEXT_KEY`, `extract_request_secrets`). The existing `context` passthrough carries it to `runtime.context` without mirroring into `configurable`. `build_run_config` still sets `configurable.thread_id` on the context path — the checkpointer requires it.
+- **Carry**: the caller sends values out-of-band in the run request's `context.secrets` mapping (never a message). `runtime/secret_context.py` owns the contract (`SECRETS_CONTEXT_KEY`, `extract_request_secrets`). The existing `context` passthrough carries it to `runtime.context` without mirroring into `configurable`. `build_run_config` still sets `configurable.thread_id` on the context path — the checkpointer requires it. MCP tool interceptors can read the same live carrier from `langgraph.config.get_config()["context"]["secrets"]`; see `docs/MCP_SERVER.md`.
+- **Admission and redaction ownership**: `services.py::start_run()` validates both legacy request mappings, `metadata.auth_token` and `config.metadata.auth_token`, before any run or thread persistence. `runtime/secret_context.py::redact_config_secrets()` also removes nested config metadata secrets from observable and persisted config copies; historical `RunResponse.kwargs` applies the same redaction non-mutatively, leaving stored `RunRecord` data unchanged. Keep callers on `config.context.secrets` rather than adding another credential carrier. Scheduled task definitions have no durable credential carrier: `ScheduledTaskService` supplies only `scheduled_task_id`, `scheduled_task_run_id`, and `scheduled_trigger` as run metadata.
 - **Bind (point A+)**: `SkillActivationMiddleware._resolve_secret_bindings` recomputes the injection set (`runtime.context[__active_skill_secrets]`) on every model call from two unioned sources, then REPLACES the key. (1) *Slash*: the run's most recent `/skill` activation, persisted as a source on the run context (only the activated skill's **canonical container path**, never its declared secrets) so the whole tool loop after the activation call keeps the binding; a new activation replaces it. Slash reads the genuine user text via `get_original_user_content_text`; `InputSanitizationMiddleware` preserves it (`ORIGINAL_USER_CONTENT_KEY`), so activation fires even after sanitization. (2) *In-context* (autonomous invocation): skills the model actually loaded in this thread — `ThreadState.skill_context` entries. **Both sources resolve the live registry skill by normalized container path on every call** (`_resolve_registry_skill`) and bind only that skill's own declared secrets — enabled + allowlist checked for both; the `secrets-autonomous: false` opt-out (malformed values fail closed to `false`) additionally gates the in-context path but exempts explicit slash. Resolving by registry — not by trusting the source's stored data — is what makes a caller-forged `__slash_skill_secret_source` harmless (`runtime.context` is caller-mergeable; the gateway also strips caller `__`-keys in `build_run_config`), #3938. Authorization is three-gated regardless of activation style: skill **enabled** by the operator × values **supplied per-request** by the caller (`context.secrets`) × names **declared** in frontmatter (∩ semantics). Because the set is recomputed per call, a skill evicted from `skill_context` (capacity) or a caller that stops supplying a value loses injection on the next call. The injected value always comes from the caller's request, never the host environment (scrubbed first — see below), so a declared name that also exists in the host env is safe: the caller's value wins and the host value is dropped (the #3861 per-user-key-overrides-shared-key case). Missing required secrets are logged once per binding change, not injected; binding changes are recorded as a `middleware:skill_secrets` journal event (skill and secret names only, never values).
 - **Inject**: `bash_tool` reads the injection set and passes it as `execute_command(env=...)`. Scope is the activation turn/run only — a run without `/skill` activation injects nothing.
 - **AIO image requirement**: on `AioSandbox` the env path uses the `bash.exec` API (`POST /v1/bash/exec`), which upstream all-in-one-sandbox only ships since `1.9.3` — older images (including a `latest` tag frozen on the `1.0.0.x` line) 404 the whole `/v1/bash/*` namespace. `AioSandbox` detects the 404, remembers the capability gap on the instance, and fails fast with an actionable upgrade error instead of letting the model retry raw 404s; there is deliberately **no** fallback through the legacy shell path because none keeps the secret values out of the command string (#3921). Regression tests: `tests/test_aio_sandbox.py::TestBashExecUnsupportedFailFast`.
 - **Inherited-env scrub**: `execute_command` no longer leaks the Gateway's `os.environ` to skill subprocesses — `env_policy.build_sandbox_env` drops secret-looking names (`*KEY*`/`*SECRET*`/`*TOKEN*`/`*PASS*`/`*CREDENTIAL*`/`*DSN*` + a connection-string denylist like `DATABASE_URL`/`REDIS_URL`/`GH_PAT`, plus no-flag credential sources like `MYSQL_PWD`/`REDISCLI_AUTH`/`PGPASSFILE`/`PGSERVICEFILE`) so platform credentials never reach a skill; a skill that needs one must declare it.
 - **Leak surfaces sealed** (verified by a real-gateway e2e run — secret reaches the sandbox but none of these): prompt (value never in a message), trace (`tracing/metadata.py` never copies `context`), checkpoint (secrets live on `runtime.context`, not graph state), audit (journal records names only), stdout (`tools.py::mask_secret_values` redacts injected values from bash output), and **run-record persistence + run API** (`services.py::start_run` stores `redact_config_secrets(body.config)` so `runs.kwargs_json` and `RunResponse.kwargs` never carry the secret).
-- **Scope / non-goals**: no persistence/vaulting — values are request-scoped and never stored server-side, so long-lived use means the caller re-supplies `context.secrets` on each request while the skill stays in `skill_context`; subagents do not inherit the injection set; the MCP per-user-credential gap (#3322) is a sibling, not covered here. Tests: `tests/test_skill_request_scoped_secrets.py`.
+- **Historical retention**: API response hiding prevents legacy `metadata.auth_token` and `config.metadata.auth_token` from being returned now; it does not delete values already retained in databases, run events, logs, snapshots, exports, or backups. Deployments that ever used either legacy carrier must rotate the credential and clean every retained copy under their retention policy. Restarting or upgrading DeerFlow performs neither action.
+- **Scope / non-goals**: no persistence/vaulting — values are request-scoped and never stored server-side, so long-lived use means the caller re-supplies `context.secrets` on each request while the skill stays in `skill_context`; subagents do not inherit the skill injection set. MCP interceptors may independently consume the same supported request-scoped carrier. Tests: `tests/test_skill_request_scoped_secrets.py`, `tests/test_mcp_session_pool.py`.
 
 ### Model Factory (`packages/harness/deerflow/models/factory.py`)
 
@@ -727,6 +765,7 @@ Lets a caller pass per-request, short-lived end-user credentials (e.g. an ERP to
 - `VllmChatModel` subclasses `langchain_openai:ChatOpenAI` for vLLM 0.19.0 OpenAI-compatible endpoints
 - Preserves vLLM's non-standard assistant `reasoning` field on full responses, streaming deltas, and follow-up tool-call turns
 - Designed for configs that enable thinking through `extra_body.chat_template_kwargs.enable_thinking` on vLLM 0.19.0 Qwen reasoning models, while accepting the older `thinking` alias
+- `cumulative_stream_usage` is an opt-in model setting (default `false`) for endpoints that repeat cumulative token totals on each streaming chunk. The provider converts snapshots to deltas only when a stable completion id is present, isolates interleaved streams by id, and leaves the original usage untouched otherwise. Per-model tracking is lock-protected and cleared on the trailing empty-`choices` frame whether or not that frame carries usage. A soft cap of 1024 ids evicts only entries idle for at least one hour; active streams may temporarily exceed the cap so eviction cannot corrupt their deltas. Regression coverage lives in `tests/test_vllm_provider.py`.
 
 ### IM Channels System (`app/channels/`)
 
@@ -810,7 +849,7 @@ The cached value is reused for both the blocking (`runs.wait`) and streaming (`_
 - Memory is stored per-user at `{base_dir}/users/{user_id}/memory.json`
 - Per-agent facts at `{base_dir}/users/{user_id}/agents/{agent_name}/facts/{sha256-prefix}/{fact-id}.md`, where the prefix is the first two hexadecimal characters of `SHA-256(fact_id)`; there is no per-agent `memory.json`
 - Custom agent definitions (`SOUL.md` + `config.yaml`) are also per-user at `{base_dir}/users/{user_id}/agents/{agent_name}/`. The legacy shared layout `{base_dir}/agents/{agent_name}/` remains read-only fallback for unmigrated installations
-- Middleware mode captures `user_id` via `get_effective_user_id()` at enqueue time; tool mode resolves `user_id` and `agent_name` from `ToolRuntime.context` via `resolve_runtime_user_id(runtime)` so tool calls stay scoped to the authenticated user and active custom agent
+- Middleware mode captures `user_id` via `resolve_runtime_user_id(runtime)` at enqueue time; tool mode resolves `user_id` and `agent_name` from `ToolRuntime.context` via the same helper so both Gateway and standalone LangGraph Server runs stay scoped to the authenticated user and active custom agent
 - The `/api/memory*` endpoints resolve the owner through `_resolve_memory_user_id(request)`: trusted internal callers (IM channel workers carrying the `X-DeerFlow-Owner-User-Id` header, e.g. a bound `/memory` command) act for the connection owner; browser/API callers fall back to `get_effective_user_id()`. The header is only honored after `AuthMiddleware` validated the internal token, mirroring `get_trusted_internal_owner_user_id` used by the threads router
 - In no-auth mode, `user_id` defaults to `"default"` (constant `DEFAULT_USER_ID`)
 - Absolute `storage_path` in config opts out of per-user isolation
@@ -827,9 +866,31 @@ The cached value is reused for both the blocking (`runs.wait`) and streaming (`_
 - **Repository**: `get/list/upsert/delete_fact`, `apply_changes`, summary operations, migration, index lifecycle/status, and scoped search. `apply_changes` and direct fact CRUD touch only target Markdown files; direct fact CRUD accepts separate expected user-memory and fact revisions. Supplied summary child keys merge over their persisted section, while import normalizes complete replacement sections first. Whole-document `load/save` remains for compatibility but validates the complete `facts` list and diffs it before persistence. An unscoped manager clear first migrates facts from unread legacy agent JSON without adopting potentially conflicting summaries, then removes the global summaries and every agent's canonical facts while preserving agent configuration; an explicit agent clear removes only that bucket's facts and preserves the shared summaries
 
 **Workflow**:
-- `memory.mode: middleware` (default) keeps the passive path: `MemoryMiddleware` filters messages (user inputs + final AI responses), captures `user_id` via `get_effective_user_id()`, queues conversation with the captured `user_id`, and the debounced background thread invokes the LLM to extract context updates and facts using the stored `user_id`.
+- `memory.mode: middleware` (default) keeps the passive path: `MemoryMiddleware` filters messages (user inputs + final AI responses), captures `user_id` via `resolve_runtime_user_id(runtime)`, queues conversation with the captured `user_id`, and the debounced background thread invokes the LLM to extract context updates and facts using the stored `user_id`. `DynamicContextMiddleware` passes the same resolved identity to the memory read path. On standalone Agent Server runs, server-owned auth identity is also resolved during lead-agent construction, normalized through `make_safe_user_id` for DeerFlow storage, and explicitly reused for custom-agent config/SOUL, user skills, skill policy, and prompt assembly; ordinary client `user_id` values cannot override `langgraph_auth_user_id`. On the embedded Gateway path, `inject_authenticated_user_context` removes client-supplied `langgraph_auth_user` / `langgraph_auth_user_id` from both RunnableConfig sections before graph construction, so those reserved fields cannot impersonate Agent Server auth.
+- The optional `openviking` backend under
+  `packages/harness/deerflow/agents/memory/backends/openviking/` is a
+  remote-only HTTP adapter. Select it with
+  `memory.manager_class: openviking` and keep `memory.mode: middleware`. It
+  commits filtered turns to OpenViking Sessions and maps remote memory search
+  results into the shared contract. It hashes `(user_id, agent_name)` into a
+  safe OpenViking trusted-user identity for hard scope isolation and keeps
+  bounded message watermarks below
+  `{storage_path}/openviking/sessions/`. The watermark combines a constant-size
+  ordered-prefix digest for append-only histories with a bounded recent-ID
+  fallback for compaction and separately records submitted and committed
+  progress. Once batch submission succeeds, a later update never resubmits
+  those messages or retries an ambiguous commit; a future batch can commit the
+  still-open Session together with new messages. Schema-v2 recent-ID
+  watermarks migrate without duplicating their anchored history. The shared
+  HTTP client has explicit total/keep-alive connection limits and jittered
+  exponential retry delays, and its configuration representation omits the API
+  key. Session locks are weakly cached, async entrypoints offload synchronous
+  HTTP and file IO, and graceful shutdown rejects new work before draining all
+  in-flight client operations within its timeout. It does not implement
+  DeerMem fact CRUD/import/export and must not import the OpenViking embedded
+  runtime.
 - `memory.mode: tool` skips `MemoryMiddleware` and registers `memory_search`, `memory_add`, `memory_update`, and `memory_delete` on the agent. The model decides when to search, add, update, or delete facts; this is opt-in/experimental and should not be described as better than middleware mode without eval evidence.
-- Both modes share `FileMemoryStorage`, per-user/per-agent isolation, prompt injection, manual CRUD primitives, and the updater backend.
+- Both modes share `FileMemoryStorage`, per-user/per-agent isolation, manual CRUD primitives, and the updater backend. Injection is mode-aware: middleware mode injects global `user`/`history` summaries plus the selected agent's facts, while tool mode injects only the global summaries and leaves every agent fact behind `memory_search` to avoid duplicating automatically injected and retrieval-returned context. `memory.injection_enabled: false` suppresses the complete block in either mode.
 - Middleware mode queue debounces (30s default), batches updates, and commits global summaries plus the selected/default agent's fact delta through a user-level lock, optimistic user-memory revisions, per-fact revisions, and a recoverable target-file journal. Only explicitly marked point operations may rebase a stale shared revision, and only while every addressed fact still satisfies its original absent/revision precondition. Snapshot-derived clear/trim/consolidation operations instead reload the complete document and recompute their intent on a manifest conflict, with a bounded retry. Typed manifest/fact conflict subclasses keep that decision independent of exception text, and same-ID creates and stale same-fact writes fail. Scope-lock objects are weakly cached so inactive users do not grow a process-lifetime map. Cache validation does not scale with the fact-file count: its token combines the shared JSON's `(mtime_ns, size, revision)`, so the persisted revision invalidates stale caches even when a coarse-mtime filesystem reports identical metadata for same-size writes; direct out-of-band Markdown edits require `reload()`. Atomic replacement also syncs the parent directory on POSIX so the rename is durable. DeerMem translates private storage conflict/corruption exceptions to the backend-neutral MemoryManager contract; the Gateway maps them to HTTP 409 and a stable HTTP 500 response respectively. A normal default-manager read automatically migrates legacy facts from the global JSON into `__default__`; it also adopts the earlier implicit `lead-agent` fact bucket only when that directory has no custom-agent `config.yaml`, and rejects unexpected files instead of deleting them. The v1-to-v2 migration is one-way for the running application: operators must stop DeerFlow and snapshot the configured storage root before upgrade. Before any destructive v2 write, every migrated JSON source is durably retained as `{manifest_filename}.v1.bak`; a missing-write or mismatched existing backup aborts without modifying v1 data. Legacy per-agent JSON is deleted only after its non-empty summaries are safely adopted or confirmed identical; summary conflicts keep the source file and fail loudly.
 - **Proactive Markdown migration CLI**: from `backend/`, run `PYTHONPATH=. python scripts/migrate_memory_markdown.py --all-users --dry-run` to audit and omit `--dry-run` to migrate before serving traffic. Use repeated `--user-id` values when selecting exact original identities, especially standalone raw IDs containing `@` or other characters that are normalized in directory names; `--storage-path` selects a non-default DeerMem root. The CLI reuses `FileMemoryStorage.migrate`, is idempotent, continues across per-user failures, and exits non-zero if any user fails. It is optional because the first normal read still performs the same migration automatically.
 - `retrieval_adapter` owns indexing and retrieval. `fts5` is the DeerMem default and uses a persistent derived SQLite index under `.retrieval/`; an empty value disables the adapter and selects `substring_fallback`. File storage sends upsert/remove notifications for normal writes and both explicit and lazy migrations after releasing durable storage locks, then delegates search. Gateway startup schedules `DeerMem.warm_retrieval()` as a background full rebuild so readiness is not delayed, while a first search lazily rebuilds its exact scope until warm-up completes. Individual malformed facts are logged and skipped without triggering repeated full scans; only a fatal adapter rebuild failure keeps lazy retry enabled. During shutdown, the Gateway waits at most one second for this derived rebuild and leaves the full configured timeout to the canonical memory flush; if the rebuild is still active, its adapter remains open until process exit. Adapter failures mark the scope dirty and fall back to canonical substring search until rebuilding succeeds. `FileMemoryStorage` owns and closes the adapter so higher layers do not reach into private storage state.
@@ -918,6 +979,7 @@ This invokes `alembic revision --autogenerate` against the live ORM models. Revi
 - `migrations/versions/0004_run_ownership.py` — `runs` multi-worker ownership + the `uq_runs_thread_active` partial unique index, with a `_dedupe_active_runs_per_thread()` pre-step so `CREATE UNIQUE INDEX` cannot fail on a field DB that already has duplicate active rows per thread
 - `migrations/versions/0007_scheduled_run_active_index.py` — the `uq_scheduled_task_run_active` partial unique index (at most one queued/running `scheduled_task_runs` row per `task_id`), with a `_dedupe_active_scheduled_runs_per_task()` pre-step (keeps the newest active row per task, supersedes the rest to `interrupted` with an explanatory `error` + `finished_at`) mirroring 0004; chains after `0006_agents`
 - `migrations/versions/0008_thread_operation_kind.py` — adds `runs.operation_kind` for durable non-run thread reservations; chains after `0007_scheduled_run_active_index`
+- `migrations/versions/0010_run_cancel_request.py` — adds the nullable `runs.cancel_action` / `cancel_requested_at` handoff used by non-owning workers; chains after `0009_webhook_dedupe`
 - `persistence/bootstrap.py` — `bootstrap_schema(engine, backend=...)`, the three-branch decision + locking
 - Tests: `tests/test_persistence_bootstrap.py` (branches), `tests/test_persistence_bootstrap_concurrency.py` (concurrency), `tests/test_persistence_bootstrap_regression.py` (issue #3682), `tests/test_persistence_migrations_env.py` (filter), `tests/blocking_io/test_persistence_bootstrap.py` (asyncio.to_thread anchor), `tests/test_migration_0004_run_ownership_dedupe.py` + `tests/test_migration_0007_scheduled_run_active_dedupe.py` (dedupe-before-unique-index pre-steps)
 
@@ -925,13 +987,17 @@ This invokes `alembic revision --autogenerate` against the live ORM models. Revi
 
 Checkpointer storage runs in one of two channel modes, selected by `checkpoint_channel_mode` in `config.yaml` (default `full`). `delta` mode adopts LangGraph 1.2's `DeltaChannel` for `messages`: checkpoints store a sentinel + per-step writes instead of the full message list, so storage/serde grows O(N) instead of O(N²) in turns. All checkpointer backends (memory/sqlite/postgres) serve both modes unchanged — the semantics live in the compiled graph's channel table, not in the saver.
 
-**Mode is process-frozen and restart-required.** `make_lead_agent` and the embedded `DeerFlowClient` freeze the resolved mode (`runtime/checkpoint_mode.py::freeze_checkpoint_channel_mode`) and the delta snapshot frequency (`agents/thread_state.py::freeze_delta_snapshot_frequency`, from the `checkpoint_delta_snapshot_frequency` config knob, default 1000 — restart-required like the mode) before compiling the graph with the mode-matched schema (`agents/thread_state.py::get_thread_state_schema`, plus `adapt_state_schema_for_mode` / `normalize_middleware_state_schemas` for middleware state). Adapted middleware schemas are cached by schema, mode, and resolved snapshot frequency so a pre-freeze ephemeral graph cannot leave a stale default-frequency schema behind. A second, different mode or frequency in the same process raises `CheckpointModeReconfigurationError`. To switch: edit config, restart.
+**Mode is process-frozen and restart-required.** `make_lead_agent` and the embedded `DeerFlowClient` freeze the resolved mode (`runtime/checkpoint_mode.py::freeze_checkpoint_channel_mode`) before compiling the graph with the mode-matched schema (`agents/thread_state.py::get_thread_state_schema`, plus `adapt_state_schema_for_mode` / `normalize_middleware_state_schemas` for middleware state). Adapted middleware schemas are cached by schema, mode, and resolved snapshot frequency so a pre-freeze ephemeral graph cannot leave a stale default-frequency schema behind. A second, different mode or frequency in the same process raises `CheckpointModeReconfigurationError`. To switch: edit config, restart.
+
+**Delta snapshot cadence is configurable but frozen with the mode.** `database.checkpoint_delta.snapshot_frequency` (default `10`) sets the `DeltaChannel` snapshot cadence. It is frozen alongside the mode (`freeze_checkpoint_snapshot_frequency`; non-positive direct inputs raise `ValueError`, while a frozen-value mismatch raises `CheckpointModeReconfigurationError`), restart-required, and must match across every process sharing one checkpoint database — the cadence lives in each compiled graph's channel table and is deliberately NOT stamped into checkpoint metadata, so the mode-compatibility marker and full -> delta migration semantics are unchanged. Schema helpers resolve it explicit-arg -> frozen -> default, and every schema/graph cache (`_delta_thread_state_schema`, `_adapt_state_schema_for_delta`, the client agent-config key, the gateway accessor-graph cache) keys on the resolved value.
+
+**Compiled-graph cache cap is configurable and hot-reloadable.** `database.checkpoint_graph_cache.accessor_graph_max` (default `64`) bounds the gateway accessor-graph cache, which clears wholesale at the cap. The cap is re-read on every eviction check (`resolve_checkpoint_graph_cache_max`), so a config.yaml reload takes effect without a restart — a size change never affects graph semantics, only eviction timing.
 
 **Compatibility is asymmetric and fail-closed.** Every checkpoint written in delta mode carries metadata marker `deerflow_checkpoint_channel_mode: "delta"` (injected via `inject_checkpoint_mode`; absence of marker = full, so pre-feature checkpoints need no migration). Before any state read/write, `ensure_checkpoint_mode_compatible` rejects a full-mode process opening a delta thread with `CheckpointModeMismatchError` (surfaced as HTTP 409 with the cause and thread id by the threads router; `CheckpointModeReconfigurationError` maps to 503) — a full-mode raw read of a delta blob would silently return empty/partial `messages`. The reverse direction is allowed: delta-mode processes read full checkpoints transparently (old full checkpoints seed the delta channel), so full → delta is the smooth migration path; delta → full requires materializing/converting the data first. Detection also honors upstream's `counters_since_delta_snapshot.messages` metadata, and an explicit config marker takes precedence over any ambient context value.
 
 **Never bypass `CheckpointStateAccessor` (`runtime/checkpoint_state.py`) for thread-state access.** It is the single choke point binding graph + checkpointer + mode: it injects the mode marker into configs, runs the compatibility check before every `get`/`update`/`history`, and returns materialized state (delta checkpoints lack `channel_values.messages` — raw `get_tuple` reads see a sentinel). Gateway `services.py` builds and passes the accessor; thread-owned reads (state/history/regeneration) must use `build_thread_checkpoint_state_accessor` so the recorded assistant's middleware schema materializes every channel. `history(limit)` semantics: `0` means zero items (explicit empty), `None` means unlimited — do not pass `limit=0` through to `graph.get_state_history`. Assistant metadata lookup is fail-closed for mutation accessors so a store outage cannot silently select the default schema and discard extension channels. In `full` mode the read path degrades to a raw checkpointer read (`_RawCheckpointReadAccessor`) when the agent factory cannot build the graph (bad model config, MCP outage) — full checkpoints carry complete `channel_values`, so reads don't need the graph; degraded snapshots take `created_at` from the standard checkpoint `ts` field, falling back to metadata only for compatibility. The delta gate still applies on the degraded path; `next`/`tasks` degrade to empty and thread status falls back to the stored status because task presence is not derivable, while delta mode has no fallback (materialization needs the channel table).
 
-**Replay checkpoint lookup prefers lineage and degrades only for an explicitly missing legacy parent link.** Branch and regenerate paths first walk `parent_config`, which prevents a global chronological scan from selecting a sibling created by regeneration. `CheckpointParentMissingError` alone enables the bounded newest-first history fallback in `app/gateway/checkpoint_lineage.py`; cycles, dangling/non-addressable parents, target mismatches, and depth exhaustion raise `CheckpointLineageIntegrityError` and fail closed instead of selecting a sibling. The compatibility scans request 400 raw checkpoints so up to 200 duration-only entries do not consume the effective branch-history budget; the fallback scans oldest-to-newest internally, skips duration-only checkpoints, and accepts only checkpoints with an addressable id as the replay base. A source history with no discoverable pre-user checkpoint preserves the historical single-checkpoint branch behavior instead of rejecting the branch; regeneration remains unavailable for that inherited response. Existing single-checkpoint branches are not mutated by regenerate preparation, and no raw checkpoint tuple is copied across threads because delta state depends on ancestry and pending writes. Regenerate source-run lookup uses the current thread's exact event, then the server-stamped `run_id` on the copied human message, then verified RunManager content matching; it does not read parent-thread events. Storage or checkpoint-mode failures are not treated as a missing base and still fail closed.
+**Replay checkpoint lookup prefers lineage and degrades only for an explicitly missing legacy parent link.** Branch and regenerate paths first walk `parent_config`, which prevents a global chronological scan from selecting a sibling created by regeneration. `CheckpointParentMissingError` alone enables the bounded newest-first history fallback in `app/gateway/checkpoint_lineage.py`; cycles, dangling/non-addressable parents, target mismatches, and depth exhaustion raise `CheckpointLineageIntegrityError` and fail closed instead of selecting a sibling. The compatibility scans request 400 raw checkpoints so up to 200 duration-only entries do not consume the effective branch-history budget; the fallback scans oldest-to-newest internally, skips duration-only checkpoints, and accepts only checkpoints with an addressable id as the replay base. A source history with no discoverable pre-user checkpoint preserves the historical single-checkpoint branch behavior instead of rejecting the branch; regeneration remains unavailable for that inherited response. Existing single-checkpoint branches are not mutated by regenerate preparation, and no raw checkpoint tuple is copied across threads because delta state depends on ancestry and pending writes. Regenerate source-run lookup uses the current thread's exact event, then the server-stamped `run_id` on the copied human message, then verified RunManager content matching; it does not read parent-thread events. When an interrupted response was streamed but never checkpointed, regeneration accepts only the latest visible human message's server-stamped `run_id` after verifying that it belongs to the same thread and still has `interrupted` status. Storage or checkpoint-mode failures are not treated as a missing base and still fail closed.
 
 **A delta-mode run cannot fork; `runtime/runs/worker.py` linearizes the resume instead.** Resuming from an older checkpoint (regenerate, or any client-supplied `checkpoint`) forks the lineage, and delta state for a fork is not materializable: `BaseCheckpointSaver.get_delta_channel_history` — and the bespoke overrides in `InMemorySaver`/`PostgresSaver` — collect **every** `pending_writes` entry stored on each on-path ancestor, but a shared parent also carries the writes of the sibling child that was abandoned. Those writes replay into the fork, so the run starts from a message list still containing the answer it was supposed to replace (#4458: regenerating in a branched thread showed the superseded assistant message beside the new one after a reload; reproduced on postgres, sqlite, and the in-memory saver). Write-to-child ownership belongs to the upstream delta contract, so DeerFlow does not reimplement the walk: `_linearize_delta_checkpoint_resume` materializes the requested checkpoint's complete state and writes every channel onto the **current head** (which has no siblings) through the state mutation graph, using `Overwrite` for reducer channels and resetting newer head-only channels to their schema default (or `None` when no constructible default exists); it then drops the `checkpoint_id` selector and lets the run proceed linearly, while the abandoned turn stays in history as the rewritten head's ancestry. The worker holds `_checkpoint_thread_lock` across `_capture_rollback_point` and the optional linear rewrite, making the rollback snapshot and rewrite atomic with graph streaming and the preceding run's duration-metadata checkpoint write. Capture preserves the complete real pre-run state; cancel-with-rollback then linearly replaces the current delta head with that captured state rather than forking the now-shared pre-run checkpoint, so the abandoned turn is restored without replaying the resume sibling's writes. The worker also recomputes the current-run message boundary from the rewritten state and fails closed (an unreadable resume checkpoint raises rather than falling back to the corrupt fork). `full` mode keeps forking — its checkpoints carry complete `channel_values` and need no replay — so LangGraph branching semantics are unchanged there. Root namespace only; subgraph namespaces are left alone.
 
@@ -940,10 +1006,10 @@ Checkpointer storage runs in one of two channel modes, selected by `checkpoint_c
 **Run rollback flow** (`runtime/runs/worker.py`): `_capture_rollback_point` materializes the complete pre-run state via the accessor and captures raw `pending_writes` via `aget_tuple` into an immutable `RollbackPoint` before the run starts — capture failure disables rollback (fail-closed), never restores partial state. In `full` mode, cancel-with-rollback forks from the pre-run checkpoint via the mutation graph and inherits non-message channels from that parent. In `delta` mode, forking is unsafe once the cancelled path has attached sibling writes to the pre-run checkpoint, so rollback replaces every captured channel on the current head, using `Overwrite` for reducers and schema defaults for current-head-only channels. Both modes reattach only the captured pre-run pending writes to the restored checkpoint. Edit replay runs (`metadata.replay_kind="edit"`) also restore the pre-run checkpoint on failed, timed-out, or interrupted completion and publish the restored `values` snapshot to the stream before `end`, so clients do not remain on a transient edited branch when the replay did not produce a successful replacement.
 
 **Where things live**:
-- `runtime/checkpoint_mode.py` — mode freeze, marker injection, delta detection, compatibility gate, both error types
+- `runtime/checkpoint_mode.py` — mode + snapshot-frequency freeze, marker injection, delta detection, compatibility gate, both error types
 - `runtime/checkpoint_state.py` — `CheckpointStateAccessor`, `build_state_mutation_graph`, `RollbackPoint`
 - `checkpoint_patches.py` (package root) — checkpoint-machinery patches: delta-history folding for `InMemorySaver` (delegating to the base walk), stable message IDs across materialization, upstream first-write drop fix, and `BinaryOperatorAggregate` unwrapping an `Overwrite` first write into an empty (MISSING) channel — Union-typed reducer channels (`sandbox`/`goal`/`todos`/`promoted`) have no constructible default, so a replace-style write into a fresh branch thread or a never-written channel stored the wrapper literally and crashed the next consumer (#4380; probe-guarded, stands down if upstream fixes it)
-- `agents/thread_state.py` — `ThreadState`/`DeltaThreadState`, `DELTA_MESSAGES_FIELD` (`DeltaChannel` whose `snapshot_frequency` resolves from the `checkpoint_delta_snapshot_frequency` config knob via `freeze_delta_snapshot_frequency`/`resolved_delta_snapshot_frequency`; 1000 is only the default), schema adaptation helpers
+- `agents/thread_state.py` — `ThreadState`/`DeltaThreadState`, `delta_messages_field` / `DELTA_MESSAGES_FIELD` (`DeltaChannel` at the configured `snapshot_frequency`, default 10), schema adaptation helpers
 - `runtime/context_compaction.py` — compaction via accessor + mutation graph (reference consumer)
 - Tests: `tests/test_checkpoint_mode.py` (freeze/detect/gate), `tests/test_checkpoint_state.py` (accessor/mutation graph), `tests/test_delta_channel_checkpointers.py` (saver parity), `tests/test_threads_checkpoint_mode.py`, `tests/test_gateway_checkpoint_mode.py` (dual-mode e2e parity), `tests/test_context_compaction.py` (mutation-graph write, no scheduling), `tests/test_run_worker_rollback.py`
 
@@ -956,8 +1022,9 @@ logical checkpoint/write bytes, SQLite DB/WAL/SHM footprint, reducer replay time
 and peak RSS as versioned JSONL. The controller alternates mode order and rejects
 performance data when paired modes materialize different state. Its default 1 GiB
 estimated cumulative full-payload cap skips both modes of an oversized pair when
-`full` is selected; intentional `--modes delta` diagnostics bypass this
-full-payload cap, so size those runs explicitly. Use `--allow-large-cases` only
+`full` is selected, including every delta cadence in a `--snapshot-frequencies`
+sweep; intentional `--modes delta` diagnostics bypass this full-payload cap, so
+size those runs explicitly. Use `--allow-large-cases` only
 on a provisioned machine. Duplicate CSV matrix values are ignored with a warning;
 use `--repetitions` for repeated samples. Summarize paired successful repetitions
 with `scripts/benchmark/checkpoint/summarize_channels.py` (all ratios are
@@ -985,8 +1052,8 @@ deterministic model, real `AsyncSqliteSaver`), then measure
 `GET /threads/{id}/state` and `POST /threads/{id}/history` through the real
 Gateway route stack in the same event loop (httpx ASGITransport), split into
 cold/warm accessor-graph-cache samples. It sweeps `snapshot_frequency`
-(config: `checkpoint_delta_snapshot_frequency`, process-frozen like the
-mode), pairs every delta frequency against the same full row, and fails both
+(config: `checkpoint_delta.snapshot_frequency`, process-frozen like the mode),
+pairs every delta frequency against the same full row, and fails both
 rows of a pair when materialized or wire digests diverge. Each case must have
 more than the two discarded warm-up turns, and SQLite DB/WAL/SHM sizes are
 captured while the saver is still open so they represent the online storage
@@ -1235,6 +1302,7 @@ Multi-file upload with automatic document conversion:
 - Duplicate filenames in a single upload request are auto-renamed with `_N` suffixes so later files do not truncate earlier files
 - Gateway HTTP uploads stage bytes as `.upload-*.part` files and atomically replace the destination only after size validation. These staging files are hidden from upload listings, agent upload context, and sandbox listing/search tools, and swept on Gateway startup if a hard crash leaves one behind.
 - Gateway HTTP upload/list/delete handlers offload filesystem work through `deerflow.utils.file_io.run_file_io`, a dedicated ContextVar-preserving file IO executor. Non-mounted sandbox uploads acquire sandboxes with `SandboxProvider.acquire_async()` and offload `read_bytes()` plus `sandbox.update_file()` together.
+- Mounted upload paths skip both sandbox acquisition and per-file synchronization. For AIO remote/provisioner deployments this requires an explicit, accurate `sandbox.thread_data_mounts: true`; omission preserves backend auto-detection.
 - Agent receives uploaded file list via `UploadsMiddleware`
 
 See [docs/FILE_UPLOAD.md](docs/FILE_UPLOAD.md) for details.

backend/app/gateway/app.py

@@ -10,7 +10,7 @@ from app.gateway.auth_disabled import warn_if_auth_disabled_enabled
 from app.gateway.auth_middleware import AuthMiddleware
 from app.gateway.browser_capability import ensure_browser_runtime_available
 from app.gateway.config import get_gateway_config
-from app.gateway.csrf_middleware import CSRFMiddleware, get_configured_cors_origins
+from app.gateway.csrf_middleware import CORS_EXPOSED_HEADERS, CSRFMiddleware, get_configured_cors_origins
 from app.gateway.deps import langgraph_runtime
 from app.gateway.routers import (
     agents,
@@ -228,7 +228,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
         from deerflow.agents.memory import get_memory_manager
 
         if startup_config.memory.enabled:
-            manager = get_memory_manager()
+            manager = await asyncio.to_thread(get_memory_manager)
             warm_retrieval = getattr(manager, "warm_retrieval", None)
             if callable(warm_retrieval):
                 retrieval_warm_task = asyncio.create_task(
@@ -252,7 +252,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
     try:
         from deerflow.agents.memory import get_memory_manager
 
-        manager = get_memory_manager()
+        manager = await asyncio.to_thread(get_memory_manager)
         warmed = await asyncio.wait_for(
             asyncio.to_thread(manager.warm),
             timeout=5,
@@ -411,7 +411,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
             if app_cfg.memory.enabled:
                 from deerflow.agents.memory import get_memory_manager
 
-                manager = get_memory_manager()
+                manager = await asyncio.to_thread(get_memory_manager)
                 flush_timeout = app_cfg.memory.shutdown_flush_timeout_seconds
                 completed = await asyncio.to_thread(manager.shutdown_flush, flush_timeout)
                 if completed:
@@ -539,7 +539,11 @@ This gateway provides runtime endpoints for agent runs plus custom endpoints for
 
     # CORS: the unified nginx endpoint is same-origin by default. Split-origin
     # browser clients must opt in with this explicit Gateway allowlist so CORS
-    # and CSRF origin checks share the same source of truth.
+    # and CSRF origin checks share the same source of truth. They also need the
+    # run id the Gateway returns in a non-safelisted response header; without
+    # exposing it the SDK never reports a created run, so a new thread keeps its
+    # placeholder route and every action gated on an established thread stays
+    # hidden until the page is reloaded.
     cors_origins = sorted(get_configured_cors_origins())
     if cors_origins:
         app.add_middleware(
@@ -548,6 +552,7 @@ This gateway provides runtime endpoints for agent runs plus custom endpoints for
             allow_credentials=True,
             allow_methods=["*"],
             allow_headers=["*"],
+            expose_headers=list(CORS_EXPOSED_HEADERS),
         )
 
     # Request trace correlation: when logging.enhance.enabled=true, bind one

backend/app/gateway/checkpoint_lineage.py

@@ -45,6 +45,23 @@ def is_duration_only_checkpoint(checkpoint_tuple: Any) -> bool:
     return isinstance(writes, dict) and "runtime_run_duration" in writes
 
 
+def has_pending_tasks(checkpoint_tuple: Any) -> bool:
+    """Return whether *checkpoint_tuple* still has graph work scheduled.
+
+    A replay base must be a state the thread was at rest in. A mid-run
+    checkpoint owns the writes of the node that was about to run, and resuming
+    from it replays them — re-adding the very turn the replay is meant to
+    replace. Message ids alone cannot detect this, because middleware may
+    rewrite a message's id in the same run that produced it.
+
+    ``next`` is not derivable on the degraded raw-checkpoint read path, which
+    reports no tasks at all. Absence of evidence therefore stays permissive:
+    those reads keep selecting the same base they always did.
+    """
+
+    return bool(getattr(checkpoint_tuple, "next", None))
+
+
 def _message_id(message: Any) -> str | None:
     value = getattr(message, "id", None)
     if value is None and isinstance(message, dict):
@@ -96,7 +113,8 @@ async def find_checkpoint_before_message(
     Following ``parent_config`` is important after a regenerate: a thread can contain
     sibling checkpoint branches, and a global time-ordered scan can otherwise select
     a checkpoint from the wrong branch. Duration-only metadata checkpoints do not
-    represent an addressable conversation state and are skipped.
+    represent an addressable conversation state and are skipped, and so are
+    checkpoints that still have pending tasks (see :func:`has_pending_tasks`).
     """
 
     if message_id not in {_message_id(message) for message in checkpoint_messages(head_checkpoint)}:
@@ -132,7 +150,7 @@ async def find_checkpoint_before_message(
             continue
 
         parent_message_ids = {_message_id(message) for message in checkpoint_messages(parent)}
-        if message_id not in parent_message_ids:
+        if message_id not in parent_message_ids and not has_pending_tasks(parent):
             return parent
         current = parent
 
@@ -148,8 +166,9 @@ def find_checkpoint_before_message_chronologically(
     This is a compatibility fallback for imported or legacy checkpoints that do
     not carry ``parent_config`` links. Callers must prefer the lineage walk when
     links are available because a chronological scan cannot distinguish sibling
-    checkpoint branches. Duration-only checkpoints are ignored, and only
-    checkpoints with an addressable id can become the replay base.
+    checkpoint branches. Duration-only checkpoints are ignored, and only settled
+    checkpoints (see :func:`has_pending_tasks`) with an addressable id can become
+    the replay base.
     """
 
     previous_checkpoint = None
@@ -159,6 +178,6 @@ def find_checkpoint_before_message_chronologically(
         message_ids = {_message_id(message) for message in checkpoint_messages(checkpoint_tuple)}
         if message_id in message_ids:
             return previous_checkpoint, True
-        if checkpoint_configurable(checkpoint_tuple).get("checkpoint_id"):
+        if checkpoint_configurable(checkpoint_tuple).get("checkpoint_id") and not has_pending_tasks(checkpoint_tuple):
             previous_checkpoint = checkpoint_tuple
     return None, False

backend/app/gateway/csrf_middleware.py

@@ -122,6 +122,13 @@ def get_configured_cors_origins() -> set[str]:
     return _configured_cors_origins()
 
 
+# Response headers a split-origin browser client must be able to read. Only the
+# CORS-safelisted set is visible to JS by default, and the created run's id
+# travels in `Content-Location` — the LangGraph SDK resolves run metadata from
+# it, so withholding it leaves such a client unable to learn its own run id.
+CORS_EXPOSED_HEADERS: tuple[str, ...] = ("Content-Location",)
+
+
 def _first_header_value(value: str | None) -> str | None:
     """Return the first value from a comma-separated proxy header."""
     if not value:

backend/app/gateway/deps.py

@@ -371,7 +371,7 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
     """
     from deerflow.persistence.engine import close_engine, get_session_factory, init_engine_from_config
     from deerflow.runtime import make_store, make_stream_bridge
-    from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode
+    from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode, freeze_checkpoint_snapshot_frequency
     from deerflow.runtime.checkpointer.async_provider import make_checkpointer
     from deerflow.runtime.events.store import make_run_event_store
 
@@ -387,6 +387,7 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
     async with AsyncExitStack() as stack:
         config = startup_config
         app.state.checkpoint_channel_mode = freeze_checkpoint_channel_mode(config.database.checkpoint_channel_mode)
+        app.state.checkpoint_snapshot_frequency = freeze_checkpoint_snapshot_frequency(config.database.checkpoint_delta.snapshot_frequency)
 
         app.state.stream_bridge = await stack.enter_async_context(make_stream_bridge(config))
 
@@ -597,6 +598,7 @@ def get_run_context(request: Request) -> RunContext:
         event_store=get_run_event_store(request),
         run_events_config=getattr(request.app.state, "run_events_config", None),
         checkpoint_channel_mode=getattr(request.app.state, "checkpoint_channel_mode", "full"),
+        checkpoint_snapshot_frequency=getattr(request.app.state, "checkpoint_snapshot_frequency", None),
         thread_store=get_thread_store(request),
         app_config=get_config(),
         on_run_completed=getattr(request.app.state, "scheduled_task_service", None).handle_run_completion if getattr(request.app.state, "scheduled_task_service", None) is not None else None,

backend/app/gateway/routers/integrations.py

@@ -78,7 +78,7 @@ class LarkIntegrationStatusResponse(BaseModel):
     install_path: str = Field(..., description="Host path of the managed Lark skill pack")
     cli: LarkCliProbeResponse
     auth: LarkAuthProbeResponse
-    sandbox_runtime_mode: str = Field("none", description="How lark-cli is provisioned into the sandbox: none, gateway-download, or init-container")
+    sandbox_runtime_mode: str = Field("none", description="How lark-cli is provisioned into the sandbox: none, gateway-download, init-container, or broker")
     sandbox_runtime_ready: bool = Field(False, description="Whether the sandbox lark-cli runtime is provisioned and usable at chat time")
     sandbox_runtime_detail: str | None = Field(None, description="Human-readable reason when the sandbox runtime is not ready")
 

backend/app/gateway/routers/memory.py

@@ -1,5 +1,6 @@
 """Memory API router for retrieving and managing global memory data."""
 
+import asyncio
 from typing import Any, Literal
 
 from fastapi import APIRouter, HTTPException, Request
@@ -146,7 +147,7 @@ def _unsupported_501(manager: object, label: str) -> HTTPException:
     )
 
 
-def _get_memory_or_501(manager: MemoryManager, user_id: str, label: str) -> dict[str, Any]:
+async def _get_memory_or_501(manager: MemoryManager, user_id: str, label: str) -> dict[str, Any]:
     """Read the full memory doc; 501 if the backend doesn't expose one.
 
     ``get_memory`` is tier-2 (default ``raise NotImplementedError``); a minimal
@@ -157,7 +158,7 @@ def _get_memory_or_501(manager: MemoryManager, user_id: str, label: str) -> dict
     endpoint's verb, e.g. "get memory" / "export memory" / "reload memory").
     """
     try:
-        return manager.get_memory(user_id=user_id)
+        return await asyncio.to_thread(manager.get_memory, user_id=user_id)
     except NotImplementedError:
         raise _unsupported_501(manager, label) from None
     except (MemoryConflictError, MemoryCorruptionError) as exc:
@@ -239,8 +240,8 @@ async def get_memory(http_request: Request) -> MemoryResponse:
         }
         ```
     """
-    manager = get_memory_manager()
-    memory_data = _get_memory_or_501(manager, _resolve_memory_user_id(http_request), "get memory")
+    manager = await asyncio.to_thread(get_memory_manager)
+    memory_data = await _get_memory_or_501(manager, _resolve_memory_user_id(http_request), "get memory")
     return MemoryResponse(**memory_data)
 
 
@@ -261,9 +262,9 @@ async def reload_memory(http_request: Request) -> MemoryResponse:
         The reloaded memory data.
     """
     user_id = _resolve_memory_user_id(http_request)
-    manager = get_memory_manager()
+    manager = await asyncio.to_thread(get_memory_manager)
     try:
-        memory_data = manager.reload_memory(user_id=user_id)
+        memory_data = await asyncio.to_thread(manager.reload_memory, user_id=user_id)
     except NotImplementedError:
         # Non-DeerMem backends have no reload concept; fall back to get_memory
         # (read-only refresh, so degrading is safe and still useful -- vs fact
@@ -271,7 +272,7 @@ async def reload_memory(http_request: Request) -> MemoryResponse:
         # would hide data loss). If get_memory is also unsupported (a minimal
         # backend with no full doc), surface 501 rather than a raw 500: reads
         # degrade only when there is a doc to degrade to.
-        memory_data = _get_memory_or_501(manager, user_id, "reload memory")
+        memory_data = await _get_memory_or_501(manager, user_id, "reload memory")
     except (MemoryConflictError, MemoryCorruptionError) as exc:
         raise _map_memory_manager_error(exc) from exc
     return MemoryResponse(**memory_data)
@@ -286,9 +287,9 @@ async def reload_memory(http_request: Request) -> MemoryResponse:
 )
 async def clear_memory(http_request: Request) -> MemoryResponse:
     """Clear all persisted memory data."""
-    manager = get_memory_manager()
+    manager = await asyncio.to_thread(get_memory_manager)
     try:
-        memory_data = manager.clear_memory(user_id=_resolve_memory_user_id(http_request))
+        memory_data = await asyncio.to_thread(manager.clear_memory, user_id=_resolve_memory_user_id(http_request))
     except NotImplementedError:
         raise _unsupported_501(manager, "clear memory") from None
     except (MemoryConflictError, MemoryCorruptionError) as exc:
@@ -308,9 +309,10 @@ async def clear_memory(http_request: Request) -> MemoryResponse:
 )
 async def create_memory_fact_endpoint(request: FactCreateRequest, http_request: Request) -> MemoryResponse:
     """Create a single fact manually."""
-    manager = get_memory_manager()
+    manager = await asyncio.to_thread(get_memory_manager)
     try:
-        memory_data, fact_id = manager.create_fact(
+        memory_data, fact_id = await asyncio.to_thread(
+            manager.create_fact,
             content=request.content,
             category=request.category,
             confidence=request.confidence,
@@ -340,9 +342,9 @@ async def create_memory_fact_endpoint(request: FactCreateRequest, http_request:
 )
 async def delete_memory_fact_endpoint(fact_id: str, http_request: Request) -> MemoryResponse:
     """Delete a single fact from memory by fact id."""
-    manager = get_memory_manager()
+    manager = await asyncio.to_thread(get_memory_manager)
     try:
-        memory_data = manager.delete_fact(fact_id, user_id=_resolve_memory_user_id(http_request))
+        memory_data = await asyncio.to_thread(manager.delete_fact, fact_id, user_id=_resolve_memory_user_id(http_request))
     except NotImplementedError:
         raise _unsupported_501(manager, "delete fact") from None
     except KeyError as exc:
@@ -364,9 +366,10 @@ async def delete_memory_fact_endpoint(fact_id: str, http_request: Request) -> Me
 )
 async def update_memory_fact_endpoint(fact_id: str, request: FactPatchRequest, http_request: Request) -> MemoryResponse:
     """Partially update a single fact manually."""
-    manager = get_memory_manager()
+    manager = await asyncio.to_thread(get_memory_manager)
     try:
-        memory_data = manager.update_fact(
+        memory_data = await asyncio.to_thread(
+            manager.update_fact,
             fact_id=fact_id,
             content=request.content,
             category=request.category,
@@ -396,8 +399,8 @@ async def update_memory_fact_endpoint(fact_id: str, request: FactPatchRequest, h
 )
 async def export_memory(http_request: Request) -> MemoryResponse:
     """Export the current memory data."""
-    manager = get_memory_manager()
-    memory_data = _get_memory_or_501(manager, _resolve_memory_user_id(http_request), "export memory")
+    manager = await asyncio.to_thread(get_memory_manager)
+    memory_data = await _get_memory_or_501(manager, _resolve_memory_user_id(http_request), "export memory")
     return MemoryResponse(**memory_data)
 
 
@@ -410,9 +413,13 @@ async def export_memory(http_request: Request) -> MemoryResponse:
 )
 async def import_memory(request: MemoryResponse, http_request: Request) -> MemoryResponse:
     """Import and persist memory data."""
-    manager = get_memory_manager()
+    manager = await asyncio.to_thread(get_memory_manager)
     try:
-        memory_data = manager.import_memory(request.model_dump(exclude_none=True), user_id=_resolve_memory_user_id(http_request))
+        memory_data = await asyncio.to_thread(
+            manager.import_memory,
+            request.model_dump(exclude_none=True),
+            user_id=_resolve_memory_user_id(http_request),
+        )
     except NotImplementedError:
         raise _unsupported_501(manager, "import memory") from None
     except (MemoryConflictError, MemoryCorruptionError) as exc:
@@ -486,8 +493,8 @@ async def get_memory_status(http_request: Request) -> MemoryStatusResponse:
         Combined memory configuration and current data.
     """
     config = get_memory_config()
-    manager = get_memory_manager()
-    memory_data = _get_memory_or_501(manager, _resolve_memory_user_id(http_request), "get memory status")
+    manager = await asyncio.to_thread(get_memory_manager)
+    memory_data = await _get_memory_or_501(manager, _resolve_memory_user_id(http_request), "get memory status")
 
     return MemoryStatusResponse(
         config=MemoryConfigResponse(

backend/app/gateway/routers/suggestions.py

@@ -8,6 +8,7 @@ import deerflow.utils.llm_text as llm_text
 from app.gateway.authz import require_permission
 from app.gateway.deps import get_config
 from deerflow.config.app_config import AppConfig
+from deerflow.config.suggestions_config import DEFAULT_MAX_SUGGESTIONS, MAX_SUGGESTIONS_LIMIT
 from deerflow.utils.oneshot_llm import run_oneshot_llm
 
 logger = logging.getLogger(__name__)
@@ -22,7 +23,7 @@ class SuggestionMessage(BaseModel):
 
 class SuggestionsRequest(BaseModel):
     messages: list[SuggestionMessage] = Field(..., description="Recent conversation messages")
-    n: int = Field(default=3, ge=1, le=5, description="Number of suggestions to generate")
+    n: int = Field(default=DEFAULT_MAX_SUGGESTIONS, ge=1, le=MAX_SUGGESTIONS_LIMIT, description="Number of suggestions to generate")
     model_name: str | None = Field(default=None, description="Optional model override")
 
 
@@ -32,6 +33,7 @@ class SuggestionsResponse(BaseModel):
 
 class SuggestionsConfigResponse(BaseModel):
     enabled: bool = Field(..., description="Whether follow-up suggestions are enabled globally")
+    max_suggestions: int = Field(..., ge=1, le=MAX_SUGGESTIONS_LIMIT, description="Maximum number of follow-up suggestions to generate")
 
 
 _strip_markdown_code_fence = llm_text.strip_markdown_code_fence
@@ -76,6 +78,10 @@ def _format_conversation(messages: list[SuggestionMessage]) -> str:
     return "\n".join(parts).strip()
 
 
+def _configured_max_suggestions(config: AppConfig) -> int:
+    return getattr(config.suggestions, "max_suggestions", DEFAULT_MAX_SUGGESTIONS)
+
+
 @router.get(
     "/suggestions/config",
     response_model=SuggestionsConfigResponse,
@@ -85,7 +91,7 @@ def _format_conversation(messages: list[SuggestionMessage]) -> str:
 async def get_suggestions_config(
     config: AppConfig = Depends(get_config),
 ) -> SuggestionsConfigResponse:
-    return SuggestionsConfigResponse(enabled=config.suggestions.enabled)
+    return SuggestionsConfigResponse(enabled=config.suggestions.enabled, max_suggestions=_configured_max_suggestions(config))
 
 
 @router.post(
@@ -106,7 +112,7 @@ async def generate_suggestions(
     if not body.messages:
         return SuggestionsResponse(suggestions=[])
 
-    n = body.n
+    n = min(body.n, _configured_max_suggestions(config))
     conversation = _format_conversation(body.messages)
     if not conversation:
         return SuggestionsResponse(suggestions=[])

backend/app/gateway/routers/thread_runs.py

@@ -38,7 +38,9 @@ from app.gateway.pagination import trim_run_message_page
 from app.gateway.run_models import RunCreateRequest
 from app.gateway.services import build_checkpoint_state_accessor, build_thread_checkpoint_state_accessor, sse_consumer, start_run, wait_for_run_completion
 from app.gateway.utils import sanitize_log_param
+from deerflow.agents.middlewares.dynamic_context_middleware import strip_injected_user_message_id_suffix
 from deerflow.runtime import CancelOutcome, RunRecord, RunStatus, serialize_channel_values_for_api
+from deerflow.runtime.secret_context import redact_config_secrets, redact_metadata_secrets
 from deerflow.utils.messages import ORIGINAL_USER_CONTENT_KEY, get_original_user_content_text, message_to_text
 from deerflow.workspace_changes import get_workspace_changes_response
 
@@ -76,6 +78,41 @@ def compute_run_durations(runs) -> dict[str, int]:
     return durations
 
 
+def stamp_turn_duration_on_last_ai(messages, run_durations: dict[str, int]) -> None:
+    """Attach each run's elapsed seconds to that run's final visible AI message only.
+
+    ``turn_duration`` is the run's wall-clock lifetime (``compute_run_durations``),
+    not model thinking time — it belongs to the run, not to individual messages.
+    Stamping every AI message made the UI repeat the same number once per
+    message and let tool-wait time read as thinking latency (#4152).
+    Middleware-caller messages (e.g. title generation) are skipped so the badge
+    lands on the assistant's actual final answer.
+
+    Accepts both message shapes that carry ``run_id``: event-store rows, which
+    wrap the message payload in a ``content`` dict, and flat serialized
+    checkpoint messages (``/history``), where the payload is the row itself.
+
+    The middleware skip is only effective on the event-store shape: checkpoint
+    messages replayed on ``/history`` never carry ``metadata.caller`` (they are
+    plain serialized LangChain messages), so this skip is inert there. That is
+    not a gap in practice — middleware writes (e.g. title generation) go to
+    thread metadata, not the ``messages`` channel, so no middleware message
+    reaches a checkpoint's ``messages`` list to begin with.
+    """
+    stamped: set[str] = set()
+    for msg in reversed(messages):
+        rid = msg.get("run_id")
+        if not rid or rid in stamped or rid not in run_durations:
+            continue
+        content = msg.get("content")
+        payload = content if isinstance(content, dict) else msg
+        metadata = msg.get("metadata") or {}
+        is_middleware = str(metadata.get("caller", "")).startswith("middleware:")
+        if payload.get("type") == "ai" and not is_middleware:
+            payload.setdefault("additional_kwargs", {})["turn_duration"] = run_durations[rid]
+            stamped.add(rid)
+
+
 # ---------------------------------------------------------------------------
 # Request / response models
 # ---------------------------------------------------------------------------
@@ -213,13 +250,17 @@ async def _raise_lease_valid_elsewhere(
 
 
 def _record_to_response(record: RunRecord) -> RunResponse:
+    kwargs = dict(record.kwargs or {})
+    if "config" in kwargs:
+        kwargs["config"] = redact_config_secrets(kwargs["config"])
+
     return RunResponse(
         run_id=record.run_id,
         thread_id=record.thread_id,
         assistant_id=record.assistant_id,
         status=record.status.value,
-        metadata=record.metadata,
-        kwargs=record.kwargs,
+        metadata=redact_metadata_secrets(record.metadata),
+        kwargs=kwargs,
         multitask_strategy=record.multitask_strategy,
         created_at=record.created_at,
         updated_at=record.updated_at,
@@ -339,7 +380,12 @@ def _clean_human_message_for_regenerate(message: Any) -> dict[str, Any]:
         "content": [{"type": "text", "text": content}],
         "additional_kwargs": additional_kwargs,
     }
-    message_id = _message_id(message)
+    # Replay the id the client originally sent. The dynamic-context reminder
+    # re-keys the first user message of a thread to `{id}__user`, and replaying
+    # that persisted id into a state that has no reminder yet makes the
+    # middleware treat the turn as already injected, silently dropping the date
+    # and memory block the original turn had.
+    message_id = strip_injected_user_message_id_suffix(_message_id(message))
     if message_id:
         clean_message["id"] = message_id
     name = _message_name(message)
@@ -371,6 +417,11 @@ def _is_terminal_assistant_text_message(message: Any) -> bool:
     return _is_visible_ai_message(message) and bool(_message_text(message).strip()) and not _message_tool_calls(message)
 
 
+def _has_title(values: dict[str, Any]) -> bool:
+    title = values.get("title")
+    return isinstance(title, str) and bool(title)
+
+
 def _has_active_goal(snapshot: Any) -> bool:
     goal = _checkpoint_values(snapshot).get("goal")
     return isinstance(goal, dict) and goal.get("status") == "active"
@@ -541,6 +592,33 @@ async def _require_successful_source_run(thread_id: str, run_id: str, request: R
     return record
 
 
+async def _find_interrupted_target_run_id(
+    thread_id: str,
+    source_human: Any,
+    request: Request,
+) -> str | None:
+    source_run_id = _message_additional_kwargs(source_human).get("run_id")
+    if not isinstance(source_run_id, str) or not source_run_id:
+        return None
+
+    run_mgr = get_run_manager(request)
+    user_id = await get_current_user(request)
+    record = await run_mgr.get(source_run_id, user_id=user_id)
+    if record is None:
+        records = await run_mgr.list_by_thread(thread_id, user_id=user_id, limit=20)
+        record = next(
+            (candidate for candidate in records if getattr(candidate, "run_id", None) == source_run_id),
+            None,
+        )
+    if record is None:
+        return None
+    if getattr(record, "thread_id", None) != thread_id:
+        return None
+    if _run_status_value(record) != RunStatus.interrupted.value:
+        return None
+    return source_run_id
+
+
 async def _prepare_regenerate_payload(thread_id: str, message_id: str, request: Request) -> RegeneratePrepareResponse:
     accessor, latest_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id)
     try:
@@ -555,18 +633,41 @@ async def _prepare_regenerate_payload(thread_id: str, message_id: str, request:
     messages = _checkpoint_messages(latest_checkpoint)
     target_index = next((i for i, message in enumerate(messages) if _message_id(message) == message_id), None)
     if target_index is None:
-        raise HTTPException(status_code=404, detail=f"Message {message_id} not found")
-    target_message = messages[target_index]
-    if not _is_visible_ai_message(target_message):
-        raise HTTPException(status_code=409, detail="Only visible assistant messages can be regenerated")
+        # A response interrupted during an LLM call can be visible in the live
+        # stream without ever reaching a checkpoint. The server-stamped run ID
+        # on the latest user message is the durable link to that partial turn.
+        previous_human = next(
+            (message for message in reversed(messages) if _is_visible_human_message(message)),
+            None,
+        )
+        target_run_id = await _find_interrupted_target_run_id(thread_id, previous_human, request) if previous_human is not None else None
+        if target_run_id is None:
+            raise HTTPException(status_code=404, detail=f"Message {message_id} not found")
+    else:
+        target_message = messages[target_index]
+        if not _is_visible_ai_message(target_message):
+            raise HTTPException(status_code=409, detail="Only visible assistant messages can be regenerated")
 
-    latest_visible_ai = next((message for message in reversed(messages) if _is_visible_ai_message(message)), None)
-    if _message_id(latest_visible_ai) != message_id:
-        raise HTTPException(status_code=409, detail="Only the latest assistant message can be regenerated")
+        latest_visible_ai = next((message for message in reversed(messages) if _is_visible_ai_message(message)), None)
+        if _message_id(latest_visible_ai) != message_id:
+            raise HTTPException(status_code=409, detail="Only the latest assistant message can be regenerated")
 
-    previous_human = next((message for message in reversed(messages[:target_index]) if _is_visible_human_message(message)), None)
+        previous_human = next((message for message in reversed(messages[:target_index]) if _is_visible_human_message(message)), None)
+        target_run_id = (
+            await _find_target_run_id(
+                thread_id,
+                message_id,
+                target_message,
+                previous_human,
+                request,
+            )
+            if previous_human is not None
+            else None
+        )
     if previous_human is None:
         raise HTTPException(status_code=409, detail="Could not find the user message for this assistant response")
+    if target_run_id is None:
+        raise HTTPException(status_code=409, detail="Could not find source run for assistant message")
     previous_human_id = _message_id(previous_human)
     if not previous_human_id:
         raise HTTPException(status_code=409, detail="The source user message is missing an id")
@@ -577,13 +678,6 @@ async def _prepare_regenerate_payload(thread_id: str, message_id: str, request:
         request,
         head_checkpoint=latest_checkpoint,
     )
-    target_run_id = await _find_target_run_id(
-        thread_id,
-        message_id,
-        target_message,
-        previous_human,
-        request,
-    )
     checkpoint = _checkpoint_response(base_checkpoint_tuple)
     metadata = {
         "regenerate_from_message_id": message_id,
@@ -643,7 +737,12 @@ async def _prepare_edit_regenerate_payload(
     if not source_ai_id:
         raise HTTPException(status_code=409, detail="The source assistant message is missing an id")
 
-    base_checkpoint_tuple = await _find_base_checkpoint_before_human(thread_id, source_human_id, request)
+    base_checkpoint_tuple = await _find_base_checkpoint_before_human(
+        thread_id,
+        source_human_id,
+        request,
+        head_checkpoint=latest_checkpoint,
+    )
     target_run_id = await _find_target_run_id(thread_id, source_ai_id, source_ai, source_human, request)
     source_record = await _require_successful_source_run(thread_id, target_run_id, request)
     checkpoint = _checkpoint_response(base_checkpoint_tuple)
@@ -662,16 +761,28 @@ async def _prepare_edit_regenerate_payload(
         "edit_message_id": replacement_human_message_id,
         "edit_version_group_id": edit_version_group_id,
     }
+    edit_input: dict[str, Any] = {
+        "messages": [
+            _clean_human_message_for_edit(
+                source_human,
+                replacement_id=replacement_human_message_id,
+                replacement_text=normalized_text,
+            )
+        ]
+    }
+    base_values = base_checkpoint_tuple.values if isinstance(getattr(base_checkpoint_tuple, "values", None), dict) else {}
+    latest_values = latest_checkpoint.values if isinstance(latest_checkpoint.values, dict) else {}
+    latest_title = latest_values.get("title")
+    if _has_title(base_values) and isinstance(latest_title, str) and latest_title:
+        # The replay base can predate a manual rename, so replay the current
+        # title rather than letting checkpoint rollback restore the older one
+        # (#4457, the same rollback regenerate already guards against). An
+        # untitled base is deliberately left alone: it belongs to a thread the
+        # title middleware has not named yet, and pinning the current title
+        # there would keep a name generated from the prompt this edit replaced.
+        edit_input["title"] = latest_title
     return EditRegeneratePrepareResponse(
-        input={
-            "messages": [
-                _clean_human_message_for_edit(
-                    source_human,
-                    replacement_id=replacement_human_message_id,
-                    replacement_text=normalized_text,
-                )
-            ]
-        },
+        input=edit_input,
         checkpoint=checkpoint,
         metadata=metadata,
         target_run_id=target_run_id,
@@ -817,8 +928,8 @@ async def cancel_run(
     - wait=false: Return immediately with 202
 
     In multi-worker deployments, a cancel landing on a non-owning worker
-    can take over the run when the owner's lease has expired.  When the
-    lease is still valid a 409 + ``Retry-After`` header is returned.
+    durably notifies the owner when its lease is live, or takes over and
+    terminalizes the run when that lease has expired.
     """
     run_mgr = get_run_manager(request)
     record = await run_mgr.get(run_id)
@@ -827,15 +938,30 @@ async def cancel_run(
 
     outcome = await run_mgr.cancel(run_id, action=action)
 
-    # Success paths — the run was either cancelled locally or taken over
-    # from a dead worker.
-    if outcome in (CancelOutcome.cancelled, CancelOutcome.taken_over):
+    # Success paths — the run was cancelled locally, durably requested from
+    # a live owner, or taken over from a dead worker.
+    if outcome in (
+        CancelOutcome.cancelled,
+        CancelOutcome.requested,
+        CancelOutcome.taken_over,
+    ):
         if wait and record.task is not None:
             try:
                 await record.task
             except asyncio.CancelledError:
                 pass
             return Response(status_code=204)
+        if wait and outcome == CancelOutcome.requested:
+            bridge = get_stream_bridge(request)
+            if record.store_only and bridge.supports_cross_process:
+                completed = await wait_for_run_completion(
+                    bridge,
+                    record,
+                    request,
+                    run_mgr,
+                )
+                if completed:
+                    return Response(status_code=204)
         return Response(status_code=202)
 
     if outcome == CancelOutcome.lease_valid_elsewhere:
@@ -906,16 +1032,29 @@ async def stream_existing_run(
             # the client doesn't hang on an SSE subscription this worker can
             # never serve.
             return Response(status_code=202)
-        if outcome != CancelOutcome.cancelled:
+        if outcome not in (CancelOutcome.cancelled, CancelOutcome.requested):
             if outcome == CancelOutcome.lease_valid_elsewhere:
                 await _raise_lease_valid_elsewhere(run_id, run_mgr, record)
             raise HTTPException(status_code=409, detail=_cancel_conflict_detail(run_id, record))
+        if outcome == CancelOutcome.requested and record.store_only and not bridge.supports_cross_process:
+            # The request is durable, but this bridge cannot observe the
+            # owner's stream. Returning 202 is safer than hanging forever on
+            # a process-local subscription.
+            return Response(status_code=202)
         if wait and record.task is not None:
             try:
                 await record.task
             except (asyncio.CancelledError, Exception):
                 pass
             return Response(status_code=204)
+        if wait and outcome == CancelOutcome.requested:
+            completed = await wait_for_run_completion(
+                bridge,
+                record,
+                request,
+                run_mgr,
+            )
+            return Response(status_code=204 if completed else 202)
 
     return StreamingResponse(
         sse_consumer(bridge, record, request, run_mgr),
@@ -999,14 +1138,7 @@ async def list_thread_messages(
     run_durations = compute_run_durations(runs)
 
     if run_durations:
-        for msg in messages:
-            content = msg.get("content", {})
-            if isinstance(content, dict) and content.get("type") == "ai":
-                rid = msg.get("run_id")
-                if rid and rid in run_durations:
-                    if "additional_kwargs" not in content:
-                        content["additional_kwargs"] = {}
-                    content["additional_kwargs"]["turn_duration"] = run_durations[rid]
+        stamp_turn_duration_on_last_ai(messages, run_durations)
 
     return messages
 
@@ -1253,16 +1385,8 @@ async def list_run_messages(
         record = await run_mgr.get(run_id)
         if record:
             durations = compute_run_durations([record])
-            duration = durations.get(run_id)
-            if duration is not None:
-                for msg in reversed(data):
-                    content = msg.get("content")
-                    metadata = msg.get("metadata", {})
-                    is_middleware = str(metadata.get("caller", "")).startswith("middleware:")
-                    if isinstance(content, dict) and content.get("type") == "ai" and not is_middleware:
-                        if "additional_kwargs" not in content:
-                            content["additional_kwargs"] = {}
-                        content["additional_kwargs"]["turn_duration"] = duration
+            if durations:
+                stamp_turn_duration_on_last_ai(data, durations)
 
     return {"data": data, "has_more": has_more}
 
@@ -1285,7 +1409,23 @@ async def list_run_events(
     """
     event_store = get_run_event_store(request)
     types = event_types.split(",") if event_types else None
-    return await event_store.list_events(thread_id, run_id, event_types=types, task_id=task_id, limit=limit, after_seq=after_seq)
+    events = await event_store.list_events(
+        thread_id,
+        run_id,
+        event_types=types,
+        task_id=task_id,
+        limit=limit,
+        after_seq=after_seq,
+    )
+    return [
+        {
+            **event,
+            "metadata": redact_metadata_secrets(event.get("metadata")),
+        }
+        if isinstance(event, dict) and "metadata" in event
+        else event
+        for event in events
+    ]
 
 
 @router.get("/{thread_id}/runs/{run_id}/workspace-changes")

backend/app/gateway/routers/threads.py

@@ -65,6 +65,7 @@ from deerflow.runtime.goal import (
 from deerflow.runtime.journal import build_branch_history_seed_events
 from deerflow.runtime.runs.manager import ConflictError
 from deerflow.runtime.runs.worker import valid_duration_entry
+from deerflow.runtime.secret_context import redact_metadata_secrets
 from deerflow.runtime.user_context import get_effective_user_id
 from deerflow.utils.file_io import run_file_io
 from deerflow.utils.time import coerce_iso, now_iso
@@ -349,7 +350,14 @@ class ThreadDeleteResponse(BaseModel):
     message: str
 
 
-class ThreadResponse(BaseModel):
+class _MetadataRedactingResponse(BaseModel):
+    @field_validator("metadata", mode="before", check_fields=False)
+    @classmethod
+    def _redact_legacy_metadata_secret(cls, value: Any) -> Any:
+        return redact_metadata_secrets(value)
+
+
+class ThreadResponse(_MetadataRedactingResponse):
     """Response model for a single thread."""
 
     thread_id: str = Field(description="Unique thread identifier")
@@ -402,7 +410,7 @@ class ThreadSearchRequest(BaseModel):
         return v
 
 
-class ThreadStateResponse(BaseModel):
+class ThreadStateResponse(_MetadataRedactingResponse):
     """Response model for thread state."""
 
     values: dict[str, Any] = Field(default_factory=dict, description="Current channel values")
@@ -472,7 +480,7 @@ class ThreadCompactResponse(BaseModel):
     total_tokens: int = 0
 
 
-class HistoryEntry(BaseModel):
+class HistoryEntry(_MetadataRedactingResponse):
     """Single checkpoint history entry."""
 
     checkpoint_id: str
@@ -1304,11 +1312,6 @@ async def update_thread_state(thread_id: str, body: ThreadStateUpdateRequest, re
     )
 
 
-def _ai_message_lacks_duration(message: dict[str, Any]) -> bool:
-    additional_kwargs = message.get("additional_kwargs")
-    return message.get("type") == "ai" and (not isinstance(additional_kwargs, dict) or "turn_duration" not in additional_kwargs)
-
-
 def _checkpoint_run_durations(metadata: Any) -> dict[str, int]:
     raw_durations = metadata.get("run_durations") if isinstance(metadata, dict) else None
     if not isinstance(raw_durations, dict):
@@ -1316,16 +1319,6 @@ def _checkpoint_run_durations(metadata: Any) -> dict[str, int]:
     return {run_id: duration_seconds for run_id, duration_seconds in raw_durations.items() if valid_duration_entry(run_id, duration_seconds)}
 
 
-def _set_message_turn_duration(message: dict[str, Any], run_id: str, run_durations: dict[str, int]) -> None:
-    if message.get("type") != "ai" or run_id not in run_durations:
-        return
-    additional_kwargs = message.get("additional_kwargs")
-    if not isinstance(additional_kwargs, dict):
-        additional_kwargs = {}
-        message["additional_kwargs"] = additional_kwargs
-    additional_kwargs.setdefault("turn_duration", run_durations[run_id])
-
-
 @router.post("/{thread_id}/history", response_model=list[HistoryEntry])
 @require_permission("threads", "read", owner_check=True)
 async def get_thread_history(
@@ -1373,11 +1366,14 @@ async def get_thread_history(
                 if messages:
                     serialized_msgs = serialize_channel_values_for_api({"messages": messages}).get("messages", [])
                     try:
+                        from app.gateway.routers.thread_runs import stamp_turn_duration_on_last_ai
+
                         # Human messages define turn boundaries. New checkpoints
                         # carry the completed turns' durations in metadata, so the
                         # messages channel stays unchanged.
                         checkpoint_run_durations = _checkpoint_run_durations(metadata)
                         current_turn_run_id = None
+                        turn_run_ids: set[str] = set()
                         for msg in serialized_msgs:
                             if msg.get("type") == "human":
                                 additional_kwargs = msg.get("additional_kwargs")
@@ -1391,12 +1387,21 @@ async def get_thread_history(
                                 continue
 
                             msg.setdefault("run_id", current_turn_run_id)
-                            _set_message_turn_duration(msg, current_turn_run_id, checkpoint_run_durations)
+                            if msg.get("type") == "ai":
+                                turn_run_ids.add(current_turn_run_id)
 
-                        # Legacy checkpoints without duration metadata are
-                        # correlated once via event-store + run-manager, then
-                        # upgraded by a metadata-only checkpoint write.
-                        if any(_ai_message_lacks_duration(msg) for msg in serialized_msgs):
+                        # Stamp each run's duration on its last AI message only,
+                        # same as the live message endpoints — never every AI
+                        # message in a multi-message turn (#4152).
+                        stamp_turn_duration_on_last_ai(serialized_msgs, checkpoint_run_durations)
+
+                        # Runs referenced by this checkpoint's AI messages but
+                        # absent from checkpoint metadata are either legacy
+                        # (never migrated) or just completed. Correlate once via
+                        # event-store + run-manager, then upgrade by a
+                        # metadata-only checkpoint write.
+                        missing_run_ids = turn_run_ids - set(checkpoint_run_durations)
+                        if missing_run_ids:
                             from app.gateway.deps import get_run_event_store, get_run_manager
                             from app.gateway.routers.thread_runs import compute_run_durations
                             from deerflow.runtime.runs.worker import persist_run_durations
@@ -1431,7 +1436,8 @@ async def get_thread_history(
                                     run_id = msg_to_run.get(msg.get("id")) or current_turn_run_id
                                     if run_id:
                                         msg["run_id"] = run_id
-                                        _set_message_turn_duration(msg, run_id, run_durations)
+
+                                stamp_turn_duration_on_last_ai(serialized_msgs, run_durations)
 
                                 # Intentional, best-effort write-on-read migration:
                                 # persist legacy metadata after the response so the

backend/app/gateway/services.py

@@ -34,6 +34,7 @@ from app.gateway.utils import sanitize_log_param
 from deerflow.agents.middlewares.dynamic_context_middleware import _DYNAMIC_CONTEXT_REMINDER_KEY, _REMINDER_DATE_KEY
 from deerflow.agents.middlewares.view_image_middleware import _IMAGE_CONTEXT_MESSAGE_MARKER_KEY
 from deerflow.config.app_config import get_app_config
+from deerflow.config.database_config import resolve_checkpoint_graph_cache_max
 from deerflow.runtime import (
     END_SENTINEL,
     HEARTBEAT_SENTINEL,
@@ -61,7 +62,11 @@ from deerflow.runtime.checkpoint_mode import (
 from deerflow.runtime.checkpoint_state import graph_state_schema
 from deerflow.runtime.goal import goal_thread_lock
 from deerflow.runtime.runs.naming import resolve_root_run_name
-from deerflow.runtime.secret_context import redact_config_secrets
+from deerflow.runtime.secret_context import (
+    LegacyRunMetadataSecretError,
+    redact_config_secrets,
+    validate_run_metadata_secrets,
+)
 from deerflow.runtime.stream_modes import normalize_stream_modes
 from deerflow.runtime.user_context import reset_current_user, set_current_user
 from deerflow.utils.messages import ORIGINAL_USER_CONTENT_KEY
@@ -303,12 +308,21 @@ _CONTEXT_INTERNAL_CALLER_KEYS: frozenset[str] = frozenset({"non_interactive"})
 
 # Server-owned authorization identity fields. These must never be accepted from
 # client-supplied ``body.config.context`` or ``body.config.configurable``. They
-# are either produced by Gateway auth state or admitted from a separately
-# authenticated internal request channel.
-#   ``is_internal``       — derived from ``request.state.auth_source``
-#   ``authz_attributes`` — Phase 1A has no Gateway-side producer; always cleared.
-#   ``channel_user_id``  — accepted only from trusted internal ``body.context``.
-_SERVER_OWNED_AUTHZ_CONTEXT_KEYS: frozenset[str] = frozenset({"is_internal", "authz_attributes", "channel_user_id"})
+# are either produced by Gateway auth state, admitted from a separately
+# authenticated internal request channel, or reserved for LangGraph Server.
+#   ``is_internal``             — derived from ``request.state.auth_source``
+#   ``authz_attributes``        — Phase 1A has no Gateway-side producer; cleared.
+#   ``channel_user_id``         — accepted only from trusted internal context.
+#   ``langgraph_auth_user*``    — populated only by LangGraph Server auth.
+_SERVER_OWNED_AUTHZ_CONTEXT_KEYS: frozenset[str] = frozenset(
+    {
+        "is_internal",
+        "authz_attributes",
+        "channel_user_id",
+        "langgraph_auth_user",
+        "langgraph_auth_user_id",
+    }
+)
 
 # Keys forwarded from ``body.context`` into ``config['context']`` ONLY (the
 # runtime context that becomes ``ToolRuntime.context`` / ``runtime.context``),
@@ -677,23 +691,35 @@ def build_checkpoint_state_mutation_accessor(
 # Cache of factory-built accessor graphs. Accessor operations (aget_state /
 # aupdate_state) never execute graph nodes or middleware, so per-request
 # variations (user, model, skills) cannot affect materialization semantics;
-# the compiled graph is stable per (assistant_id, mode, app_config). The
+# the compiled graph is stable per (assistant_id, mode, snapshot_frequency,
+# app_config). The
 # factory and app_config identities are re-validated on every call so patched
 # factories take effect immediately and a config.yaml hot-reload (which
 # rebuilds the AppConfig object) never serves a stale compiled graph — the
 # cached reference keeps the old config alive, so id-reuse cannot produce a
-# false hit. Bounded: cleared when too many distinct assistants appear.
+# false hit. Bounded: cleared when too many distinct assistants appear. The
+# cap is configurable (database.checkpoint_graph_cache.accessor_graph_max)
+# and re-read on every eviction check, so a hot-reload takes effect without
+# a restart.
 _STATE_ACCESSOR_GRAPH_CACHE_MAX = 64
-_state_accessor_graph_cache: dict[tuple[str | None, str], tuple[Any, Any, Any]] = {}
+_state_accessor_graph_cache: dict[tuple[str | None, str, int | None], tuple[Any, Any, Any]] = {}
 
 
-def _state_accessor_graph(agent_factory: Any, assistant_id: str | None, mode: str, config: dict[str, Any]) -> Any:
+def _accessor_graph_cache_max(app_config: Any) -> int:
+    return resolve_checkpoint_graph_cache_max(
+        getattr(app_config, "database", None),
+        "accessor_graph_max",
+        _STATE_ACCESSOR_GRAPH_CACHE_MAX,
+    )
+
+
+def _state_accessor_graph(agent_factory: Any, assistant_id: str | None, mode: str, snapshot_frequency: int | None, config: dict[str, Any]) -> Any:
     app_config = (config.get("context") or {}).get("app_config")
-    key = (assistant_id, mode)
+    key = (assistant_id, mode, snapshot_frequency)
     cached = _state_accessor_graph_cache.get(key)
     if cached is not None and cached[0] is agent_factory and cached[1] is app_config:
         return cached[2]
-    if len(_state_accessor_graph_cache) >= _STATE_ACCESSOR_GRAPH_CACHE_MAX:
+    if len(_state_accessor_graph_cache) >= _accessor_graph_cache_max(app_config):
         _state_accessor_graph_cache.clear()
     graph = agent_factory(config=config)
     _state_accessor_graph_cache[key] = (agent_factory, app_config, graph)
@@ -798,7 +824,7 @@ def build_checkpoint_state_accessor(
 
     agent_factory = resolve_agent_factory(assistant_id)
     try:
-        graph = _state_accessor_graph(agent_factory, assistant_id, ctx.checkpoint_channel_mode, config)
+        graph = _state_accessor_graph(agent_factory, assistant_id, ctx.checkpoint_channel_mode, getattr(ctx, "checkpoint_snapshot_frequency", None), config)
     except Exception:
         if ctx.checkpoint_channel_mode != "full":
             # Delta materialization needs the graph's channel table; there is
@@ -978,6 +1004,14 @@ async def start_run(
     request : Request
         FastAPI request — used to retrieve singletons from ``app.state``.
     """
+    body_config = getattr(body, "config", None)
+    config_metadata = body_config.get("metadata") if isinstance(body_config, dict) else None
+    try:
+        validate_run_metadata_secrets(getattr(body, "metadata", None))
+        validate_run_metadata_secrets(config_metadata)
+    except LegacyRunMetadataSecretError as exc:
+        raise HTTPException(status_code=422, detail=str(exc)) from exc
+
     stream_modes = normalize_stream_modes(body.stream_mode)
     bridge = get_stream_bridge(request)
     run_mgr = get_run_manager(request)
@@ -1271,10 +1305,10 @@ async def sse_consumer(
             yield format_sse(entry.event, entry.data, event_id=entry.id or None)
 
     finally:
-        # store_only records are cross-worker runs hydrated from the RunStore; this
-        # worker holds no in-memory task/abort state for them, so run_mgr.cancel()
-        # cannot stop the task (it would 409). Skip on_disconnect cancellation for
-        # those and only act on runs this worker actually owns.
+        # store_only records are cross-worker observation handles. An explicit
+        # cancel-then-stream action has already persisted its request before
+        # subscribing; a plain join disconnect must not invent a new
+        # cancellation request. Only apply on_disconnect to locally-owned runs.
         if not gap_emitted and not record.store_only and record.status in (RunStatus.pending, RunStatus.running):
             if record.on_disconnect == DisconnectMode.cancel:
                 await run_mgr.cancel(record.run_id)

backend/docs/ARCHITECTURE.md

@@ -337,7 +337,7 @@ SKILL.md Format:
 │ ---                                                                      │
 │                                                                          │
 │ # Skill Instructions                                                     │
-│ Content injected into system prompt...                                   │
+│ Loaded on demand after discovery or explicit slash activation...         │
 └─────────────────────────────────────────────────────────────────────────┘
 ```
 

backend/docs/AUTH_DESIGN.md

@@ -219,7 +219,21 @@ agent 在 sandbox 内看到统一虚拟路径：
 /mnt/user-data/outputs
 ```
 
-`ThreadDataMiddleware` 使用 `get_effective_user_id()` 解析当前用户并生成线程路径。没有认证上下文时会落到 `default` 用户桶，主要用于内部调用、嵌入式 client 或无 HTTP 的本地执行路径。
+`ThreadDataMiddleware`、`UploadsMiddleware` 与 memory 读写路径统一使用
+`resolve_runtime_user_id(runtime)` 解析当前用户。当前 LangGraph runtime
+优先使用 server-owned 的 `runtime.server_info.user.identity`；旧版或缺少
+`server_info` 的 standalone 路径继续读取 server-owned 的
+`configurable.langgraph_auth_user_id`；Gateway 内嵌路径在没有 Agent Server
+认证身份时使用认证后注入的 `runtime.context.user_id`。LangGraph 允许
+`BaseUser.identity` 使用邮箱等任意非空字符串，因此 server-owned auth
+身份会先通过 `make_safe_user_id` 转换为稳定、抗碰撞且目录安全的 DeerFlow
+storage ID；Agent Server 自身用于 metadata 授权过滤的原始 identity 不变。
+这些通道都缺失时才回落到请求 ContextVar 和 `default` 用户桶，最后一级
+主要用于内部调用、嵌入式 client 或无 HTTP 的本地执行路径。
+
+lead-agent 工厂使用同一身份边界：Agent Server 的保留 auth 字段优先于普通
+`user_id`，并将解析结果显式传给 custom agent、SOUL、skills、skill policy
+与静态 prompt 构建，避免 graph 构建阶段和 middleware 执行阶段落入不同用户桶。
 
 ### Memory
 
@@ -376,6 +390,13 @@ Gateway 内嵌 runtime 路径由 `AuthMiddleware` 和 `CSRFMiddleware` 保护。
 
 - `@auth.authenticate` 校验 JWT cookie、CSRF、用户存在性和 `token_version`。
 - `@auth.on` 在写入 metadata 时注入 `user_id`，并在读路径返回 `{"user_id": current_user}` 过滤条件。
+- LangGraph Server 将认证结果写入运行配置的保留
+  `langgraph_auth_user` / `langgraph_auth_user_id` 字段；harness 消费该身份，
+  让 uploads、thread data 与 memory 在直连模式下继续使用正确的 per-user
+  文件桶。
+- Gateway 内嵌 runtime 不接受这两个保留字段：run config 组装后会从
+  `context` 与 `configurable` 同时剥离客户端传入值，再注入 Gateway
+  自己认证得到的 `runtime.context.user_id`，避免伪装成 Agent Server 身份。
 
 这保证 Gateway 路由和 LangGraph-compatible 直连模式使用同一 JWT 语义。
 

backend/docs/CONFIGURATION.md

@@ -413,6 +413,25 @@ sandbox:
 
 When using Docker development (`make docker-start`), DeerFlow starts the `provisioner` service only if this provisioner mode is configured. In local or plain Docker sandbox modes, `provisioner` is skipped.
 
+Remote/provisioner backends default to explicit file synchronization because
+DeerFlow cannot infer whether their `/mnt/user-data` mount points reference the
+same storage as the Gateway. When the deployment guarantees that both sides use
+the same thread user-data directories, opt out of that extra transfer:
+
+```yaml
+sandbox:
+  use: deerflow.community.aio_sandbox:AioSandboxProvider
+  provisioner_url: http://provisioner:8002
+  thread_data_mounts: true
+```
+
+Leave `thread_data_mounts` unset to retain backend auto-detection. Set it to
+`false` to force explicit synchronization even for a local container backend.
+Only set it to `true` after verifying the Gateway's
+`users/{user_id}/threads/{thread_id}/user-data` directory and the sandbox's
+`/mnt/user-data` are the same storage; a false positive skips synchronization
+and makes newly uploaded files unavailable inside the sandbox.
+
 See [Provisioner Setup Guide](../../docker/provisioner/README.md) for detailed configuration, prerequisites, and troubleshooting.
 
 **E2B Cloud Sandbox** (runs sandbox code in [E2B](https://e2b.dev) cloud micro-VMs):
@@ -608,13 +627,15 @@ skill_scan:
 Set `skill_scan.enabled: false` to disable only the deterministic analyzers. Safe archive extraction and the LLM-based skill scanner still run.
 
 **Per-Agent Skill Filtering**:
-Custom agents can restrict which skills they load by defining a `skills` field in their `config.yaml` (located at `workspace/agents/<agent_name>/config.yaml`):
-- **Omitted or `null`**: Loads all globally enabled skills (default fallback).
+Custom agents can restrict which skills they discover and activate by defining a `skills` field in their `config.yaml` (located at `workspace/agents/<agent_name>/config.yaml`):
+- **Omitted or `null`**: Makes all globally enabled skills available (default fallback).
 - **`[]` (empty list)**: Disables all skills for this specific agent.
-- **`["skill-name"]`**: Loads only the explicitly specified skills.
+- **`["skill-name"]`**: Makes only the explicitly specified skills available.
 
 This field is a discovery and activation allowlist; it does not activate every listed skill's `allowed-tools` policy when the agent is constructed. Use `tool_groups` to define the agent's baseline tools. A listed skill's policy applies only after slash activation or an actual `SKILL.md` load.
 
+The same semantics apply to `subagents.agents.<name>.skills` and `subagents.custom_agents.<name>.skills`: omitted or `null` exposes all enabled skills, `[]` exposes none, and a list limits discovery and activation. A passive subagent skill never removes baseline tools; its `allowed-tools` declaration becomes active only after slash activation or a completed `SKILL.md` read.
+
 ### Title Generation
 
 Automatic conversation title generation:

backend/docs/FILE_UPLOAD.md

@@ -154,7 +154,9 @@ read_file(path="/mnt/user-data/uploads/document.md")
 上传流程采用“线程目录优先”策略：
 - 先写入 `backend/.deer-flow/threads/{thread_id}/user-data/uploads/` 作为权威存储
 - 本地沙箱（`sandbox_id=local`）直接使用线程目录内容
-- 非本地沙箱会额外同步到 `/mnt/user-data/uploads/*`，确保运行时可见
+- 默认情况下，非本地沙箱通过 `acquire_async` 获取后，再额外同步到 `/mnt/user-data/uploads/*`，确保运行时可见
+- 如果 Gateway 与远端沙箱保证挂载同一份线程 user-data（例如正确对齐的共享 PVC、NFS 或 hostPath），可设置 `sandbox.thread_data_mounts: true`；上传路由会跳过 sandbox acquire 和逐文件同步
+- 不确定挂载关系时应省略该配置并保留自动检测。错误地设为 `true` 会导致文件只存在于 Gateway 存储、沙箱内不可见
 
 ## 测试示例
 

backend/docs/MCP_SERVER.md

@@ -154,24 +154,65 @@ Declare interceptors in `extensions_config.json` using the `mcpInterceptors` fie
 
 Each entry is a Python import path in `module:variable` format (resolved via `resolve_variable`). The variable must be a **no-arg builder function** that returns an async interceptor compatible with `MultiServerMCPClient`’s `tool_interceptors` interface, or `None` to skip.
 
-Example interceptor that injects auth headers from LangGraph metadata:
+Example interceptor that injects an authorization header from the request-scoped
+LangGraph secret context:
 
 ```python
+from langgraph.config import get_config
+
+
 def build_auth_interceptor():
     async def interceptor(request, handler):
-        from langgraph.config import get_config
-        metadata = get_config().get("metadata", {})
-        headers = dict(request.headers or {})
-        if token := metadata.get("auth_token"):
-            headers["X-Auth-Token"] = token
-        return await handler(request.override(headers=headers))
+        config = get_config()
+        secrets = (config.get("context") or {}).get("secrets") or {}
+        token = secrets.get("MCP_AUTH_TOKEN")
+        if token:
+            request = request.override(
+                headers={**(request.headers or {}), "Authorization": f"Bearer {token}"}
+            )
+        return await handler(request)
+
     return interceptor
 ```
 
+Supply the credential on each run request through `config.context.secrets`:
+
+```json
+{
+  "metadata": {"source": "my-client"},
+  "config": {
+    "context": {
+      "secrets": {"MCP_AUTH_TOKEN": "<request-scoped credential>"}
+    }
+  }
+}
+```
+
+Both `metadata.auth_token` and `config.metadata.auth_token` are rejected with HTTP 422 at run admission and are never supported
+interceptor paths. Do not put credentials in either metadata surface; use
+`config.context.secrets`, whose values remain available to the live interceptor
+but are removed from persisted and API-visible run configuration copies.
+
 - A single string value is accepted and normalized to a one-element list.
 - Invalid paths or builder failures are logged as warnings without blocking other interceptors.
 - The builder return value must be `callable`; non-callable values are skipped with a warning.
 
+### Migrating legacy MCP credentials
+
+Deployments that previously sent `metadata.auth_token` or `config.metadata.auth_token` must:
+
+1. Update the caller and interceptor to use `config.context.secrets` as shown
+   above.
+2. Rotate the exposed credential before resuming authenticated MCP traffic.
+3. Locate and remove every retained legacy copy according to the deployment's
+   retention policy, including database rows, run events, application or proxy
+   logs, snapshots, exports, and backups.
+
+Current history APIs hide legacy `metadata.auth_token` and `config.metadata.auth_token` values, but hiding a response does not erase
+material already retained by those systems. Restarting or upgrading DeerFlow does
+not rotate credentials or perform historical cleanup; operators must complete
+both actions explicitly.
+
 ## How It Works
 
 MCP servers expose tools that are automatically discovered and integrated into DeerFlow’s agent system at runtime. Once enabled, these tools become available to agents without additional code changes.

backend/packages/harness/deerflow/agents/factory.py

@@ -72,6 +72,7 @@ def create_deerflow_agent(
     plan_mode: bool = False,
     state_schema: type | None = None,
     checkpoint_channel_mode: CheckpointChannelMode = "full",
+    checkpoint_snapshot_frequency: int | None = None,
     checkpointer: BaseCheckpointSaver | None = None,
     name: str = "default",
 ) -> CompiledStateGraph:
@@ -107,6 +108,10 @@ def create_deerflow_agent(
         persistence paths (mode markers + compatibility gate) and is therefore
         rejected when combined with *checkpointer* in this factory; without a
         checkpointer the graph is ephemeral and delta is allowed.
+    checkpoint_snapshot_frequency:
+        DeltaChannel snapshot cadence for ``"delta"`` mode.  ``None`` uses the
+        process-frozen value, falling back to the config default.  Ignored in
+        ``"full"`` mode.
     checkpointer:
         Optional persistence backend.
     name:
@@ -135,7 +140,7 @@ def create_deerflow_agent(
                 raise TypeError(f"extra_middleware items must be AgentMiddleware instances, got {type(mw).__name__}")
 
     effective_tools: list[BaseTool] = list(tools or [])
-    effective_state = get_thread_state_schema(checkpoint_channel_mode) if state_schema is None else adapt_state_schema_for_mode(state_schema, checkpoint_channel_mode)
+    effective_state = get_thread_state_schema(checkpoint_channel_mode, checkpoint_snapshot_frequency) if state_schema is None else adapt_state_schema_for_mode(state_schema, checkpoint_channel_mode, checkpoint_snapshot_frequency)
 
     if middleware is not None:
         effective_middleware = list(middleware)
@@ -157,6 +162,7 @@ def create_deerflow_agent(
     effective_middleware = normalize_middleware_state_schemas(
         effective_middleware,
         checkpoint_channel_mode,
+        checkpoint_snapshot_frequency,
     )
 
     return create_agent(
@@ -269,6 +275,7 @@ def _assemble_from_features(
 
             memory_cfg: MemoryConfig = feat.memory_config or get_memory_config()
             if should_use_memory_tools(memory_cfg):
+                from deerflow.agents.memory.manager import backend_requires_passive_writes_in_tool_mode
                 from deerflow.agents.memory.tools import get_memory_tools
 
                 existing_names = {tool.name for tool in extra_tools}
@@ -278,8 +285,10 @@ def _assemble_from_features(
                         continue
                     extra_tools.append(memory_tool)
                     existing_names.add(memory_tool.name)
-                # MemoryMiddleware is intentionally NOT appended in tool mode.
-                # The model drives memory via tools instead of passive middleware.
+                if backend_requires_passive_writes_in_tool_mode(memory_cfg.manager_class):
+                    from deerflow.agents.middlewares.memory_middleware import MemoryMiddleware
+
+                    chain.append(MemoryMiddleware(agent_name=name, memory_config=memory_cfg))
             else:
                 if memory_cfg.mode == "tool" and not memory_cfg.enabled:
                     logger.warning("memory.mode is 'tool' but memory.enabled is false; memory tools will not be registered.")

backend/packages/harness/deerflow/agents/lead_agent/agent.py

@@ -46,7 +46,7 @@ from deerflow.agents.middlewares.todo_middleware import TodoMiddleware
 from deerflow.agents.middlewares.token_usage_middleware import TokenUsageMiddleware
 from deerflow.agents.middlewares.tool_error_handling_middleware import build_lead_runtime_middlewares
 from deerflow.agents.middlewares.view_image_middleware import ViewImageMiddleware
-from deerflow.agents.thread_state import freeze_delta_snapshot_frequency, get_thread_state_schema, normalize_middleware_state_schemas
+from deerflow.agents.thread_state import get_thread_state_schema, normalize_middleware_state_schemas
 from deerflow.authz.tool_filter import apply_tool_authorization
 from deerflow.config.agents_config import load_agent_config, validate_agent_name
 from deerflow.config.app_config import AppConfig, get_app_config
@@ -56,6 +56,7 @@ from deerflow.models import create_chat_model
 from deerflow.runtime.checkpoint_mode import (
     INTERNAL_CHECKPOINT_MODE_KEY,
     freeze_checkpoint_channel_mode,
+    freeze_checkpoint_snapshot_frequency,
     frozen_checkpoint_channel_mode,
     inject_checkpoint_mode,
 )
@@ -383,9 +384,13 @@ def build_middlewares(
     # Add TitleMiddleware
     middlewares.append(TitleMiddleware(app_config=resolved_app_config))
 
-    # Add MemoryMiddleware (after TitleMiddleware) — skipped in enabled tool mode
+    # Add MemoryMiddleware after TitleMiddleware. Tool mode normally skips it;
+    # conversation-extraction backends may explicitly retain passive writes.
     if should_use_memory_tools(resolved_app_config.memory):
-        pass
+        from deerflow.agents.memory.manager import backend_requires_passive_writes_in_tool_mode
+
+        if backend_requires_passive_writes_in_tool_mode(resolved_app_config.memory.manager_class):
+            middlewares.append(MemoryMiddleware(agent_name=agent_name, memory_config=resolved_app_config.memory))
     else:
         if resolved_app_config.memory.mode == "tool" and not resolved_app_config.memory.enabled:
             logger.warning("memory.mode is 'tool' but memory.enabled is false; memory tools will not be registered.")
@@ -518,7 +523,10 @@ def make_lead_agent(config: RunnableConfig):
             runtime_app_config.database.checkpoint_channel_mode,
         )
     mode = freeze_checkpoint_channel_mode(requested_mode)
-    freeze_delta_snapshot_frequency(runtime_app_config.database.checkpoint_delta_snapshot_frequency)
+    # The snapshot cadence travels with the mode: restart-required, frozen
+    # from the app config, and deliberately not client-injectable (a forged
+    # configurable key must not recompile the channel table either).
+    freeze_checkpoint_snapshot_frequency(runtime_app_config.database.checkpoint_delta.snapshot_frequency)
     inject_checkpoint_mode(config, mode)
     return _make_lead_agent(config, app_config=runtime_app_config)
 
@@ -536,13 +544,12 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
         resolved_app_config.database.checkpoint_channel_mode,
     )
 
-    # Extract user_id for user-scoped skill loading.
-    # LangGraph gateway injects user_id into config["configurable"];
-    # fall back to the runtime contextvar when not present.
-    from deerflow.runtime.user_context import get_effective_user_id
+    # Resolve one authoritative identity for every user-scoped factory input.
+    # Agent Server's reserved auth fields win over ordinary client-supplied
+    # context/configurable values; the embedded Gateway path uses context.user_id.
+    from deerflow.runtime.user_context import resolve_config_user_id
 
-    runtime_user_id = cfg.get("user_id")
-    resolved_user_id = str(runtime_user_id) if runtime_user_id else get_effective_user_id()
+    resolved_user_id = resolve_config_user_id(config)
 
     requested_model_name: str | None = cfg.get("model_name") or cfg.get("model")
     is_plan_mode = cfg.get("is_plan_mode", False)
@@ -553,7 +560,7 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
     non_interactive = bool(cfg.get("non_interactive", False))
     agent_name = validate_agent_name(cfg.get("agent_name"))
 
-    agent_config = load_agent_config(agent_name) if not is_bootstrap else None
+    agent_config = load_agent_config(agent_name, user_id=resolved_user_id) if not is_bootstrap else None
     available_skills = _available_skill_names(agent_config, is_bootstrap)
     # Custom agent model from agent config (if any), or None to let _resolve_model_name pick the default
     agent_model_name = agent_config.model if agent_config and agent_config.model else None

backend/packages/harness/deerflow/agents/lead_agent/prompt.py

@@ -727,20 +727,28 @@ combined with a FastAPI gateway for REST API access [citation:FastAPI](https://f
 """
 
 
-def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig | None = None) -> str:
+def _get_memory_context(
+    agent_name: str | None = None,
+    *,
+    app_config: AppConfig | None = None,
+    user_id: str | None = None,
+) -> str:
     """Get memory context for injection into system prompt.
 
     Args:
         agent_name: If provided, loads per-agent memory. If None, loads global memory.
         app_config: Explicit application config. When provided, memory options
             are read from this value instead of the global config singleton.
+        user_id: Explicit user bucket. When omitted, resolves the current
+            Gateway or standalone LangGraph Server identity.
 
     Returns:
         Formatted memory context string wrapped in XML tags, or empty string if disabled.
     """
+    config = None
     try:
         from deerflow.agents.memory import get_memory_manager
-        from deerflow.runtime.user_context import get_effective_user_id
+        from deerflow.runtime.user_context import resolve_runtime_user_id
 
         if app_config is None:
             from deerflow.config.memory_config import get_memory_config
@@ -753,7 +761,7 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
             return ""
 
         memory_content = get_memory_manager().get_context(
-            user_id=get_effective_user_id(),
+            user_id=user_id or resolve_runtime_user_id(None),
             agent_name=agent_name,
         )
 
@@ -764,8 +772,13 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
 {memory_content}
 </memory>
 """
-    except Exception:
+    except Exception as exc:
         logger.exception("Failed to load memory context")
+        from deerflow.agents.memory import MemoryManagerError
+
+        failure_policy = getattr(config, "backend_config", {}).get("failure_policy", {}) if config is not None else {}
+        if isinstance(exc, MemoryManagerError) and failure_policy.get("read") == "fail_closed":
+            raise
         return ""
 
 
@@ -889,9 +902,9 @@ def get_skills_prompt_section(
     return _get_cached_skills_prompt_section(skill_signature, disabled_skill_signature, available_key, container_base_path, skill_evolution_section)
 
 
-def get_agent_soul(agent_name: str | None) -> str:
+def get_agent_soul(agent_name: str | None, *, user_id: str | None = None) -> str:
     # Append SOUL.md (agent personality) if present
-    soul = load_agent_soul(agent_name)
+    soul = load_agent_soul(agent_name, user_id=user_id)
     if soul:
         # SOUL.md is agent-editable (setup_agent / update_agent persist it) and is
         # rendered into the <soul> block of the lead-agent system prompt. Escape it
@@ -994,8 +1007,8 @@ def _build_memory_tool_section(*, app_config: AppConfig | None = None) -> str:
         return ""
 
     return """<memory_tool_system>
-Memory is running in tool mode. Use the injected <memory> block as current context, and use the memory tools to keep durable user memory accurate:
-- Call `memory_search` before relying on memory that may be absent, stale, or too broad for the injected context.
+Memory is running in tool mode. When present, the injected <memory> block contains only global user and history summaries; agent facts are not injected automatically. Use the memory tools to keep durable user memory accurate:
+- Call `memory_search` whenever prior preferences, constraints, corrections, or durable context may be relevant. Do not assume an absent fact does not exist until you have searched with an appropriate query.
 - Call `memory_add` only for stable facts useful in future sessions: explicit user preferences, corrections, personal/work context, or durable project context.
 - Call `memory_update` when an existing fact is outdated or imprecise; prefer updating over adding a near-duplicate.
 - Call `memory_delete` only when a fact is clearly wrong or no longer relevant.
@@ -1074,7 +1087,7 @@ def apply_prompt_template(
     # identical across users and sessions for maximum prefix-cache reuse.
     return SYSTEM_PROMPT_TEMPLATE.format(
         agent_name=agent_name or "DeerFlow 2.0",
-        soul=get_agent_soul(agent_name),
+        soul=get_agent_soul(agent_name, user_id=user_id),
         self_update_section=_build_self_update_section(agent_name),
         skills_section=skills_section,
         deferred_tools_section=deferred_tools_section,

backend/packages/harness/deerflow/agents/memory/backends/README.md

@@ -4,6 +4,7 @@ Each subfolder under `agents/memory/backends/` is a pluggable memory backend. Sw
 
 - `deermem/` - the default backend (deer-flow's own: structured facts + JSON storage).
 - `noop/` - an empty backend and the **template** to copy when adding a new one.
+- `openviking/` - optional remote OpenViking backend over HTTP (middleware mode).
 
 This guide tells you **which files to touch** when you change, swap, or add a memory system. Paths are relative to `backend/` unless noted.
 

backend/packages/harness/deerflow/agents/memory/backends/deermem/deer_mem.py

@@ -302,12 +302,18 @@ class DeerMem(MemoryManager):
     ) -> str:
         """Load memory and format it for injection (plain text, no wrap).
 
+        Middleware mode injects the selected agent's facts together with the
+        user-global summaries. Tool mode injects only those global summaries;
+        facts stay behind ``memory_search`` so they are not duplicated in the
+        prompt and a later retrieval result.
+
         Format parameters come from DeerMem's own ``DeerMemConfig`` (set at
         construction from ``backend_config``). The ``enabled``/
         ``injection_enabled`` gate and the ``<memory>`` wrapping stay at the
         call site (``_get_memory_context``); this returns only the body.
         """
-        memory_data = _call_backend(lambda: self._updater.get_memory_data(agent_name=_resolve_agent_name(agent_name), user_id=user_id))
+        injection_agent = None if self.mode == "tool" else _resolve_agent_name(agent_name)
+        memory_data = _call_backend(lambda: self._updater.get_memory_data(agent_name=injection_agent, user_id=user_id))
         return format_memory_for_injection(
             memory_data,
             max_tokens=self._config.max_injection_tokens,

backend/packages/harness/deerflow/agents/memory/backends/mem0/README.md

@@ -0,0 +1,73 @@
+# mem0 memory backend
+
+Uses mem0 (Platform hosted API, or any API-compatible self-hosted server) as
+DeerFlow's memory store. Fully stateless in-process: dedup, fact extraction,
+and storage are server-side, so it is safe for multi-worker Gateway
+deployments.
+
+## Configuration
+
+```yaml
+memory:
+  enabled: true
+  injection_enabled: true
+  manager_class: mem0
+  mode: middleware            # or "tool"
+  backend_config:
+    api_key_env: MEM0_API_KEY          # key read from env, never in config.yaml
+    base_url: https://api.mem0.ai      # or your self-hosted mem0 server
+    allow_insecure_http: false         # true only for trusted local HTTP dev
+    top_k: 8
+    score_threshold: 0.1
+    max_injection_chars: 12000
+    timeout_seconds: 10
+    startup_policy: fail_fast          # fail_fast | tolerate
+    failure_policy:
+      read: fail_open                  # fail_open | fail_closed
+      write: log_and_drop              # log_and_drop | raise
+```
+
+Set the key in the environment: `export MEM0_API_KEY=...`
+
+`base_url` must use HTTPS because every request carries the API key. For a
+trusted local-development server that only exposes HTTP, opt in explicitly
+with `allow_insecure_http: true`; do not use that setting across an untrusted
+network.
+
+## Identity mapping
+
+| DeerFlow | mem0 |
+|---|---|
+| `user_id` | `user_id` |
+| `agent_name` | `agent_id` |
+| `thread_id` | `run_id` |
+
+## Limitations
+
+- `mode: middleware` recall is query-less (the `get_context` contract carries
+  no query): the bucket's most recent `top_k` memories are injected. For
+  query-aware semantic recall use `mode: tool`.
+- `mode: tool` retains the passive per-turn write middleware for this backend,
+  because mem0 extracts and deduplicates facts from conversations through
+  `add()`. The agent still gains query-aware `memory_search`, while new
+  conversations continue accumulating memory even though fact CRUD is not
+  available.
+- Fact CRUD, `import_memory`, and Settings-page memory editing are not
+  implemented (gateway returns 501). DeerMem remains the default backend.
+- No migration of existing DeerMem data.
+- `log_and_drop` write policy is at-most-once: a failed write is dropped.
+- `memory_add`/`memory_update`/`memory_delete` are backed by fact CRUD, which
+  this backend does not implement; they return a clear unsupported-operation
+  error. Conversation writes still happen through the retained middleware.
+
+## Async execution and failure behavior
+
+The mem0 HTTP client is synchronous for compatibility with the
+`MemoryManager` contract. DeerFlow offloads it at every async boundary: the
+async middleware uses the manager's `a*` methods, and Gateway memory routes run
+sync management calls in worker threads. A slow mem0 request therefore does
+not block unrelated ASGI handlers or SSE heartbeats.
+
+`failure_policy.read: fail_open` logs a recall failure and continues without
+new memory context. `fail_closed` propagates the backend error through prompt
+construction and aborts the run instead of silently degrading.

backend/packages/harness/deerflow/agents/memory/backends/mem0/__init__.py

@@ -0,0 +1,9 @@
+"""mem0 memory backend -- HTTP client against the mem0 Platform API.
+
+Drop-in contract: folder name == backend name == ``manager_class: mem0``.
+"""
+
+from .mem0_manager import Mem0Manager
+
+#: Discovered by the factory's ``_scan_backends`` under the folder name ``mem0``.
+MANAGER_CLASS = Mem0Manager

backend/packages/harness/deerflow/agents/memory/backends/mem0/client.py

@@ -0,0 +1,128 @@
+"""Synchronous httpx client for the mem0 REST API (v3; delete is v1).
+
+The MemoryManager contract is synchronous (DeerMem's LLM calls are sync too),
+so this client is a plain ``httpx.Client``. It is constructed with an
+optional ``transport`` so tests can inject ``httpx.MockTransport``.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import httpx
+
+
+class Mem0APIError(RuntimeError):
+    """Any mem0 request failure (transport, 4xx/5xx)."""
+
+
+class Mem0AuthError(Mem0APIError):
+    """401 -- missing or invalid API key."""
+
+
+class Mem0Client:
+    """Thin wrapper over the mem0 endpoints DeerFlow uses."""
+
+    def __init__(
+        self,
+        *,
+        base_url: str,
+        api_key: str,
+        timeout_seconds: float = 10.0,
+        transport: httpx.BaseTransport | None = None,
+    ) -> None:
+        self._http = httpx.Client(
+            base_url=base_url.rstrip("/"),
+            headers={"Authorization": f"Token {api_key}", "Accept": "application/json"},
+            timeout=timeout_seconds,
+            transport=transport,
+        )
+
+    def close(self) -> None:
+        self._http.close()
+
+    def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]:
+        try:
+            resp = self._http.request(method, path, **kwargs)
+        except httpx.HTTPError as e:
+            raise Mem0APIError(f"mem0 request failed: {e}") from e
+        if resp.status_code == 401:
+            raise Mem0AuthError("mem0 authentication failed (check the API key)")
+        if resp.status_code >= 400:
+            raise Mem0APIError(f"mem0 {method} {path} -> {resp.status_code}: {resp.text[:200]}")
+        if not resp.content:
+            return {}
+        try:
+            return resp.json()
+        except json.JSONDecodeError as e:
+            raise Mem0APIError(f"mem0 {method} {path} returned malformed JSON: {e}") from e
+
+    def add_memories(
+        self,
+        *,
+        messages: list[dict[str, str]],
+        user_id: str | None = None,
+        agent_id: str | None = None,
+        run_id: str | None = None,
+    ) -> dict[str, Any]:
+        """Queue extraction (async server-side; response carries an event_id)."""
+        body: dict[str, Any] = {"messages": messages}
+        if user_id:
+            body["user_id"] = user_id
+        if agent_id:
+            body["agent_id"] = agent_id
+        if run_id:
+            body["run_id"] = run_id
+        return self._request("POST", "/v3/memories/add/", json=body)
+
+    def search_memories(
+        self,
+        *,
+        query: str,
+        filters: dict[str, Any],
+        top_k: int,
+        threshold: float,
+    ) -> list[dict[str, Any]]:
+        body = {"query": query, "filters": filters, "top_k": top_k, "threshold": threshold}
+        return self._request("POST", "/v3/memories/search/", json=body).get("results", [])
+
+    def list_memories(
+        self,
+        *,
+        filters: dict[str, Any],
+        page_size: int = 200,
+        max_items: int | None = None,
+    ) -> list[dict[str, Any]]:
+        """List memories across pages until exhausted or ``max_items`` reached."""
+        results: list[dict[str, Any]] = []
+        page = 1
+        while True:
+            data = self._request(
+                "POST",
+                "/v3/memories/",
+                params={"page": page, "page_size": page_size},
+                json={"filters": filters},
+            )
+            results.extend(data.get("results", []))
+            if not data.get("next") or (max_items is not None and len(results) >= max_items):
+                return results[:max_items] if max_items is not None else results
+            page += 1
+
+    def delete_all_memories(
+        self,
+        *,
+        user_id: str | None = None,
+        agent_id: str | None = None,
+        run_id: str | None = None,
+    ) -> None:
+        params = {k: v for k, v in {"user_id": user_id, "agent_id": agent_id, "run_id": run_id}.items() if v}
+        self._request("DELETE", "/v1/memories/", params=params)
+
+    def ping(self) -> None:
+        """Startup auth check: a 1-item list scoped to a sentinel user id.
+
+        Proves the API key works without touching real data (the sentinel
+        bucket is always empty).
+        """
+        self.list_memories(filters={"user_id": "__deerflow_startup_check__"}, page_size=1, max_items=1)

backend/packages/harness/deerflow/agents/memory/backends/mem0/config.py

@@ -0,0 +1,123 @@
+"""mem0 backend config -- parses and validates ``backend_config``.
+
+Follows the noop-template pattern: a plain dataclass + ``from_backend_config``.
+The host injects ``storage_path`` (and optionally ``should_keep_hidden_message``)
+into every backend's config dict; those keys are accepted and ignored. Any
+OTHER unknown key is rejected -- a typo in persistent-state config must fail
+fast, not silently fall back to defaults.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlsplit
+
+#: Keys the host factory injects into backend_config; accepted and ignored.
+_HOST_INJECTED_KEYS = frozenset({"storage_path", "should_keep_hidden_message"})
+
+_STARTUP_POLICIES = frozenset({"fail_fast", "tolerate"})
+_READ_POLICIES = frozenset({"fail_open", "fail_closed"})
+_WRITE_POLICIES = frozenset({"log_and_drop", "raise"})
+
+
+@dataclass(frozen=True)
+class Mem0Config:
+    """Validated knobs for the mem0 HTTP backend."""
+
+    #: Name of the environment variable holding the mem0 API key. The key
+    #: itself never appears in config.yaml.
+    api_key_env: str = "MEM0_API_KEY"
+    #: mem0 Platform API root; point at a self-hosted server for on-prem.
+    base_url: str = "https://api.mem0.ai"
+    #: Permit sending the API token over plaintext HTTP. Intended only for
+    #: trusted local development networks.
+    allow_insecure_http: bool = False
+    #: Max memories injected by get_context / default search breadth (1-1000).
+    top_k: int = 8
+    #: Minimum relevance score for search() results (mem0 `threshold`, 0-1).
+    score_threshold: float = 0.1
+    #: Hard cap on the injection text returned by get_context.
+    max_injection_chars: int = 12000
+    #: Per-request HTTP timeout in seconds.
+    timeout_seconds: float = 10.0
+    #: "fail_fast" = auth-check in from_config; "tolerate" = defer to first use.
+    startup_policy: str = "fail_fast"
+    #: "fail_open" = recall errors inject nothing and continue;
+    #: "fail_closed" = recall errors raise MemoryManagerError.
+    read_policy: str = "fail_open"
+    #: "log_and_drop" = write errors are logged and dropped (at-most-once);
+    #: "raise" = write errors raise MemoryManagerError.
+    write_policy: str = "log_and_drop"
+
+    @classmethod
+    def from_backend_config(cls, backend_config: dict[str, Any] | None) -> Mem0Config:
+        cfg = dict(backend_config or {})
+        failure_policy = cfg.pop("failure_policy", {}) or {}
+        unknown = (
+            set(cfg)
+            - {
+                "api_key_env",
+                "base_url",
+                "allow_insecure_http",
+                "top_k",
+                "score_threshold",
+                "max_injection_chars",
+                "timeout_seconds",
+                "startup_policy",
+            }
+            - _HOST_INJECTED_KEYS
+        )
+        if unknown:
+            raise ValueError(f"mem0 backend_config has unknown keys: {sorted(unknown)}")
+        if not isinstance(failure_policy, dict):
+            raise ValueError("mem0 failure_policy must be a mapping {read, write}")
+        unknown_fp = set(failure_policy) - {"read", "write"}
+        if unknown_fp:
+            raise ValueError(f"mem0 failure_policy has unknown keys: {sorted(unknown_fp)}")
+        allow_insecure_http = cfg.get("allow_insecure_http", False)
+        if not isinstance(allow_insecure_http, bool):
+            raise ValueError("mem0 allow_insecure_http must be a boolean")
+
+        config = cls(
+            api_key_env=str(cfg.get("api_key_env", "MEM0_API_KEY")),
+            base_url=str(cfg.get("base_url", "https://api.mem0.ai")).rstrip("/"),
+            allow_insecure_http=allow_insecure_http,
+            top_k=int(cfg.get("top_k", 8)),
+            score_threshold=float(cfg.get("score_threshold", 0.1)),
+            max_injection_chars=int(cfg.get("max_injection_chars", 12000)),
+            timeout_seconds=float(cfg.get("timeout_seconds", 10.0)),
+            startup_policy=str(cfg.get("startup_policy", "fail_fast")),
+            read_policy=str(failure_policy.get("read", "fail_open")),
+            write_policy=str(failure_policy.get("write", "log_and_drop")),
+        )
+        if config.startup_policy not in _STARTUP_POLICIES:
+            raise ValueError(f"mem0 startup_policy must be one of {sorted(_STARTUP_POLICIES)}")
+        if config.read_policy not in _READ_POLICIES:
+            raise ValueError(f"mem0 failure_policy.read must be one of {sorted(_READ_POLICIES)}")
+        if config.write_policy not in _WRITE_POLICIES:
+            raise ValueError(f"mem0 failure_policy.write must be one of {sorted(_WRITE_POLICIES)}")
+        if not 1 <= config.top_k <= 1000:
+            raise ValueError("mem0 top_k must be in [1, 1000]")
+        if not 0.0 <= config.score_threshold <= 1.0:
+            raise ValueError("mem0 score_threshold must be in [0, 1]")
+        if config.max_injection_chars <= 0:
+            raise ValueError("mem0 max_injection_chars must be positive")
+        if config.timeout_seconds <= 0:
+            raise ValueError("mem0 timeout_seconds must be positive")
+        if not config.api_key_env.strip():
+            raise ValueError("mem0 api_key_env must be a non-empty env var name")
+        parsed_base_url = urlsplit(config.base_url)
+        if parsed_base_url.scheme not in {"http", "https"} or not parsed_base_url.netloc:
+            raise ValueError("mem0 base_url must be an absolute http:// or https:// URL")
+        if parsed_base_url.scheme == "http" and not config.allow_insecure_http:
+            raise ValueError("mem0 base_url must use https:// because it carries the API key; set allow_insecure_http: true only for trusted local development")
+        return config
+
+    def resolve_api_key(self) -> str:
+        """Read the API key from the configured environment variable."""
+        key = os.environ.get(self.api_key_env, "").strip()
+        if not key:
+            raise ValueError(f"mem0 API key missing: environment variable {self.api_key_env} is unset or empty")
+        return key

backend/packages/harness/deerflow/agents/memory/backends/mem0/mem0_manager.py

@@ -0,0 +1,315 @@
+"""mem0 memory backend -- a stateless HTTP MemoryManager.
+
+All state lives server-side in mem0 (dedup, extraction, storage): this backend
+keeps no queue, watermark, or cache, so it is safe for multi-worker Gateway
+deployments. Identity maps 1:1: (user_id, agent_name) -> mem0 (user_id,
+agent_id); thread_id -> mem0 run_id.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any, ClassVar, Literal
+
+from pydantic import PrivateAttr
+
+# ABC contract -- the ONE allowed `from deerflow` import in this backend folder.
+from deerflow.agents.memory.manager import MemoryManager, MemoryManagerError
+
+from .client import Mem0APIError, Mem0Client
+from .config import Mem0Config
+from .message_filtering import extract_message_text, filter_messages_for_memory
+
+logger = logging.getLogger(__name__)
+
+_ROLE_MAP = {"human": "user", "ai": "assistant"}
+
+
+def _build_filters(
+    *,
+    user_id: str | None = None,
+    agent_name: str | None = None,
+    run_id: str | None = None,
+) -> dict[str, Any] | None:
+    """Build a mem0 ``filters`` object from the available identity parts.
+
+    Returns None when no entity id is available (mem0 requires at least one).
+    """
+    parts: list[dict[str, Any]] = []
+    if user_id:
+        parts.append({"user_id": user_id})
+    if agent_name:
+        parts.append({"agent_id": agent_name})
+    if run_id:
+        parts.append({"run_id": run_id})
+    if not parts:
+        return None
+    if len(parts) == 1:
+        return parts[0]
+    return {"AND": parts}
+
+
+def _to_fact(record: dict[str, Any]) -> dict[str, Any]:
+    """Map a mem0 record to the backend-neutral fact shape consumed by the
+    host (agents/memory/tools.py): id/content/category/confidence/createdAt/
+    source. mem0's relevance score doubles as confidence."""
+    categories = record.get("categories") or []
+    metadata = record.get("metadata") or {}
+    return {
+        "id": str(record.get("id", "")),
+        "content": str(record.get("memory", "")),
+        "category": str(categories[0]) if categories else "context",
+        "confidence": float(record.get("score") or 0.0),
+        "createdAt": str(record.get("created_at", "")),
+        "source": str(metadata.get("source", "")),
+    }
+
+
+class Mem0Manager(MemoryManager):
+    """MemoryManager backed by the mem0 Platform API (or compatible server)."""
+
+    # search() is overridden below -> flag must be True (contract invariant);
+    # this also enables memory mode="tool".
+    supports_search: ClassVar[bool] = True
+    # mem0 extracts/deduplicates facts from full conversations through add();
+    # its fact CRUD hooks are intentionally unsupported, so tool mode retains
+    # passive writes while exposing query-aware search.
+    requires_passive_writes_in_tool_mode: ClassVar[bool] = True
+
+    _config: Mem0Config = PrivateAttr()
+    _client: Any = PrivateAttr(default=None)  # Mem0Client; tests inject a fake
+
+    def model_post_init(self, __context: Any) -> None:
+        self._config = Mem0Config.from_backend_config(self.backend_config)
+        self._client = Mem0Client(
+            base_url=self._config.base_url,
+            api_key=self._config.resolve_api_key(),
+            timeout_seconds=self._config.timeout_seconds,
+        )
+
+    @classmethod
+    def from_config(
+        cls,
+        backend_config: dict[str, Any] | None = None,
+        *,
+        mode: Literal["middleware", "tool"] = "middleware",
+        **host_hooks: Any,
+    ) -> Mem0Manager:
+        """Build the manager; ``fail_fast`` startup policy auth-checks via ping."""
+        mgr = cls(backend_config=backend_config, mode=mode)
+        if mgr._config.startup_policy == "fail_fast":
+            mgr._client.ping()
+        return mgr
+
+    def close(self) -> None:
+        """Release the underlying HTTP connection pool."""
+        self._client.close()
+
+    # ── Error policies ───────────────────────────────────────────────────
+    def _read_or_fallback(self, fallback: Any, fn: Any) -> Any:
+        try:
+            return fn()
+        except Mem0APIError as e:
+            if self._config.read_policy == "fail_open":
+                logger.warning("mem0 read failed (%s); continuing without memory", e)
+                return fallback
+            raise MemoryManagerError(f"mem0 read failed: {e}") from e
+
+    def _write_or_drop(self, fn: Any) -> None:
+        try:
+            fn()
+        except Mem0APIError as e:
+            if self._config.write_policy == "log_and_drop":
+                logger.warning("mem0 write failed (%s); dropping update", e)
+                return
+            raise MemoryManagerError(f"mem0 write failed: {e}") from e
+
+    # ── Tier 1: write ────────────────────────────────────────────────────
+    def add(
+        self,
+        thread_id: str,
+        messages: list[Any],
+        *,
+        agent_name: str | None = None,
+        user_id: str | None = None,
+        trace_id: str | None = None,
+    ) -> None:
+        """Submit the filtered conversation to mem0 for server-side extraction.
+
+        Fire-and-forget: mem0 processes asynchronously (response event_id is
+        not polled). ``thread_id`` maps to mem0 ``run_id`` and always satisfies
+        mem0's at-least-one-entity-id requirement.
+        """
+        kept = filter_messages_for_memory(messages)
+        payload = [{"role": _ROLE_MAP[getattr(m, "type", "")], "content": extract_message_text(m).strip()} for m in kept if getattr(m, "type", "") in _ROLE_MAP]
+        payload = [p for p in payload if p["content"]]
+        if not payload:
+            return
+        self._write_or_drop(
+            lambda: self._client.add_memories(
+                messages=payload,
+                user_id=user_id,
+                agent_id=agent_name,
+                run_id=thread_id,
+            )
+        )
+
+    async def aadd(
+        self,
+        thread_id: str,
+        messages: list[Any],
+        *,
+        agent_name: str | None = None,
+        user_id: str | None = None,
+        trace_id: str | None = None,
+    ) -> None:
+        await asyncio.to_thread(
+            self.add,
+            thread_id,
+            messages,
+            agent_name=agent_name,
+            user_id=user_id,
+            trace_id=trace_id,
+        )
+
+    # ── Tier 1: read-inject ──────────────────────────────────────────────
+    def get_context(
+        self,
+        user_id: str | None,
+        *,
+        agent_name: str | None = None,
+        thread_id: str | None = None,
+    ) -> str:
+        """Query-less recall: the contract passes no current query, so inject
+        the bucket's most recent memories (top_k). Query-aware recall is
+        available via search() in mode="tool"."""
+        filters = _build_filters(user_id=user_id, agent_name=agent_name, run_id=thread_id)
+        if filters is None:
+            return ""
+        top_k = self._config.top_k
+        records = self._read_or_fallback(
+            [],
+            lambda: self._client.list_memories(
+                filters=filters,
+                page_size=min(top_k, 200),
+                max_items=top_k,
+            ),
+        )
+        seen: set[str] = set()
+        lines: list[str] = []
+        for record in records:
+            rid = record.get("id")
+            if rid in seen:
+                continue
+            seen.add(rid)
+            text = str(record.get("memory") or "").strip()
+            if text:
+                lines.append(f"- {text}")
+        context = "\n".join(lines)
+        if len(context) > self._config.max_injection_chars:
+            context = context[: self._config.max_injection_chars]
+        return context
+
+    async def aget_context(
+        self,
+        user_id: str | None,
+        *,
+        agent_name: str | None = None,
+        thread_id: str | None = None,
+    ) -> str:
+        return await asyncio.to_thread(
+            self.get_context,
+            user_id,
+            agent_name=agent_name,
+            thread_id=thread_id,
+        )
+
+    # ── Tier 2: search ───────────────────────────────────────────────────
+    def search(
+        self,
+        query: str,
+        top_k: int = 5,
+        *,
+        user_id: str | None = None,
+        agent_name: str | None = None,
+        category: str | None = None,
+    ) -> list[dict[str, Any]]:
+        filters = _build_filters(user_id=user_id, agent_name=agent_name)
+        if filters is None:
+            return []
+        if category:
+            parts = filters["AND"] if "AND" in filters else [filters]
+            filters = {"AND": [*parts, {"categories": {"contains": category}}]}
+        results = self._read_or_fallback(
+            [],
+            lambda: self._client.search_memories(
+                query=query,
+                filters=filters,
+                top_k=top_k,
+                threshold=self._config.score_threshold,
+            ),
+        )
+        return [_to_fact(r) for r in results]
+
+    async def asearch(
+        self,
+        query: str,
+        top_k: int = 5,
+        *,
+        user_id: str | None = None,
+        agent_name: str | None = None,
+        category: str | None = None,
+    ) -> list[dict[str, Any]]:
+        return await asyncio.to_thread(
+            self.search,
+            query,
+            top_k,
+            user_id=user_id,
+            agent_name=agent_name,
+            category=category,
+        )
+
+    # ── Tier 2: management ───────────────────────────────────────────────
+    def get_memory(
+        self,
+        *,
+        user_id: str | None = None,
+        agent_name: str | None = None,
+    ) -> dict[str, Any]:
+        filters = _build_filters(user_id=user_id, agent_name=agent_name)
+        if filters is None:
+            return {"facts": []}
+        records = self._read_or_fallback([], lambda: self._client.list_memories(filters=filters))
+        return {"facts": [_to_fact(r) for r in records]}
+
+    def export_memory(
+        self,
+        *,
+        user_id: str | None = None,
+        agent_name: str | None = None,
+    ) -> dict[str, Any]:
+        return self.get_memory(user_id=user_id, agent_name=agent_name)
+
+    def clear_memory(
+        self,
+        *,
+        user_id: str | None = None,
+        agent_name: str | None = None,
+    ) -> dict[str, Any]:
+        """Clear the bucket. agent_name=None clears the user's whole memory;
+        an explicit agent clears only that agent's bucket."""
+        if not user_id and not agent_name:
+            return {"facts": []}
+        self._write_or_drop(lambda: self._client.delete_all_memories(user_id=user_id, agent_id=agent_name, run_id=None))
+        return {"facts": []}
+
+    def delete_memory(
+        self,
+        *,
+        user_id: str | None = None,
+        agent_name: str | None = None,
+    ) -> None:
+        if not user_id and not agent_name:
+            return None
+        self._write_or_drop(lambda: self._client.delete_all_memories(user_id=user_id, agent_id=agent_name, run_id=None))

backend/packages/harness/deerflow/agents/memory/backends/mem0/message_filtering.py

@@ -0,0 +1,93 @@
+"""Message filtering for the mem0 write path -- self-contained mirror of
+DeerMem's ``filter_messages_for_memory`` rules (the portability rule forbids
+importing across backend folders, so the logic is duplicated, not shared).
+
+Keeps: visible user inputs, well-formed human clarification answers, and final
+assistant responses. Drops: framework-internal ``hide_from_ui`` messages,
+tool-call AI messages, tool outputs, empty/upload-only turns.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from copy import copy
+from typing import Any
+
+_UPLOAD_BLOCK_RE = re.compile(r"<(?P<tag>uploaded_files|current_uploads)>[\s\S]*?</(?P=tag)>\n*", re.IGNORECASE)
+
+
+def extract_message_text(message: Any) -> str:
+    """Extract plain text from message content (str or content-block list)."""
+    content = getattr(message, "content", "")
+    if content is None:
+        return ""
+    if isinstance(content, list):
+        parts: list[str] = []
+        for part in content:
+            if isinstance(part, str):
+                parts.append(part)
+            elif isinstance(part, dict):
+                text = part.get("text")
+                if isinstance(text, str):
+                    parts.append(text)
+        return " ".join(parts)
+    return str(content)
+
+
+def _non_empty_str(value: object) -> str | None:
+    return value if isinstance(value, str) and value.strip() else None
+
+
+def _is_human_clarification_response(additional_kwargs: Any) -> bool:
+    """Structural check for a user-authored clarification answer carried in a
+    hidden message (mirrors DeerMem's host-agnostic fallback)."""
+    if not isinstance(additional_kwargs, Mapping):
+        return False
+    raw = additional_kwargs.get("human_input_response")
+    if not isinstance(raw, Mapping):
+        return False
+    if raw.get("version") != 1 or raw.get("kind") != "human_input_response":
+        return False
+    if _non_empty_str(raw.get("source")) is None or _non_empty_str(raw.get("request_id")) is None or _non_empty_str(raw.get("value")) is None:
+        return False
+    response_kind = raw.get("response_kind")
+    if response_kind == "text":
+        return True
+    if response_kind == "option":
+        return _non_empty_str(raw.get("option_id")) is not None
+    return False
+
+
+def filter_messages_for_memory(messages: list[Any]) -> list[Any]:
+    """Keep only user inputs and final assistant responses."""
+    filtered: list[Any] = []
+    skip_next_ai = False
+    for msg in messages:
+        msg_type = getattr(msg, "type", None)
+        if msg_type == "human":
+            additional_kwargs = getattr(msg, "additional_kwargs", {}) or {}
+            if additional_kwargs.get("hide_from_ui") and not _is_human_clarification_response(additional_kwargs):
+                continue
+            text = extract_message_text(msg)
+            if "<uploaded_files>" in text.lower() or "<current_uploads>" in text.lower():
+                stripped = _UPLOAD_BLOCK_RE.sub("", text).strip()
+                if not stripped:
+                    # Upload-only turn: the following AI ack carries no user content.
+                    skip_next_ai = True
+                    continue
+                clean_msg = copy(msg)
+                clean_msg.content = stripped
+                filtered.append(clean_msg)
+                skip_next_ai = False
+            else:
+                filtered.append(msg)
+                skip_next_ai = False
+        elif msg_type == "ai":
+            if getattr(msg, "tool_calls", None):
+                continue
+            if skip_next_ai:
+                skip_next_ai = False
+                continue
+            filtered.append(msg)
+    return filtered

backend/packages/harness/deerflow/agents/memory/backends/openviking/__init__.py

@@ -0,0 +1,7 @@
+"""OpenViking HTTP memory backend."""
+
+from .openviking_manager import OpenVikingMemoryManager
+
+MANAGER_CLASS = OpenVikingMemoryManager
+
+__all__ = ["OpenVikingMemoryManager"]

backend/packages/harness/deerflow/agents/memory/backends/openviking/client.py

@@ -0,0 +1,247 @@
+"""Thin synchronous HTTP client for the OpenViking server API."""
+
+from __future__ import annotations
+
+import random
+import time
+from typing import Any
+
+import httpx
+
+from .config import OpenVikingConfig
+from .models import OpenVikingCommitResult, OpenVikingIdentity, OpenVikingMessage, OpenVikingSearchHit, OpenVikingSessionContext
+
+
+class OpenVikingClientError(RuntimeError):
+    """Base error raised by the remote OpenViking adapter."""
+
+    def __init__(self, operation: str, message: str, *, status_code: int | None = None, code: str | None = None):
+        super().__init__(message)
+        self.operation = operation
+        self.status_code = status_code
+        self.code = code
+
+
+class OpenVikingAuthenticationError(OpenVikingClientError):
+    pass
+
+
+class OpenVikingTimeoutError(OpenVikingClientError):
+    pass
+
+
+class OpenVikingUnavailableError(OpenVikingClientError):
+    pass
+
+
+class OpenVikingProtocolError(OpenVikingClientError):
+    pass
+
+
+class OpenVikingHttpClient:
+    """OpenViking API wrapper with bounded timeout and conservative retries."""
+
+    def __init__(self, config: OpenVikingConfig, *, transport: httpx.BaseTransport | None = None):
+        self._config = config
+        timeout = httpx.Timeout(
+            connect=config.connect_timeout_seconds,
+            read=config.read_timeout_seconds,
+            write=config.write_timeout_seconds,
+            pool=config.pool_timeout_seconds,
+        )
+        limits = httpx.Limits(
+            max_connections=config.max_connections,
+            max_keepalive_connections=config.max_keepalive_connections,
+        )
+        self._client = httpx.Client(
+            base_url=config.base_url,
+            timeout=timeout,
+            limits=limits,
+            transport=transport,
+        )
+
+    def close(self) -> None:
+        self._client.close()
+
+    def health(self) -> bool:
+        response = self._request("health", "GET", "/health", identity=None, retryable=True)
+        if response.status_code != 200:
+            return False
+        try:
+            payload = response.json()
+        except ValueError:
+            return False
+        return payload.get("status") == "ok"
+
+    def ensure_session(self, identity: OpenVikingIdentity, session_id: str) -> None:
+        self._request_result(
+            "session.ensure",
+            "GET",
+            f"/api/v1/sessions/{session_id}",
+            identity=identity,
+            params={"auto_create": "true"},
+            retryable=True,
+        )
+
+    def add_messages(self, identity: OpenVikingIdentity, session_id: str, messages: list[OpenVikingMessage]) -> int:
+        if not messages:
+            return 0
+        added = 0
+        # OpenViking caps one batch at 100 messages.
+        for offset in range(0, len(messages), 100):
+            batch = messages[offset : offset + 100]
+            result = self._request_result(
+                "messages.add",
+                "POST",
+                f"/api/v1/sessions/{session_id}/messages/batch",
+                identity=identity,
+                json={"messages": [message.as_request() for message in batch]},
+                retryable=False,
+            )
+            added += int(result.get("added", len(batch)))
+        return added
+
+    def commit_session(self, identity: OpenVikingIdentity, session_id: str) -> OpenVikingCommitResult:
+        result = self._request_result(
+            "session.commit",
+            "POST",
+            f"/api/v1/sessions/{session_id}/commit",
+            identity=identity,
+            json={"keep_recent_count": 0},
+            retryable=False,
+        )
+        return OpenVikingCommitResult(
+            status=str(result.get("status") or ""),
+            task_id=str(result["task_id"]) if result.get("task_id") else None,
+            archive_uri=str(result["archive_uri"]) if result.get("archive_uri") else None,
+            archived=bool(result.get("archived", False)),
+        )
+
+    def search(
+        self,
+        identity: OpenVikingIdentity,
+        query: str,
+        *,
+        top_k: int,
+        category: str | None = None,
+        session_id: str | None = None,
+    ) -> list[OpenVikingSearchHit]:
+        body: dict[str, Any] = {
+            "query": query,
+            "target_uri": "viking://user/memories",
+            "context_type": "memory",
+            "node_limit": top_k,
+        }
+        if session_id:
+            body["session_id"] = session_id
+        if self._config.score_threshold is not None:
+            body["score_threshold"] = self._config.score_threshold
+        result = self._request_result(
+            "search",
+            "POST",
+            "/api/v1/search/search" if session_id else "/api/v1/search/find",
+            identity=identity,
+            json=body,
+            retryable=True,
+        )
+        values = result.get("memories", [])
+        if not isinstance(values, list):
+            raise OpenVikingProtocolError("search", "OpenViking response field result.memories is not a list")
+        hits = [OpenVikingSearchHit.from_response(value) for value in values if isinstance(value, dict)]
+        if category:
+            category_key = category.casefold()
+            hits = [hit for hit in hits if hit.category.casefold() == category_key]
+        return hits[:top_k]
+
+    def get_session_context(
+        self,
+        identity: OpenVikingIdentity,
+        session_id: str,
+        *,
+        token_budget: int,
+    ) -> OpenVikingSessionContext:
+        result = self._request_result(
+            "session.context",
+            "GET",
+            f"/api/v1/sessions/{session_id}/context",
+            identity=identity,
+            params={"token_budget": token_budget},
+            retryable=True,
+        )
+        messages = result.get("messages", [])
+        return OpenVikingSessionContext(
+            latest_archive_overview=str(result.get("latest_archive_overview") or ""),
+            messages=messages if isinstance(messages, list) else [],
+            estimated_tokens=int(result.get("estimatedTokens") or 0),
+        )
+
+    def _headers(self, identity: OpenVikingIdentity | None) -> dict[str, str]:
+        headers = {"Accept": "application/json"}
+        if self._config.api_key:
+            headers["X-API-Key"] = self._config.api_key
+        if identity is not None and self._config.auth_mode == "trusted":
+            headers["X-OpenViking-Account"] = identity.account
+            headers["X-OpenViking-User"] = identity.user
+        return headers
+
+    def _request_result(self, operation: str, method: str, path: str, *, identity: OpenVikingIdentity, retryable: bool, **kwargs: Any) -> dict[str, Any]:
+        response = self._request(operation, method, path, identity=identity, retryable=retryable, **kwargs)
+        try:
+            payload = response.json()
+        except ValueError as exc:
+            raise OpenVikingProtocolError(operation, "OpenViking returned non-JSON data", status_code=response.status_code) from exc
+        if not isinstance(payload, dict) or payload.get("status") != "ok":
+            error = payload.get("error", {}) if isinstance(payload, dict) else {}
+            code = str(error.get("code") or "UNKNOWN") if isinstance(error, dict) else "UNKNOWN"
+            message = str(error.get("message") or "OpenViking request failed") if isinstance(error, dict) else "OpenViking request failed"
+            error_type = OpenVikingAuthenticationError if response.status_code in {401, 403} else OpenVikingProtocolError
+            raise error_type(operation, message, status_code=response.status_code, code=code)
+        result = payload.get("result")
+        if not isinstance(result, dict):
+            raise OpenVikingProtocolError(operation, "OpenViking response field result is not an object", status_code=response.status_code)
+        return result
+
+    def _request(
+        self,
+        operation: str,
+        method: str,
+        path: str,
+        *,
+        identity: OpenVikingIdentity | None,
+        retryable: bool,
+        **kwargs: Any,
+    ) -> httpx.Response:
+        attempts = self._config.max_retries + 1 if retryable else 1
+        for attempt in range(attempts):
+            try:
+                response = self._client.request(method, path, headers=self._headers(identity), **kwargs)
+            except httpx.TimeoutException as exc:
+                if attempt + 1 < attempts:
+                    time.sleep(_retry_delay(attempt))
+                    continue
+                raise OpenVikingTimeoutError(operation, "OpenViking request timed out") from exc
+            except httpx.TransportError as exc:
+                if attempt + 1 < attempts:
+                    time.sleep(_retry_delay(attempt))
+                    continue
+                raise OpenVikingUnavailableError(operation, "OpenViking is unavailable") from exc
+            if response.status_code in {429, 502, 503, 504} and attempt + 1 < attempts:
+                time.sleep(_retry_delay(attempt))
+                continue
+            if response.status_code >= 400:
+                try:
+                    payload = response.json()
+                except ValueError:
+                    payload = {}
+                error = payload.get("error", {}) if isinstance(payload, dict) else {}
+                code = str(error.get("code") or "HTTP_ERROR") if isinstance(error, dict) else "HTTP_ERROR"
+                message = str(error.get("message") or f"OpenViking HTTP {response.status_code}") if isinstance(error, dict) else f"OpenViking HTTP {response.status_code}"
+                error_type = OpenVikingAuthenticationError if response.status_code in {401, 403} else OpenVikingProtocolError
+                raise error_type(operation, message, status_code=response.status_code, code=code)
+            return response
+        raise OpenVikingUnavailableError(operation, "OpenViking request failed")
+
+
+def _retry_delay(attempt: int) -> float:
+    base_delay = 0.05 * (2**attempt)
+    return base_delay + random.uniform(0.0, base_delay)

backend/packages/harness/deerflow/agents/memory/backends/openviking/config.py

@@ -0,0 +1,135 @@
+"""Configuration for the OpenViking HTTP memory backend."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from typing import Any, Literal
+from urllib.parse import urlparse
+
+
+@dataclass(frozen=True)
+class OpenVikingConfig:
+    """Parsed backend-private configuration.
+
+    The backend is intentionally remote-only: DeerFlow talks to an independent
+    OpenViking server and does not import the OpenViking Python runtime.
+    """
+
+    base_url: str
+    storage_path: str
+    auth_mode: Literal["trusted", "dev"]
+    account: str
+    api_key: str | None = field(repr=False)
+    connect_timeout_seconds: float
+    read_timeout_seconds: float
+    write_timeout_seconds: float
+    pool_timeout_seconds: float
+    max_connections: int
+    max_keepalive_connections: int
+    max_retries: int
+    search_top_k: int
+    score_threshold: float | None
+    max_injection_chars: int
+    injection_query: str
+    startup_policy: Literal["fail_fast", "warn"]
+    read_failure_policy: Literal["fail_open", "raise"]
+    write_failure_policy: Literal["log_and_drop", "raise"]
+    allow_insecure_http: bool
+    allow_insecure_dev: bool
+    max_seen_message_ids: int
+
+    @classmethod
+    def from_backend_config(cls, backend_config: dict[str, Any] | None) -> OpenVikingConfig:
+        cfg = dict(backend_config or {})
+        retrieval = _mapping(cfg.pop("retrieval", {}), "retrieval")
+        failure_policy = _mapping(cfg.pop("failure_policy", {}), "failure_policy")
+
+        api_key_env = str(cfg.pop("api_key_env", "OPENVIKING_API_KEY")).strip()
+        api_key = os.environ.get(api_key_env) if api_key_env else None
+
+        result = cls(
+            base_url=str(cfg.pop("base_url", "http://127.0.0.1:1933")).rstrip("/"),
+            storage_path=str(cfg.pop("storage_path", "")),
+            auth_mode=str(cfg.pop("auth_mode", "trusted")).lower(),  # type: ignore[arg-type]
+            account=str(cfg.pop("account", "deerflow")).strip(),
+            api_key=api_key,
+            connect_timeout_seconds=float(cfg.pop("connect_timeout_seconds", 2.0)),
+            read_timeout_seconds=float(cfg.pop("read_timeout_seconds", 10.0)),
+            write_timeout_seconds=float(cfg.pop("write_timeout_seconds", 10.0)),
+            pool_timeout_seconds=float(cfg.pop("pool_timeout_seconds", 2.0)),
+            max_connections=int(cfg.pop("max_connections", 100)),
+            max_keepalive_connections=int(cfg.pop("max_keepalive_connections", 20)),
+            max_retries=int(cfg.pop("max_retries", 1)),
+            search_top_k=int(retrieval.pop("top_k", 8)),
+            score_threshold=_optional_float(retrieval.pop("score_threshold", None)),
+            max_injection_chars=int(retrieval.pop("max_injection_chars", 12_000)),
+            injection_query=str(
+                retrieval.pop(
+                    "injection_query",
+                    "user profile preferences important entities events ongoing goals constraints and prior decisions",
+                )
+            ).strip(),
+            startup_policy=str(cfg.pop("startup_policy", "fail_fast")).lower(),  # type: ignore[arg-type]
+            read_failure_policy=str(failure_policy.pop("read", "fail_open")).lower(),  # type: ignore[arg-type]
+            write_failure_policy=str(failure_policy.pop("write", "log_and_drop")).lower(),  # type: ignore[arg-type]
+            allow_insecure_http=bool(cfg.pop("allow_insecure_http", False)),
+            allow_insecure_dev=bool(cfg.pop("allow_insecure_dev", False)),
+            max_seen_message_ids=int(cfg.pop("max_seen_message_ids", 512)),
+        )
+
+        unknown = sorted([*cfg, *(f"retrieval.{key}" for key in retrieval), *(f"failure_policy.{key}" for key in failure_policy)])
+        if unknown:
+            raise ValueError(f"Unknown OpenViking backend_config fields: {', '.join(unknown)}")
+        result._validate()
+        return result
+
+    def _validate(self) -> None:
+        parsed = urlparse(self.base_url)
+        if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+            raise ValueError("OpenViking base_url must be an absolute http(s) URL")
+        if parsed.scheme == "http" and not self.allow_insecure_http and parsed.hostname not in {"127.0.0.1", "localhost", "openviking"}:
+            raise ValueError("OpenViking plain HTTP is allowed only for localhost/openviking; set allow_insecure_http=true for a trusted internal network")
+        if self.auth_mode not in {"trusted", "dev"}:
+            raise ValueError("OpenViking auth_mode must be 'trusted' or 'dev'")
+        if self.auth_mode == "dev" and not self.allow_insecure_dev:
+            raise ValueError("OpenViking auth_mode='dev' requires allow_insecure_dev=true")
+        if self.auth_mode == "trusted" and not self.account:
+            raise ValueError("OpenViking trusted auth requires a non-empty account")
+        for field_name in ("connect_timeout_seconds", "read_timeout_seconds", "write_timeout_seconds", "pool_timeout_seconds"):
+            if getattr(self, field_name) <= 0:
+                raise ValueError(f"OpenViking {field_name} must be > 0")
+        if not 1 <= self.max_connections <= 1000:
+            raise ValueError("OpenViking max_connections must be between 1 and 1000")
+        if not 0 <= self.max_keepalive_connections <= self.max_connections:
+            raise ValueError("OpenViking max_keepalive_connections must be between 0 and max_connections")
+        if not 0 <= self.max_retries <= 5:
+            raise ValueError("OpenViking max_retries must be between 0 and 5")
+        if not 1 <= self.search_top_k <= 100:
+            raise ValueError("OpenViking retrieval.top_k must be between 1 and 100")
+        if self.score_threshold is not None and not 0 <= self.score_threshold <= 1:
+            raise ValueError("OpenViking retrieval.score_threshold must be between 0 and 1")
+        if not 256 <= self.max_injection_chars <= 100_000:
+            raise ValueError("OpenViking retrieval.max_injection_chars must be between 256 and 100000")
+        if not self.injection_query:
+            raise ValueError("OpenViking retrieval.injection_query must not be empty")
+        if self.startup_policy not in {"fail_fast", "warn"}:
+            raise ValueError("OpenViking startup_policy must be 'fail_fast' or 'warn'")
+        if self.read_failure_policy not in {"fail_open", "raise"}:
+            raise ValueError("OpenViking failure_policy.read must be 'fail_open' or 'raise'")
+        if self.write_failure_policy not in {"log_and_drop", "raise"}:
+            raise ValueError("OpenViking failure_policy.write must be 'log_and_drop' or 'raise'")
+        if not 16 <= self.max_seen_message_ids <= 10_000:
+            raise ValueError("OpenViking max_seen_message_ids must be between 16 and 10000")
+
+
+def _mapping(value: Any, name: str) -> dict[str, Any]:
+    if value is None:
+        return {}
+    if not isinstance(value, dict):
+        raise ValueError(f"OpenViking {name} must be a mapping")
+    return dict(value)
+
+
+def _optional_float(value: Any) -> float | None:
+    return None if value is None else float(value)

backend/packages/harness/deerflow/agents/memory/backends/openviking/models.py

@@ -0,0 +1,63 @@
+"""Small transport-neutral models used by the OpenViking adapter."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class OpenVikingIdentity:
+    account: str
+    user: str
+
+
+@dataclass(frozen=True)
+class OpenVikingMessage:
+    message_id: str
+    role: str
+    content: str
+
+    def as_request(self) -> dict[str, Any]:
+        # OpenViking AddMessageRequest generates its own message ID and rejects
+        # unknown request fields, so the DeerFlow ID remains adapter-local for
+        # watermarking and is not sent over the wire.
+        return {"role": self.role, "content": self.content}
+
+
+@dataclass(frozen=True)
+class OpenVikingCommitResult:
+    status: str
+    task_id: str | None
+    archive_uri: str | None
+    archived: bool
+
+
+@dataclass(frozen=True)
+class OpenVikingSearchHit:
+    uri: str
+    context_type: str
+    category: str
+    score: float
+    abstract: str
+    overview: str | None
+    match_reason: str
+
+    @classmethod
+    def from_response(cls, value: dict[str, Any]) -> OpenVikingSearchHit:
+        return cls(
+            uri=str(value.get("uri") or ""),
+            context_type=str(value.get("context_type") or "memory"),
+            category=str(value.get("category") or "memory"),
+            score=float(value.get("score") or 0.0),
+            abstract=str(value.get("abstract") or ""),
+            overview=str(value["overview"]) if value.get("overview") else None,
+            match_reason=str(value.get("match_reason") or ""),
+        )
+
+
+@dataclass(frozen=True)
+class OpenVikingSessionContext:
+    latest_archive_overview: str
+    messages: list[dict[str, Any]]
+    estimated_tokens: int

backend/packages/harness/deerflow/agents/memory/backends/openviking/openviking_manager.py

@@ -0,0 +1,545 @@
+"""OpenViking HTTP implementation of the pluggable memory contract.
+
+This backend deliberately contains no OpenViking extraction or vector logic.
+It forwards filtered conversation turns to OpenViking Sessions and maps remote
+memory search results back to DeerFlow's backend-neutral shapes.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import logging
+import os
+import re
+import threading
+import time
+import weakref
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, ClassVar, Literal
+
+from pydantic import PrivateAttr
+
+# ABC contract -- the only DeerFlow import in this backend package.
+from deerflow.agents.memory.manager import MemoryManager
+
+from .client import OpenVikingClientError, OpenVikingHttpClient
+from .config import OpenVikingConfig
+from .models import OpenVikingIdentity, OpenVikingMessage, OpenVikingSearchHit
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_AGENT_SCOPE = "__default__"
+_SESSION_NAMESPACE = "deerflow-openviking-v1"
+_SAFE_SCOPE_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
+
+
+class OpenVikingMemoryManager(MemoryManager):
+    """Remote OpenViking memory backend for passive middleware mode."""
+
+    supports_search: ClassVar[bool] = True
+
+    _config: OpenVikingConfig = PrivateAttr()
+    _client: OpenVikingHttpClient = PrivateAttr()
+    _should_keep_hidden_message: Callable[[Any], bool] | None = PrivateAttr(default=None)
+    _session_locks: weakref.WeakValueDictionary[str, threading.RLock] = PrivateAttr(default_factory=weakref.WeakValueDictionary)
+    _session_locks_guard: threading.Lock = PrivateAttr(default_factory=threading.Lock)
+    _lifecycle: threading.Condition = PrivateAttr(default_factory=threading.Condition)
+    _active_operations: int = PrivateAttr(default=0)
+    _closed: bool = PrivateAttr(default=False)
+    _client_closed: bool = PrivateAttr(default=False)
+
+    def model_post_init(self, __context: Any) -> None:
+        self._config = OpenVikingConfig.from_backend_config(self.backend_config)
+        self._client = OpenVikingHttpClient(self._config)
+
+    @classmethod
+    def from_config(
+        cls,
+        backend_config: dict[str, Any] | None = None,
+        *,
+        mode: Literal["middleware", "tool"] = "middleware",
+        **host_hooks: Any,
+    ) -> OpenVikingMemoryManager:
+        if mode != "middleware":
+            raise ValueError("The OpenViking HTTP backend currently supports memory.mode='middleware' only")
+        instance = cls(backend_config=backend_config, mode=mode)
+        hook = host_hooks.get("should_keep_hidden_message")
+        instance._should_keep_hidden_message = hook if callable(hook) else None
+        return instance
+
+    def add(
+        self,
+        thread_id: str,
+        messages: list[Any],
+        *,
+        agent_name: str | None = None,
+        user_id: str | None = None,
+        trace_id: str | None = None,
+    ) -> None:
+        self._write_conversation(thread_id, messages, agent_name=agent_name, user_id=user_id)
+
+    def add_nowait(
+        self,
+        thread_id: str,
+        messages: list[Any],
+        *,
+        agent_name: str | None = None,
+        user_id: str | None = None,
+    ) -> None:
+        self._write_conversation(thread_id, messages, agent_name=agent_name, user_id=user_id)
+
+    async def aadd(
+        self,
+        thread_id: str,
+        messages: list[Any],
+        *,
+        agent_name: str | None = None,
+        user_id: str | None = None,
+        trace_id: str | None = None,
+    ) -> None:
+        await asyncio.to_thread(
+            self.add,
+            thread_id,
+            messages,
+            agent_name=agent_name,
+            user_id=user_id,
+            trace_id=trace_id,
+        )
+
+    def get_context(
+        self,
+        user_id: str | None,
+        *,
+        agent_name: str | None = None,
+        thread_id: str | None = None,
+    ) -> str:
+        if not self._begin_operation():
+            return ""
+        try:
+            try:
+                hits = self._search_hits(
+                    self._config.injection_query,
+                    top_k=self._config.search_top_k,
+                    user_id=user_id,
+                    agent_name=agent_name,
+                    category=None,
+                    thread_id=thread_id,
+                )
+            except OpenVikingClientError:
+                if self._config.read_failure_policy == "raise":
+                    raise
+                logger.warning("OpenViking context retrieval failed; continuing without injected memory", exc_info=True)
+                return ""
+            return _format_context(hits, max_chars=self._config.max_injection_chars)
+        finally:
+            self._end_operation()
+
+    async def aget_context(
+        self,
+        user_id: str | None,
+        *,
+        agent_name: str | None = None,
+        thread_id: str | None = None,
+    ) -> str:
+        return await asyncio.to_thread(
+            self.get_context,
+            user_id,
+            agent_name=agent_name,
+            thread_id=thread_id,
+        )
+
+    def search(
+        self,
+        query: str,
+        top_k: int = 5,
+        *,
+        user_id: str | None = None,
+        agent_name: str | None = None,
+        category: str | None = None,
+    ) -> list[dict[str, Any]]:
+        if not query.strip():
+            return []
+        if not self._begin_operation():
+            return []
+        try:
+            try:
+                hits = self._search_hits(
+                    query,
+                    top_k=top_k,
+                    user_id=user_id,
+                    agent_name=agent_name,
+                    category=category,
+                    thread_id=None,
+                )
+            except OpenVikingClientError:
+                if self._config.read_failure_policy == "raise":
+                    raise
+                logger.warning("OpenViking memory search failed; returning no results", exc_info=True)
+                return []
+            return [_hit_to_fact(hit) for hit in hits]
+        finally:
+            self._end_operation()
+
+    async def asearch(
+        self,
+        query: str,
+        top_k: int = 5,
+        *,
+        user_id: str | None = None,
+        agent_name: str | None = None,
+        category: str | None = None,
+    ) -> list[dict[str, Any]]:
+        return await asyncio.to_thread(
+            self.search,
+            query,
+            top_k,
+            user_id=user_id,
+            agent_name=agent_name,
+            category=category,
+        )
+
+    def warm(self) -> bool | None:
+        if not self._begin_operation():
+            return False
+        try:
+            try:
+                healthy = self._client.health()
+            except OpenVikingClientError:
+                if self._config.startup_policy == "fail_fast":
+                    raise
+                logger.warning("OpenViking health check failed; memory will run in degraded mode", exc_info=True)
+                return False
+            if not healthy and self._config.startup_policy == "fail_fast":
+                raise RuntimeError("OpenViking health check returned an unhealthy response")
+            if not healthy:
+                logger.warning("OpenViking health check returned unhealthy; memory will run in degraded mode")
+            return healthy
+        finally:
+            self._end_operation()
+
+    def shutdown_flush(self, timeout: float) -> bool:
+        """Stop new operations, drain in-flight work, then close the HTTP pool."""
+        deadline = time.monotonic() + max(0.0, timeout)
+        with self._lifecycle:
+            self._closed = True
+            while self._active_operations:
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    return False
+                self._lifecycle.wait(remaining)
+            if self._client_closed:
+                return True
+            self._client_closed = True
+        self._client.close()
+        return True
+
+    def _write_conversation(
+        self,
+        thread_id: str,
+        messages: list[Any],
+        *,
+        agent_name: str | None,
+        user_id: str | None,
+    ) -> None:
+        if not self._begin_operation():
+            logger.warning("OpenViking write ignored after backend shutdown")
+            return
+        try:
+            if not thread_id:
+                raise ValueError("OpenViking memory write requires thread_id")
+
+            identity = self._identity(user_id, agent_name)
+            session_id = _session_id(identity, thread_id)
+            lock = self._session_lock(session_id)
+            with lock:
+                state = self._load_state(session_id)
+                submitted_ids = list(state.get("submitted_message_ids", state.get("seen_message_ids", [])))
+                committed_ids = list(state.get("committed_message_ids", state.get("seen_message_ids", [])))
+                converted = _convert_messages(messages, self._should_keep_hidden_message)
+                prefix_count = _matching_submitted_prefix_count(state, submitted_ids, converted)
+                if prefix_count is not None:
+                    pending = converted[prefix_count:]
+                else:
+                    submitted = set(submitted_ids)
+                    pending = [message for message in converted if message.message_id not in submitted]
+                if not pending:
+                    if prefix_count is None and converted:
+                        state = {
+                            **state,
+                            "schema_version": 3,
+                            "submitted_prefix_count": len(converted),
+                            "submitted_prefix_digest": _message_sequence_digest(converted),
+                        }
+                        self._save_state(session_id, state)
+                    return
+                try:
+                    self._client.ensure_session(identity, session_id)
+                    self._client.add_messages(identity, session_id, pending)
+                except OpenVikingClientError:
+                    if self._config.write_failure_policy == "raise":
+                        raise
+                    logger.error(
+                        "OpenViking memory message submission failed; dropping this update (session=%s, messages=%d)",
+                        session_id,
+                        len(pending),
+                        exc_info=True,
+                    )
+                    return
+                submitted_ids.extend(message.message_id for message in pending)
+                state = {
+                    "schema_version": 3,
+                    "session_id": session_id,
+                    "submitted_message_ids": submitted_ids[-self._config.max_seen_message_ids :],
+                    "committed_message_ids": committed_ids[-self._config.max_seen_message_ids :],
+                    "submitted_prefix_count": len(converted),
+                    "submitted_prefix_digest": _message_sequence_digest(converted),
+                    "committed_prefix_count": state.get("committed_prefix_count"),
+                    "committed_prefix_digest": state.get("committed_prefix_digest"),
+                    "last_commit_task_id": state.get("last_commit_task_id"),
+                    "last_archive_uri": state.get("last_archive_uri"),
+                }
+                self._save_state(session_id, state)
+
+                try:
+                    commit = self._client.commit_session(identity, session_id)
+                except OpenVikingClientError:
+                    if self._config.write_failure_policy == "raise":
+                        raise
+                    logger.error(
+                        "OpenViking memory commit failed; preserving submitted watermark without retry (session=%s)",
+                        session_id,
+                        exc_info=True,
+                    )
+                    return
+
+                state = {
+                    **state,
+                    "schema_version": 3,
+                    "committed_message_ids": state["submitted_message_ids"],
+                    "committed_prefix_count": state["submitted_prefix_count"],
+                    "committed_prefix_digest": state["submitted_prefix_digest"],
+                    "last_commit_task_id": commit.task_id,
+                    "last_archive_uri": commit.archive_uri,
+                }
+                self._save_state(session_id, state)
+        finally:
+            self._end_operation()
+
+    def _search_hits(
+        self,
+        query: str,
+        *,
+        top_k: int,
+        user_id: str | None,
+        agent_name: str | None,
+        category: str | None,
+        thread_id: str | None,
+    ) -> list[OpenVikingSearchHit]:
+        identity = self._identity(user_id, agent_name)
+        session_id = _session_id(identity, thread_id) if thread_id else None
+        return self._client.search(
+            identity,
+            query,
+            top_k=max(1, min(top_k, 100)),
+            category=category,
+            session_id=session_id,
+        )
+
+    def _identity(self, user_id: str | None, agent_name: str | None) -> OpenVikingIdentity:
+        raw_user = str(user_id or "anonymous")
+        agent_scope = _canonical_agent_scope(agent_name)
+        # OpenViking trusted identity must be a safe path segment. Hashing the
+        # DeerFlow scope also prevents raw emails/usernames from leaving the
+        # Gateway and gives each agent a hard-isolated memory namespace.
+        digest = hashlib.sha256(f"{self._config.account}\0{raw_user}\0{agent_scope}".encode()).hexdigest()
+        return OpenVikingIdentity(account=self._config.account, user=f"df_{digest[:40]}")
+
+    def _session_lock(self, session_id: str) -> threading.RLock:
+        with self._session_locks_guard:
+            return self._session_locks.setdefault(session_id, threading.RLock())
+
+    def _begin_operation(self) -> bool:
+        with self._lifecycle:
+            if self._closed:
+                return False
+            self._active_operations += 1
+            return True
+
+    def _end_operation(self) -> None:
+        with self._lifecycle:
+            self._active_operations -= 1
+            if self._active_operations == 0:
+                self._lifecycle.notify_all()
+
+    def _state_path(self, session_id: str) -> Path:
+        root = Path(self._config.storage_path or ".") / "openviking" / "sessions"
+        return root / f"{session_id}.json"
+
+    def _load_state(self, session_id: str) -> dict[str, Any]:
+        path = self._state_path(session_id)
+        try:
+            value = json.loads(path.read_text(encoding="utf-8"))
+        except FileNotFoundError:
+            return {}
+        except (OSError, ValueError):
+            logger.warning("Ignoring unreadable OpenViking session watermark: %s", path, exc_info=True)
+            return {}
+        return value if isinstance(value, dict) else {}
+
+    def _save_state(self, session_id: str, state: dict[str, Any]) -> None:
+        path = self._state_path(session_id)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temp_path = path.with_suffix(f".{os.getpid()}.{threading.get_ident()}.tmp")
+        try:
+            temp_path.write_text(json.dumps(state, ensure_ascii=False, indent=2), encoding="utf-8")
+            os.replace(temp_path, path)
+        finally:
+            try:
+                temp_path.unlink(missing_ok=True)
+            except OSError:
+                logger.debug("Failed to remove OpenViking watermark temp file: %s", temp_path, exc_info=True)
+
+
+def _canonical_agent_scope(agent_name: str | None) -> str:
+    if agent_name is None:
+        return _DEFAULT_AGENT_SCOPE
+    value = str(agent_name).strip().lower()
+    if value == _DEFAULT_AGENT_SCOPE or not _SAFE_SCOPE_RE.fullmatch(value):
+        raise ValueError(f"Invalid OpenViking agent scope: {agent_name!r}")
+    return value
+
+
+def _session_id(identity: OpenVikingIdentity, thread_id: str) -> str:
+    digest = hashlib.sha256(f"{_SESSION_NAMESPACE}\0{identity.account}\0{identity.user}\0{thread_id}".encode()).hexdigest()
+    return f"df_{digest[:48]}"
+
+
+def _matching_submitted_prefix_count(
+    state: dict[str, Any],
+    submitted_ids: list[str],
+    messages: list[OpenVikingMessage],
+) -> int | None:
+    count = state.get("submitted_prefix_count")
+    digest = state.get("submitted_prefix_digest")
+    if isinstance(count, int) and 0 <= count <= len(messages) and isinstance(digest, str):
+        if _message_sequence_digest(messages[:count]) == digest:
+            return count
+        return None
+
+    # Schema v2 only retained a recent suffix of submitted IDs. When that
+    # suffix still appears intact, it safely anchors the append-only prefix and
+    # avoids a one-time duplicate submission during migration to schema v3.
+    if submitted_ids and len(submitted_ids) <= len(messages):
+        message_ids = [message.message_id for message in messages]
+        width = len(submitted_ids)
+        for start in range(len(message_ids) - width, -1, -1):
+            if message_ids[start : start + width] == submitted_ids:
+                return start + width
+    return None
+
+
+def _message_sequence_digest(messages: list[OpenVikingMessage]) -> str:
+    digest = hashlib.sha256()
+    for message in messages:
+        encoded = message.message_id.encode()
+        digest.update(len(encoded).to_bytes(8, "big"))
+        digest.update(encoded)
+    return digest.hexdigest()
+
+
+def _convert_messages(
+    messages: list[Any],
+    should_keep_hidden_message: Callable[[Any], bool] | None,
+) -> list[OpenVikingMessage]:
+    converted: list[OpenVikingMessage] = []
+    for index, message in enumerate(messages):
+        role = _message_role(message)
+        if role not in {"user", "assistant"}:
+            continue
+        additional_kwargs = _message_value(message, "additional_kwargs", {})
+        if not isinstance(additional_kwargs, dict):
+            additional_kwargs = {}
+        if additional_kwargs.get("hide_from_ui") and not (should_keep_hidden_message and should_keep_hidden_message(additional_kwargs)):
+            continue
+        tool_calls = _message_value(message, "tool_calls", [])
+        if role == "assistant" and tool_calls:
+            continue
+        content = _text_content(_message_value(message, "content", ""))
+        if not content.strip():
+            continue
+        native_id = _message_value(message, "id", None)
+        stable_id = str(native_id) if native_id else hashlib.sha256(f"{role}\0{index}\0{content}".encode()).hexdigest()
+        converted.append(OpenVikingMessage(message_id=f"df_{stable_id}", role=role, content=content.strip()))
+    return converted
+
+
+def _message_role(message: Any) -> str | None:
+    value = _message_value(message, "type", None) or _message_value(message, "role", None)
+    if value in {"human", "user"}:
+        return "user"
+    if value in {"ai", "assistant"}:
+        return "assistant"
+    name = type(message).__name__.lower()
+    if "human" in name:
+        return "user"
+    if "ai" in name:
+        return "assistant"
+    return None
+
+
+def _message_value(message: Any, key: str, default: Any) -> Any:
+    return message.get(key, default) if isinstance(message, dict) else getattr(message, key, default)
+
+
+def _text_content(content: Any) -> str:
+    if isinstance(content, str):
+        return content
+    if not isinstance(content, list):
+        return str(content) if content is not None else ""
+    parts: list[str] = []
+    for block in content:
+        if isinstance(block, str):
+            parts.append(block)
+        elif isinstance(block, dict) and block.get("type") in {"text", "input_text", "output_text"}:
+            text = block.get("text")
+            if text:
+                parts.append(str(text))
+    return "\n".join(parts)
+
+
+def _hit_content(hit: OpenVikingSearchHit) -> str:
+    return (hit.overview or hit.abstract).strip()
+
+
+def _hit_to_fact(hit: OpenVikingSearchHit) -> dict[str, Any]:
+    return {
+        "id": hit.uri,
+        "content": _hit_content(hit),
+        "category": hit.category or "memory",
+        "confidence": hit.score,
+        "source": hit.uri,
+        "score": hit.score,
+    }
+
+
+def _format_context(hits: list[OpenVikingSearchHit], *, max_chars: int) -> str:
+    lines: list[str] = []
+    seen: set[str] = set()
+    for hit in hits:
+        content = " ".join(_hit_content(hit).split())
+        key = content.casefold()
+        if not content or key in seen:
+            continue
+        seen.add(key)
+        line = f"- [{hit.category or 'memory'}] {content}"
+        candidate = "\n".join([*lines, line])
+        if len(candidate) > max_chars:
+            remaining = max_chars - len("\n".join(lines)) - (1 if lines else 0)
+            if remaining > 16:
+                lines.append(f"{line[: max(0, remaining - 1)]}…")
+            break
+        lines.append(line)
+    return "\n".join(lines)

backend/packages/harness/deerflow/agents/memory/manager.py

@@ -143,6 +143,10 @@ class MemoryManager(BaseModel):
     # that fails fast at instantiation rather than silently returning empty
     # results). Default False: a new backend must explicitly opt in to tool mode.
     supports_search: ClassVar[bool] = False
+    # Backends that rely on conversation-level extraction instead of fact CRUD
+    # can retain MemoryMiddleware writes while tool mode supplies query-aware
+    # search. Most backends keep tool mode fully model-directed.
+    requires_passive_writes_in_tool_mode: ClassVar[bool] = False
 
     @model_validator(mode="after")
     def _check_invariants(self) -> MemoryManager:
@@ -585,6 +589,15 @@ def _resolve_manager_class(manager_class: str) -> type[MemoryManager]:
     )
 
 
+def backend_requires_passive_writes_in_tool_mode(manager_class: str) -> bool:
+    """Return whether a backend needs middleware writes in tool mode.
+
+    Resolve the class without constructing it so agent assembly does not run
+    backend startup checks or perform network I/O.
+    """
+    return _resolve_manager_class(manager_class).requires_passive_writes_in_tool_mode
+
+
 # ── Host-default hook providers (passed to from_config by the factory) ────
 #
 # These callables are the host's defaults for the slots a backend may consume

backend/packages/harness/deerflow/agents/memory/tools.py

@@ -3,10 +3,10 @@
 Exposes memory_search, memory_add, memory_update, memory_delete as
 LangChain @tool functions the model can call directly.
 
-When memory.mode == "tool", these tools are registered on the agent
-instead of appending MemoryMiddleware.  The model gains agency over
-its own persistent memory: it decides what to remember, when to
-search, and when to update or remove stale facts.
+When memory.mode == "tool", these tools are registered on the agent. Most
+backends omit MemoryMiddleware so the model drives persistence; a backend that
+sets ``requires_passive_writes_in_tool_mode`` retains conversation writes while
+the tools provide query-aware recall.
 
 Backend-agnostic: every tool goes through the ``MemoryManager`` ABC
 (:func:`get_memory_manager`) -- ``search``/``get_memory`` are tier-2 methods;

backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py

@@ -2,6 +2,7 @@
 
 import json
 import logging
+import re
 from collections.abc import Callable
 from hashlib import sha256
 from typing import Any, override
@@ -56,6 +57,8 @@ MAX_FIELD_TEXT_CHARS = 200
 # leaving headroom for question/context.
 MAX_FORM_SERIALIZED_BYTES = 16_384
 
+_XML_TAG_RE = re.compile(r"</?[A-Za-z_][\w:.-]*(?:\s[^<>]*?)?\s*/?>")
+
 
 class ClarificationMiddlewareState(AgentState):
     """Compatible with the `ThreadState` schema."""
@@ -100,7 +103,9 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]):
 
         if options is None:
             return []
-        if not isinstance(options, list):
+        if isinstance(options, dict):
+            options = self._flatten_dict_option_values(options)
+        elif not isinstance(options, list):
             options = [options]
 
         # Trim, drop blanks, and dedupe (order-preserving): the frontend parser
@@ -109,13 +114,31 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]):
         normalized: list[str] = []
         seen: set[str] = set()
         for option in options:
-            text = str(option).strip()
+            text = _XML_TAG_RE.sub("", str(option)).strip()
             if not text or text in seen:
                 continue
             seen.add(text)
             normalized.append(text)
         return normalized
 
+    @staticmethod
+    def _flatten_dict_option_values(value: dict[str, Any]) -> list[str | int | float]:
+        """Flatten scalar leaves from XML-to-dict option payloads in source order."""
+        flattened: list[str | int | float] = []
+
+        def collect(nested: Any) -> None:
+            if isinstance(nested, dict):
+                for item in nested.values():
+                    collect(item)
+            elif isinstance(nested, list):
+                for item in nested:
+                    collect(item)
+            elif isinstance(nested, str | int | float):
+                flattened.append(nested)
+
+        collect(value)
+        return flattened
+
     @staticmethod
     def _normalize_bool(raw: Any) -> bool:
         """Coerce a model-provided boolean; some models serialize booleans as strings or 1/0."""

backend/packages/harness/deerflow/agents/middlewares/dynamic_context_middleware.py

@@ -41,6 +41,7 @@ from langchain_core.messages import HumanMessage, SystemMessage
 from langgraph.runtime import Runtime
 
 from deerflow.runtime.context_keys import CURRENT_RUN_PRE_EXISTING_MESSAGE_IDS_KEY
+from deerflow.runtime.user_context import resolve_runtime_user_id
 
 if TYPE_CHECKING:
     from deerflow.config.app_config import AppConfig
@@ -60,6 +61,23 @@ _DYNAMIC_CONTEXT_REMINDER_KEY = "dynamic_context_reminder"
 # so it is never exposed to user-influenceable memory content.
 _REMINDER_DATE_KEY = "reminder_date"
 _SUMMARY_MESSAGE_NAME = "summary"
+# Suffix the ID-swap gives the real user message; the reminder SystemMessage
+# takes the original id so ``add_messages`` can replace it in place.
+INJECTED_USER_MESSAGE_ID_SUFFIX = "__user"
+
+
+def strip_injected_user_message_id_suffix(message_id: str | None) -> str | None:
+    """Return the id *message_id* had before the reminder ID-swap.
+
+    Replaying a persisted user turn must feed the graph the id the client
+    originally sent: a ``{id}__user`` message is skipped as an injection target,
+    so replaying one into a state that has no reminder yet silently drops the
+    date and memory block for that turn.
+    """
+
+    if isinstance(message_id, str) and message_id.endswith(INJECTED_USER_MESSAGE_ID_SUFFIX):
+        return message_id[: -len(INJECTED_USER_MESSAGE_ID_SUFFIX)] or message_id
+    return message_id
 
 
 def _extract_date(content: str) -> str | None:
@@ -120,7 +138,7 @@ def _is_user_injection_target(message: object) -> bool:
     # (id__user__user__user...) and ghost-message re-execution.
     # Using endswith (not substring "in") avoids false positives on IDs that
     # happen to contain "__user" in the middle.
-    if message.id and str(message.id).endswith("__user"):
+    if message.id and str(message.id).endswith(INJECTED_USER_MESSAGE_ID_SUFFIX):
         return False
     return True
 
@@ -148,7 +166,7 @@ class DynamicContextMiddleware(AgentMiddleware):
         self._agent_name = agent_name
         self._app_config = app_config
 
-    def _build_full_reminder(self) -> tuple[str, str | None]:
+    def _build_full_reminder(self, runtime: Runtime | None = None) -> tuple[str, str | None]:
         """Return (date_reminder, memory_block | None).
 
         Framework-owned data (date) is separated from user-owned data (memory)
@@ -159,7 +177,15 @@ class DynamicContextMiddleware(AgentMiddleware):
         from deerflow.agents.lead_agent.prompt import _get_memory_context
 
         injection_enabled = self._app_config.memory.injection_enabled if self._app_config else True
-        memory_context = _get_memory_context(self._agent_name, app_config=self._app_config) if injection_enabled else ""
+        memory_context = (
+            _get_memory_context(
+                self._agent_name,
+                app_config=self._app_config,
+                user_id=resolve_runtime_user_id(runtime),
+            )
+            if injection_enabled
+            else ""
+        )
         current_date = datetime.now().strftime("%Y-%m-%d, %A")
 
         date_reminder = "\n".join(
@@ -232,14 +258,14 @@ class DynamicContextMiddleware(AgentMiddleware):
         messages.append(
             HumanMessage(
                 content=original.content,
-                id=f"{stable_id}__user",
+                id=f"{stable_id}{INJECTED_USER_MESSAGE_ID_SUFFIX}",
                 name=original.name,
                 additional_kwargs=original.additional_kwargs,
             )
         )
         return messages
 
-    def _inject(self, state) -> dict | None:
+    def _inject(self, state, runtime: Runtime | None = None) -> dict | None:
         messages = list(state.get("messages", []))
         if not messages:
             return None
@@ -258,7 +284,7 @@ class DynamicContextMiddleware(AgentMiddleware):
             first_idx = next((i for i, m in enumerate(messages) if _is_user_injection_target(m)), None)
             if first_idx is None:
                 return None
-            date_reminder, memory_block = self._build_full_reminder()
+            date_reminder, memory_block = self._build_full_reminder(runtime)
             logger.info(
                 "DynamicContextMiddleware: injecting full reminder (has_memory=%s) into first HumanMessage id=%r",
                 memory_block is not None,
@@ -282,7 +308,7 @@ class DynamicContextMiddleware(AgentMiddleware):
 
     @override
     def before_agent(self, state, runtime: Runtime) -> dict | None:
-        result = self._inject(state)
+        result = self._inject(state, runtime)
         self._record_effective_memory(state, result, runtime)
         return result
 
@@ -301,7 +327,7 @@ class DynamicContextMiddleware(AgentMiddleware):
         # rather than hanging. Frozen context already in state remains active.
         try:
             result = await asyncio.wait_for(
-                asyncio.to_thread(self._inject, state),
+                asyncio.to_thread(self._inject, state, runtime),
                 timeout=_INJECT_TIMEOUT_SECONDS,
             )
         except TimeoutError:

backend/packages/harness/deerflow/agents/middlewares/memory_middleware.py

@@ -1,5 +1,6 @@
 """Middleware for memory mechanism."""
 
+import asyncio
 import logging
 from typing import TYPE_CHECKING, override
 
@@ -10,7 +11,7 @@ from langgraph.runtime import Runtime
 
 from deerflow.agents.memory import get_memory_manager
 from deerflow.config.memory_config import get_memory_config
-from deerflow.runtime.user_context import get_effective_user_id
+from deerflow.runtime.user_context import resolve_runtime_user_id
 from deerflow.trace_context import DEERFLOW_TRACE_METADATA_KEY, get_current_trace_id, normalize_trace_id
 
 if TYPE_CHECKING:
@@ -49,17 +50,8 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
         self._agent_name = agent_name
         self._memory_config = memory_config
 
-    @override
-    def after_agent(self, state: MemoryMiddlewareState, runtime: Runtime) -> dict | None:
-        """Queue conversation for memory update after agent completes.
-
-        Args:
-            state: The current agent state.
-            runtime: The runtime context.
-
-        Returns:
-            None (no state changes needed from this middleware).
-        """
+    def _resolve_add_args(self, state: MemoryMiddlewareState, runtime: Runtime) -> tuple[str, list, str, str | None] | None:
+        """Resolve one write request without invoking the manager."""
         config = self._memory_config or get_memory_config()
         if not config.enabled:
             return None
@@ -82,7 +74,7 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
         # Capture user_id at enqueue time while the request context is still alive.
         # threading.Timer fires on a different thread where ContextVar values are not
         # propagated, so we must store user_id explicitly in ConversationContext.
-        user_id = get_effective_user_id()
+        user_id = resolve_runtime_user_id(runtime)
         runtime_context = runtime.context if isinstance(runtime.context, dict) else {}
         trace_id = normalize_trace_id(runtime_context.get(DEERFLOW_TRACE_METADATA_KEY))
         if trace_id is None:
@@ -95,6 +87,16 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
         if trace_id is None:
             trace_id = get_current_trace_id()
 
+        return thread_id, messages, user_id, trace_id
+
+    @override
+    def after_agent(self, state: MemoryMiddlewareState, runtime: Runtime) -> dict | None:
+        """Queue conversation for memory update after agent completes."""
+        add_args = self._resolve_add_args(state, runtime)
+        if add_args is None:
+            return None
+        thread_id, messages, user_id, trace_id = add_args
+
         # Hand raw messages to the manager; the backend filters to user + final-AI
         # turns, validates, detects correction/reinforcement, and enqueues.
         get_memory_manager().add(
@@ -106,3 +108,20 @@ class MemoryMiddleware(AgentMiddleware[MemoryMiddlewareState]):
         )
 
         return None
+
+    @override
+    async def aafter_agent(self, state: MemoryMiddlewareState, runtime: Runtime) -> dict | None:
+        """Use the manager's async boundary on LangGraph's async execution path."""
+        add_args = self._resolve_add_args(state, runtime)
+        if add_args is None:
+            return None
+        thread_id, messages, user_id, trace_id = add_args
+        manager = await asyncio.to_thread(get_memory_manager)
+        await manager.aadd(
+            thread_id,
+            messages,
+            agent_name=self._agent_name,
+            user_id=user_id,
+            trace_id=trace_id,
+        )
+        return None

backend/packages/harness/deerflow/agents/middlewares/skill_tool_policy_middleware.py

@@ -40,7 +40,7 @@ type _PolicySignature = tuple[str, tuple[str, ...]]
 
 
 class SkillToolPolicyMiddleware(AgentMiddleware[AgentState]):
-    """Restrict lead tools to declarations from slash/in-context skills.
+    """Restrict agent tools to declarations from slash/in-context skills.
 
     Merely enabling a skill makes it discoverable; it does not activate its
     authority policy. A skill becomes policy-active when the user slash-activates

backend/packages/harness/deerflow/agents/middlewares/thread_data_middleware.py

@@ -10,7 +10,7 @@ from langgraph.runtime import Runtime
 
 from deerflow.agents.thread_state import ThreadDataState
 from deerflow.config.paths import Paths, get_paths
-from deerflow.runtime.user_context import get_effective_user_id
+from deerflow.runtime.user_context import resolve_runtime_user_id
 
 logger = logging.getLogger(__name__)
 
@@ -89,7 +89,7 @@ class ThreadDataMiddleware(AgentMiddleware[ThreadDataMiddlewareState]):
         if thread_id is None:
             raise ValueError("Thread ID is required in runtime context or config.configurable")
 
-        user_id = get_effective_user_id()
+        user_id = resolve_runtime_user_id(runtime)
 
         if self._lazy_init:
             # Lazy initialization: only compute paths, don't create directories

backend/packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py

@@ -1,6 +1,7 @@
 """Tool error handling middleware and shared runtime middleware builders."""
 
 import logging
+import secrets
 from collections.abc import Awaitable, Callable
 from typing import TYPE_CHECKING, override
 
@@ -318,6 +319,8 @@ def build_subagent_runtime_middlewares(
     deferred_setup: "DeferredToolSetup | None" = None,
     mcp_routing_middleware: AgentMiddleware | None = None,
     agent_name: str | None = None,
+    available_skills: set[str] | None = None,
+    user_id: str | None = None,
     authorization_provider=None,
 ) -> list[AgentMiddleware]:
     """Middlewares shared by subagent runtime before subagent-only middlewares."""
@@ -335,6 +338,31 @@ def build_subagent_runtime_middlewares(
         authorization_infrastructure_tool_names=(frozenset({deferred_setup.tool_search_tool.name}) if authorization_provider is not None and deferred_setup is not None and deferred_setup.tool_search_tool is not None else frozenset()),
     )
 
+    # Enabled/configured skills are discoverable metadata, not automatically
+    # active authority. Mirror the lead agent's activation + policy pair so a
+    # subagent keeps its ordinary tool set until a slash command or a completed
+    # SKILL.md read activates the corresponding allowed-tools declaration.
+    from deerflow.agents.middlewares.skill_activation_middleware import SkillActivationMiddleware
+    from deerflow.agents.middlewares.skill_tool_policy_middleware import SkillToolPolicyMiddleware
+
+    slash_source_owner_token = secrets.token_urlsafe(24)
+    middlewares.append(
+        SkillActivationMiddleware(
+            available_skills=available_skills,
+            app_config=app_config,
+            user_id=user_id,
+            slash_source_owner_token=slash_source_owner_token,
+        )
+    )
+    middlewares.append(
+        SkillToolPolicyMiddleware(
+            available_skills=available_skills,
+            app_config=app_config,
+            user_id=user_id,
+            slash_source_owner_token=slash_source_owner_token,
+        )
+    )
+
     if model_name is None and app_config.models:
         model_name = app_config.models[0].name
 

backend/packages/harness/deerflow/agents/middlewares/uploads_middleware.py

@@ -17,7 +17,7 @@ from langgraph.runtime import Runtime
 
 from deerflow.agents.middlewares.input_sanitization_middleware import neutralize_untrusted_tags
 from deerflow.config.paths import Paths, get_paths
-from deerflow.runtime.user_context import get_effective_user_id
+from deerflow.runtime.user_context import resolve_runtime_user_id
 from deerflow.uploads.manager import is_upload_staging_file
 from deerflow.utils.file_outline import extract_outline_for_file
 from deerflow.utils.messages import ORIGINAL_USER_CONTENT_KEY, message_content_to_text
@@ -226,11 +226,17 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
                 thread_id = get_config().get("configurable", {}).get("thread_id")
             except RuntimeError:
                 pass
-        uploads_dir = self._paths.sandbox_uploads_dir(thread_id, user_id=get_effective_user_id()) if thread_id else None
+        uploads_dir = self._paths.sandbox_uploads_dir(thread_id, user_id=resolve_runtime_user_id(runtime)) if thread_id else None
 
         # Get newly uploaded files from the current message's additional_kwargs.files
         new_files = self._files_from_kwargs(last_message, uploads_dir) or []
         if not new_files:
+            if (last_message.additional_kwargs or {}).get("files"):
+                logger.info(
+                    "UploadsMiddleware: files metadata was present but no files were found on disk (thread_id=%s, uploads_dir=%s)",
+                    thread_id,
+                    uploads_dir,
+                )
             # Clear stale uploaded_files so list_uploaded_files doesn't
             # exclude files that became historical after the previous turn.
             return {"uploaded_files": []}
@@ -294,7 +300,9 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
         ``stat``, reading sibling ``.md`` outlines). When the graph runs async,
         langgraph would otherwise execute the sync hook directly on the event
         loop, so it is dispatched to a worker thread via ``run_in_executor``.
-        ``run_in_executor`` copies the current context, so the ``user_id``
-        contextvar read by ``get_effective_user_id()`` is preserved.
+        ``run_in_executor`` copies the current context, preserving both
+        LangGraph's runnable config and DeerFlow's request ContextVar fallback.
+        The runtime itself is also passed explicitly for the authoritative
+        ``runtime.context["user_id"]`` channel.
         """
         return await run_in_executor(None, self.before_agent, state, runtime)

backend/packages/harness/deerflow/agents/thread_state.py

@@ -17,10 +17,21 @@ from langgraph.graph.message import REMOVE_ALL_MESSAGES
 
 import deerflow.checkpoint_patches as _checkpoint_patches  # noqa: F401 - import-time saver fixes
 from deerflow.agents.goal_state import GoalState
-from deerflow.config.database_config import CheckpointChannelMode
+from deerflow.config.database_config import DEFAULT_CHECKPOINT_SNAPSHOT_FREQUENCY, CheckpointChannelMode
 from deerflow.subagents.status_contract import SUBAGENT_STATUS_VALUES
 
 
+def _resolve_snapshot_frequency(snapshot_frequency: int | None) -> int:
+    """Resolve the effective cadence: explicit value, else process-frozen,
+    else default. Imported lazily — ``deerflow.runtime.__init__`` reaches this
+    module via ``checkpoint_state``, so a top-level import would cycle."""
+    if snapshot_frequency is not None:
+        return snapshot_frequency
+    from deerflow.runtime.checkpoint_mode import resolve_checkpoint_snapshot_frequency
+
+    return resolve_checkpoint_snapshot_frequency()
+
+
 class SandboxState(TypedDict):
     sandbox_id: NotRequired[str | None]
 
@@ -352,61 +363,21 @@ def merge_message_writes(state: list[AnyMessage], writes: Sequence[Any]) -> list
     return [message for message in messages if message is not None]
 
 
-DEFAULT_DELTA_SNAPSHOT_FREQUENCY = 1000
-
-_frozen_delta_snapshot_frequency: int | None = None
-
-
-def delta_messages_field(snapshot_frequency: int) -> Any:
+def delta_messages_field(snapshot_frequency: int = DEFAULT_CHECKPOINT_SNAPSHOT_FREQUENCY) -> Any:
+    """Messages field annotation with a ``DeltaChannel`` at the given cadence."""
     return Annotated[
         list[AnyMessage],
         DeltaChannel(merge_message_writes, snapshot_frequency=snapshot_frequency),
     ]
 
 
-DELTA_MESSAGES_FIELD = delta_messages_field(DEFAULT_DELTA_SNAPSHOT_FREQUENCY)
+DELTA_MESSAGES_FIELD = delta_messages_field()
 
 
 class DeltaThreadState(ThreadState):
     messages: DELTA_MESSAGES_FIELD
 
 
-def freeze_delta_snapshot_frequency(frequency: int) -> int:
-    """Freeze the process-wide delta snapshot frequency (restart-required)."""
-    # Lazy import: deerflow.runtime.__init__ imports checkpoint_state, which
-    # imports this module — a top-level import would be circular.
-    from deerflow.runtime.checkpoint_mode import CheckpointModeReconfigurationError
-
-    global _frozen_delta_snapshot_frequency
-    if frequency <= 0:
-        raise ValueError("snapshot frequency must be positive")
-    if _frozen_delta_snapshot_frequency is None:
-        _frozen_delta_snapshot_frequency = frequency
-    elif _frozen_delta_snapshot_frequency != frequency:
-        raise CheckpointModeReconfigurationError(f"checkpoint delta snapshot frequency is frozen at {_frozen_delta_snapshot_frequency}; refusing to reconfigure to {frequency} — restart the process to change it")
-    return _frozen_delta_snapshot_frequency
-
-
-def resolved_delta_snapshot_frequency() -> int:
-    return _frozen_delta_snapshot_frequency or DEFAULT_DELTA_SNAPSHOT_FREQUENCY
-
-
-def reset_delta_snapshot_frequency_for_tests() -> None:
-    global _frozen_delta_snapshot_frequency
-    _frozen_delta_snapshot_frequency = None
-
-
-@cache
-def _delta_thread_state_for_frequency(snapshot_frequency: int) -> type:
-    if snapshot_frequency == DEFAULT_DELTA_SNAPSHOT_FREQUENCY:
-        # Preserve the canonical class so identity checks (and its public
-        # re-export) keep holding at the default frequency.
-        return DeltaThreadState
-    annotations = get_type_hints(ThreadState, include_extras=True)
-    annotations["messages"] = delta_messages_field(snapshot_frequency)
-    return TypedDict(f"DeltaThreadState_f{snapshot_frequency}", annotations, total=True)
-
-
 THREAD_STATE_REDUCER_FIELDS = frozenset(
     {
         "messages",
@@ -422,20 +393,35 @@ THREAD_STATE_REDUCER_FIELDS = frozenset(
 )
 
 
-def get_thread_state_schema(mode: CheckpointChannelMode) -> type:
-    if mode == "delta":
-        return _delta_thread_state_for_frequency(resolved_delta_snapshot_frequency())
-    return ThreadState
-
-
-def adapt_state_schema_for_mode(schema: type, mode: CheckpointChannelMode) -> type:
-    if mode == "full":
-        return schema
-    return _adapt_state_schema_for_mode(schema, mode, resolved_delta_snapshot_frequency())
+def get_thread_state_schema(mode: CheckpointChannelMode, snapshot_frequency: int | None = None) -> type:
+    if mode != "delta":
+        return ThreadState
+    return _delta_thread_state_schema(_resolve_snapshot_frequency(snapshot_frequency))
 
 
 @cache
-def _adapt_state_schema_for_mode(schema: type, mode: CheckpointChannelMode, snapshot_frequency: int) -> type:
+def _delta_thread_state_schema(snapshot_frequency: int) -> type:
+    """Delta thread schema keyed by cadence; the default keeps the static
+    ``DeltaThreadState`` identity so existing type checks keep holding."""
+    if snapshot_frequency == DEFAULT_CHECKPOINT_SNAPSHOT_FREQUENCY:
+        return DeltaThreadState
+    annotations = get_type_hints(ThreadState, include_extras=True)
+    annotations["messages"] = delta_messages_field(snapshot_frequency)
+    return TypedDict(
+        f"DeltaThreadState_f{snapshot_frequency}",
+        annotations,
+        total=getattr(ThreadState, "__total__", True),
+    )
+
+
+def adapt_state_schema_for_mode(schema: type, mode: CheckpointChannelMode, snapshot_frequency: int | None = None) -> type:
+    if mode == "full":
+        return schema
+    return _adapt_state_schema_for_delta(schema, _resolve_snapshot_frequency(snapshot_frequency))
+
+
+@cache
+def _adapt_state_schema_for_delta(schema: type, snapshot_frequency: int) -> type:
     annotations = get_type_hints(schema, include_extras=True)
     annotations["messages"] = delta_messages_field(snapshot_frequency)
     return TypedDict(
@@ -445,9 +431,10 @@ def _adapt_state_schema_for_mode(schema: type, mode: CheckpointChannelMode, snap
     )
 
 
-def normalize_middleware_state_schemas(middleware: Sequence[Any], mode: CheckpointChannelMode) -> list[Any]:
+def normalize_middleware_state_schemas(middleware: Sequence[Any], mode: CheckpointChannelMode, snapshot_frequency: int | None = None) -> list[Any]:
     if mode == "full":
         return list(middleware)
+    resolved_frequency = _resolve_snapshot_frequency(snapshot_frequency)
     normalized = []
     for item in middleware:
         schema = getattr(item, "state_schema", None)
@@ -455,6 +442,6 @@ def normalize_middleware_state_schemas(middleware: Sequence[Any], mode: Checkpoi
             normalized.append(item)
             continue
         adapted = copy.copy(item)
-        adapted.state_schema = adapt_state_schema_for_mode(schema, mode)
+        adapted.state_schema = adapt_state_schema_for_mode(schema, mode, resolved_frequency)
         normalized.append(adapted)
     return normalized

backend/packages/harness/deerflow/client.py

@@ -37,7 +37,7 @@ from langchain_core.runnables import RunnableConfig
 
 from deerflow.agents.lead_agent.agent import build_middlewares
 from deerflow.agents.lead_agent.prompt import apply_prompt_template, get_enabled_skills_for_config
-from deerflow.agents.thread_state import freeze_delta_snapshot_frequency, get_thread_state_schema, normalize_middleware_state_schemas
+from deerflow.agents.thread_state import get_thread_state_schema, normalize_middleware_state_schemas
 from deerflow.authz.principal import build_principal_from_context
 from deerflow.config.agents_config import AGENT_NAME_PATTERN
 from deerflow.config.app_config import get_app_config, is_trace_correlation_enabled, reload_app_config
@@ -48,6 +48,7 @@ from deerflow.runtime import CheckpointStateAccessor
 from deerflow.runtime.checkpoint_mode import (
     ensure_checkpoint_mode_compatible,
     freeze_checkpoint_channel_mode,
+    freeze_checkpoint_snapshot_frequency,
     inject_checkpoint_mode,
 )
 from deerflow.runtime.goal import DEFAULT_MAX_GOAL_CONTINUATIONS, build_goal_state, goal_thread_lock, read_thread_goal, write_thread_goal
@@ -192,7 +193,7 @@ class DeerFlowClient:
             reload_app_config(config_path)
         self._app_config = get_app_config()
         self._checkpoint_channel_mode = freeze_checkpoint_channel_mode(self._app_config.database.checkpoint_channel_mode)
-        freeze_delta_snapshot_frequency(self._app_config.database.checkpoint_delta_snapshot_frequency)
+        self._checkpoint_snapshot_frequency = freeze_checkpoint_snapshot_frequency(self._app_config.database.checkpoint_delta.snapshot_frequency)
 
         if agent_name is not None and not AGENT_NAME_PATTERN.match(agent_name):
             raise ValueError(f"Invalid agent name '{agent_name}'. Must match pattern: {AGENT_NAME_PATTERN.pattern}")
@@ -288,6 +289,7 @@ class DeerFlowClient:
             self._agent_name,
             frozenset(self._available_skills) if self._available_skills is not None else None,
             self._checkpoint_channel_mode,
+            self._checkpoint_snapshot_frequency,
             authorization_identity,
         )
 
@@ -359,6 +361,7 @@ class DeerFlowClient:
                     authorization_provider=_authz_provider,
                 ),
                 self._checkpoint_channel_mode,
+                self._checkpoint_snapshot_frequency,
             ),
             "system_prompt": apply_prompt_template(
                 subagent_enabled=subagent_enabled,
@@ -372,7 +375,7 @@ class DeerFlowClient:
                 user_id=effective_user_id,
                 skill_names=skill_setup.skill_names or None,
             ),
-            "state_schema": get_thread_state_schema(self._checkpoint_channel_mode),
+            "state_schema": get_thread_state_schema(self._checkpoint_channel_mode, self._checkpoint_snapshot_frequency),
         }
         checkpointer = self._checkpointer
         if checkpointer is None:

backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py

@@ -167,6 +167,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
         container_prefix: deer-flow-sandbox
         idle_timeout: 600               # Idle timeout in seconds (0 to disable)
         replicas: 3                     # Max concurrent sandbox containers (LRU eviction when exceeded)
+        thread_data_mounts: null        # null = backend auto-detection
         mounts:                         # Volume mounts for local containers
           - host_path: /path/on/host
             container_path: /path/in/container
@@ -255,8 +256,12 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
 
         Local container backends bind-mount the thread data directories, so files
         written by the gateway are already visible when the sandbox starts.
-        Remote backends may require explicit file sync.
+        Remote backends may require explicit file sync. Operators can override
+        this detection when gateway and remote sandboxes share the same storage.
         """
+        override = self._config.get("thread_data_mounts")
+        if override is not None:
+            return override
         return isinstance(self._backend, LocalContainerBackend)
 
     # ── Factory methods ──────────────────────────────────────────────────
@@ -302,6 +307,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
             "idle_timeout": idle_timeout if idle_timeout is not None else DEFAULT_IDLE_TIMEOUT,
             "replicas": replicas if replicas is not None else DEFAULT_REPLICAS,
             "mounts": sandbox_config.mounts or [],
+            "thread_data_mounts": getattr(sandbox_config, "thread_data_mounts", None),
             "environment": self._resolve_env_vars(sandbox_config.environment or {}),
             "ownership": getattr(sandbox_config, "ownership", None),
             # A redis stream bridge means the deployment is multi-instance, which
@@ -959,6 +965,25 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
             logger.warning(f"Could not determine Lark integration state: {e}")
             return False
 
+    @staticmethod
+    def _lark_broker_active(user_id: str | None = None) -> bool:
+        """Whether this user's sandbox should use the lark-cli broker (Pattern B).
+
+        True only when the Lark pack is installed AND the remote provisioner
+        reports a configured broker image. When true, the provisioner keeps the
+        credentials in a sidecar and the sandbox gets only a shim, so the
+        Gateway-side credential-mount overlay must not run either.
+        """
+        try:
+            if not AioSandboxProvider._lark_integration_active(user_id):
+                return False
+            from deerflow.integrations.lark_cli import sandbox_lark_broker_active
+
+            return sandbox_lark_broker_active()
+        except Exception as e:  # pragma: no cover - defensive
+            logger.warning(f"Could not determine Lark broker state: {e}")
+            return False
+
     @staticmethod
     def _get_lark_cli_runtime_mounts(*, user_id: str | None = None) -> list[tuple[str, str, bool]]:
         """Mount the per-user lark-cli config/data dirs used by Settings auth.
@@ -1915,6 +1940,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
         effective_user_id = self._effective_acquire_user_id(user_id)
         extra_mounts = self._get_extra_mounts(thread_id, user_id=effective_user_id)
         provision_lark_cli_runtime = self._lark_integration_active(effective_user_id)
+        provision_lark_cli_broker = self._lark_broker_active(effective_user_id)
 
         # Enforce replicas: only warm-pool containers count toward eviction budget.
         # Active sandboxes are in use by live threads and must not be forcibly stopped.
@@ -1929,6 +1955,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
             extra_mounts=extra_mounts or None,
             user_id=effective_user_id,
             provision_lark_cli_runtime=provision_lark_cli_runtime,
+            provision_lark_cli_broker=provision_lark_cli_broker,
         )
 
         # Wait for sandbox to be ready
@@ -1943,6 +1970,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
         effective_user_id = self._effective_acquire_user_id(user_id)
         extra_mounts = await asyncio.to_thread(self._get_extra_mounts, thread_id, user_id=effective_user_id)
         provision_lark_cli_runtime = await asyncio.to_thread(self._lark_integration_active, effective_user_id)
+        provision_lark_cli_broker = await asyncio.to_thread(self._lark_broker_active, effective_user_id)
 
         # Enforce replicas: only warm-pool containers count toward eviction budget.
         # Active sandboxes are in use by live threads and must not be forcibly stopped.
@@ -1958,6 +1986,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
             extra_mounts=extra_mounts or None,
             user_id=effective_user_id,
             provision_lark_cli_runtime=provision_lark_cli_runtime,
+            provision_lark_cli_broker=provision_lark_cli_broker,
         )
 
         # Wait for sandbox to be ready without blocking the event loop.

backend/packages/harness/deerflow/community/aio_sandbox/backend.py

@@ -110,6 +110,7 @@ class SandboxBackend(ABC):
         *,
         user_id: str | None = None,
         provision_lark_cli_runtime: bool = False,
+        provision_lark_cli_broker: bool = False,
     ) -> SandboxInfo:
         """Create/provision a new sandbox.
 
@@ -122,6 +123,10 @@ class SandboxBackend(ABC):
             provision_lark_cli_runtime: Ask the backend to provision the sandbox
                 lark-cli runtime via its native mechanism (e.g. the provisioner's
                 init container + emptyDir). Backends that can't do this ignore it.
+            provision_lark_cli_broker: Ask the backend to provision a lark-cli
+                broker sidecar (Pattern B, issue #4338) so credentials stay out of
+                the sandbox. Supersedes ``provision_lark_cli_runtime`` when the
+                backend supports it; backends that can't do this ignore it.
 
         Returns:
             SandboxInfo with connection details.

backend/packages/harness/deerflow/community/aio_sandbox/local_backend.py

@@ -272,6 +272,7 @@ class LocalContainerBackend(SandboxBackend):
         *,
         user_id: str | None = None,
         provision_lark_cli_runtime: bool = False,
+        provision_lark_cli_broker: bool = False,
     ) -> SandboxInfo:
         """Start a new container and return its connection info.
 
@@ -283,6 +284,8 @@ class LocalContainerBackend(SandboxBackend):
                 interface compatibility with remote backends.
             provision_lark_cli_runtime: Ignored — the local backend provisions the
                 lark-cli runtime via the Gateway-download bind mount in extra_mounts.
+            provision_lark_cli_broker: Ignored — the local backend has no sandbox
+                boundary to protect, so it keeps the credential-mount overlay.
 
         Returns:
             SandboxInfo with container details.
@@ -290,7 +293,7 @@ class LocalContainerBackend(SandboxBackend):
         Raises:
             RuntimeError: If the container fails to start.
         """
-        del user_id, provision_lark_cli_runtime
+        del user_id, provision_lark_cli_runtime, provision_lark_cli_broker
         container_name = f"{self._container_prefix}-{sandbox_id}"
 
         # Retry loop: if Docker rejects the port (e.g. a stale container still

backend/packages/harness/deerflow/community/aio_sandbox/remote_backend.py

@@ -39,28 +39,41 @@ _PROVISIONER_EXTRA_MOUNT_PATHS = {
 }
 
 _LARK_CLI_RUNTIME_CONTAINER_PATH = "/mnt/integrations/lark-cli/runtime"
+_LARK_CLI_CONFIG_CONTAINER_PATH = "/mnt/integrations/lark-cli/config"
+_LARK_CLI_DATA_CONTAINER_PATH = "/mnt/integrations/lark-cli/data"
 
 
 def _provisioner_extra_mounts_payload(
     extra_mounts: list[tuple[str, str, bool]] | None,
     *,
     provision_lark_cli_runtime: bool = False,
+    provision_lark_cli_broker: bool = False,
 ) -> list[dict[str, object]]:
     """Return only extra mounts the provisioner knows how to recreate safely.
 
     When ``provision_lark_cli_runtime`` is set, the provisioner supplies the
     lark-cli runtime via an init container + emptyDir, so the runtime extra mount
     is dropped here to avoid a colliding hostPath/PVC mount at the same path. The
-    per-user config/data credential mounts are always forwarded.
+    per-user config/data credential mounts are still forwarded (they are mounted
+    into the sandbox in Pattern A).
+
+    When ``provision_lark_cli_broker`` is set (Pattern B, issue #4338), the
+    provisioner runs a broker sidecar that holds the credentials, so the
+    config/data mounts are **forwarded** (the provisioner wires them into the
+    sidecar, not the sandbox) while the runtime mount is dropped. Nothing changes
+    in this payload beyond keeping config/data available for the provisioner to
+    place; the runtime entry is dropped in both modes.
     """
     if not extra_mounts:
         return []
 
+    drop_runtime = provision_lark_cli_runtime or provision_lark_cli_broker
+
     payload: list[dict[str, object]] = []
     for host_path, container_path, read_only in extra_mounts:
         if container_path not in _PROVISIONER_EXTRA_MOUNT_PATHS:
             continue
-        if provision_lark_cli_runtime and container_path == _LARK_CLI_RUNTIME_CONTAINER_PATH:
+        if drop_runtime and container_path == _LARK_CLI_RUNTIME_CONTAINER_PATH:
             continue
         payload.append(
             {
@@ -115,6 +128,7 @@ class RemoteSandboxBackend(SandboxBackend):
         *,
         user_id: str | None = None,
         provision_lark_cli_runtime: bool = False,
+        provision_lark_cli_broker: bool = False,
     ) -> SandboxInfo:
         """Create a sandbox Pod + Service via the provisioner.
 
@@ -127,6 +141,7 @@ class RemoteSandboxBackend(SandboxBackend):
             extra_mounts,
             user_id=user_id,
             provision_lark_cli_runtime=provision_lark_cli_runtime,
+            provision_lark_cli_broker=provision_lark_cli_broker,
         )
 
     def destroy(self, info: SandboxInfo) -> None:
@@ -199,6 +214,7 @@ class RemoteSandboxBackend(SandboxBackend):
         *,
         user_id: str | None = None,
         provision_lark_cli_runtime: bool = False,
+        provision_lark_cli_broker: bool = False,
     ) -> SandboxInfo:
         """POST /api/sandboxes → create Pod + Service."""
         effective_user_id = user_id or get_effective_user_id()
@@ -209,10 +225,12 @@ class RemoteSandboxBackend(SandboxBackend):
             "user_id": effective_user_id,
             "include_legacy_skills": include_legacy_skills,
             "provision_lark_cli_runtime": provision_lark_cli_runtime,
+            "provision_lark_cli_broker": provision_lark_cli_broker,
         }
         provisioner_extra_mounts = _provisioner_extra_mounts_payload(
             extra_mounts,
             provision_lark_cli_runtime=provision_lark_cli_runtime,
+            provision_lark_cli_broker=provision_lark_cli_broker,
         )
         if provisioner_extra_mounts:
             payload["extra_mounts"] = provisioner_extra_mounts

backend/packages/harness/deerflow/config/database_config.py

@@ -31,13 +31,72 @@ need to do any environment variable processing.
 
 from __future__ import annotations
 
+import logging
 import os
-from typing import Literal
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field, model_validator
+
+logger = logging.getLogger(__name__)
+
+
+def resolve_checkpoint_graph_cache_max(database_config: Any, field_name: str, default: int) -> int:
+    """Read a graph-cache cap from a database config-ish object.
+
+    Tolerates stub configs (SimpleNamespace/MagicMock in tests, missing
+    section): anything that is not a plain int >= 1 falls back to
+    ``default``.
+    """
+    section = getattr(database_config, "checkpoint_graph_cache", None)
+    value = getattr(section, field_name, None)
+    if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+        return default
+    return value
 
-from pydantic import BaseModel, Field
 
 CheckpointChannelMode = Literal["full", "delta"]
 
+DEFAULT_CHECKPOINT_SNAPSHOT_FREQUENCY = 10
+
+
+class CheckpointDeltaConfig(BaseModel):
+    """Tuning knobs for ``checkpoint_channel_mode: delta``.
+
+    Ignored in ``full`` mode. Like the mode itself, these values are
+    restart-required and must match across every process sharing one
+    checkpoint database: the snapshot cadence is baked into each compiled
+    graph's channel table, not stored in the checkpoint, so a mismatched
+    process would apply a different cadence to the same threads.
+    """
+
+    snapshot_frequency: int = Field(
+        default=DEFAULT_CHECKPOINT_SNAPSHOT_FREQUENCY,
+        ge=1,
+        description=(
+            "DeltaChannel snapshot cadence: a full messages snapshot is stored "
+            "every N per-step writes (higher = smaller checkpoints, slower "
+            "materialization). Restart is required, and all processes sharing "
+            "one checkpoint database must use the same value."
+        ),
+    )
+
+
+class CheckpointGraphCacheConfig(BaseModel):
+    """Size cap for the process-local compiled checkpoint graph cache.
+
+    Unlike the mode and snapshot cadence, this is NOT restart-required:
+    a larger/smaller cap only changes when the cache evicts, never graph
+    semantics, so a hot-reloaded value takes effect on the next eviction
+    check. The cache is keyed by (assistant, mode, cadence, app_config) and
+    cleared wholesale at the cap.
+    """
+
+    accessor_graph_max: int = Field(
+        default=64,
+        ge=1,
+        description=("Max compiled thread-state accessor graphs cached by the gateway (keyed per assistant, channel mode, and snapshot cadence)."),
+    )
+
 
 class DatabaseConfig(BaseModel):
     backend: Literal["memory", "sqlite", "postgres"] = Field(
@@ -53,15 +112,13 @@ class DatabaseConfig(BaseModel):
             "processes sharing one checkpoint database must use the same value."
         ),
     )
-    checkpoint_delta_snapshot_frequency: int = Field(
-        default=1000,
-        ge=1,
-        description=(
-            "DeltaChannel snapshot frequency used in checkpoint 'delta' mode: "
-            "every N message-channel writes the saver stores a full snapshot "
-            "instead of a delta sentinel. Restart-required, and all processes "
-            "sharing one checkpoint database must use the same value."
-        ),
+    checkpoint_delta: CheckpointDeltaConfig = Field(
+        default_factory=CheckpointDeltaConfig,
+        description="Delta-mode checkpoint tuning. Only applies when checkpoint_channel_mode is 'delta'.",
+    )
+    checkpoint_graph_cache: CheckpointGraphCacheConfig = Field(
+        default_factory=CheckpointGraphCacheConfig,
+        description="Size caps for the compiled checkpoint graph caches. Hot-reloadable; not restart-required.",
     )
     sqlite_dir: str = Field(
         default=".deer-flow/data",
@@ -95,6 +152,46 @@ class DatabaseConfig(BaseModel):
         description="Timeout in seconds for app ORM PostgreSQL commands. Set to null to disable the command timeout.",
     )
 
+    # -- Legacy key migration (not user-configured) --
+
+    @model_validator(mode="before")
+    @classmethod
+    def _migrate_legacy_snapshot_frequency(cls, data: Any) -> Any:
+        """Carry the pre-rename top-level ``checkpoint_delta_snapshot_frequency``
+        key onto ``checkpoint_delta.snapshot_frequency``.
+
+        ``DatabaseConfig`` ignores unknown keys (pydantic ``extra="ignore"``),
+        so without this shim a config.yaml written against the old flat key
+        would silently fall back to the new default cadence instead of the
+        operator's chosen value. An explicitly set nested key always wins.
+        """
+        if not isinstance(data, dict) or "checkpoint_delta_snapshot_frequency" not in data:
+            return data
+        data = dict(data)
+        legacy_value = data.pop("checkpoint_delta_snapshot_frequency")
+        nested = data.get("checkpoint_delta")
+        if isinstance(nested, dict):
+            if "snapshot_frequency" in nested:
+                logger.warning(
+                    "Both database.checkpoint_delta_snapshot_frequency (deprecated) and database.checkpoint_delta.snapshot_frequency are set; the nested key wins.",
+                )
+                return data
+            data["checkpoint_delta"] = {**nested, "snapshot_frequency": legacy_value}
+        elif nested is None:
+            data["checkpoint_delta"] = {"snapshot_frequency": legacy_value}
+        else:
+            # Programmatically constructed CheckpointDeltaConfig instance: the
+            # explicit object wins over the legacy scalar.
+            logger.warning(
+                "Ignoring deprecated database.checkpoint_delta_snapshot_frequency because database.checkpoint_delta is already set.",
+            )
+            return data
+        logger.warning(
+            "database.checkpoint_delta_snapshot_frequency is deprecated; use database.checkpoint_delta.snapshot_frequency instead. Carried the legacy value (%r) forward.",
+            legacy_value,
+        )
+        return data
+
     # -- Derived helpers (not user-configured) --
 
     @property

backend/packages/harness/deerflow/config/sandbox_config.py

@@ -88,6 +88,8 @@ class SandboxConfig(BaseModel):
         port: Base port for sandbox containers (default: 8080)
         container_prefix: Prefix for container names (default: deer-flow-sandbox)
         mounts: List of volume mounts to share directories with the container
+        thread_data_mounts: Override whether thread data is already visible to
+            the sandbox through shared mounts. Omit to auto-detect from the backend.
 
     AioSandboxProvider and E2BSandboxProvider shared options:
         ownership: Cross-instance sandbox ownership store (memory | redis). Multi-instance
@@ -153,6 +155,10 @@ class SandboxConfig(BaseModel):
         default_factory=list,
         description="List of volume mounts to share directories between host and container",
     )
+    thread_data_mounts: bool | None = Field(
+        default=None,
+        description=("AioSandboxProvider: override whether /mnt/user-data is already visible through shared mounts. Omitted uses backend auto-detection; true skips explicit upload synchronization; false forces it."),
+    )
     environment: dict[str, str] = Field(
         default_factory=dict,
         description="Environment variables to inject into the sandbox container. Values starting with $ will be resolved from host environment variables.",

backend/packages/harness/deerflow/config/suggestions_config.py

@@ -1,7 +1,16 @@
 from pydantic import BaseModel, Field
 
+DEFAULT_MAX_SUGGESTIONS = 3
+MAX_SUGGESTIONS_LIMIT = 5
+
 
 class SuggestionsConfig(BaseModel):
     """Configuration for automatic follow-up suggestions."""
 
     enabled: bool = Field(default=True, description="Whether to enable follow-up question suggestions at the end of an AI response")
+    max_suggestions: int = Field(
+        default=DEFAULT_MAX_SUGGESTIONS,
+        ge=1,
+        le=MAX_SUGGESTIONS_LIMIT,
+        description="Maximum number of follow-up suggestions to generate.",
+    )

backend/packages/harness/deerflow/integrations/lark_broker.py

@@ -0,0 +1,457 @@
+"""Lark CLI sandbox credential broker (Pattern B, issue #4338).
+
+Pattern A (PR #3971) provisions the ``lark-cli`` *binary* into the sandbox but
+still mounts the per-user credential directories (``config`` with the long-lived
+``appSecret`` and ``data`` with OAuth tokens) into the sandbox container, where
+the agent's ``bash`` tool can read them.
+
+This module implements the broker half of Pattern B: a long-lived process that
+owns ``lark-cli`` + the credentials and exposes only the *command surface* over
+loopback. The sandbox gets a tiny ``lark-cli`` shim on ``PATH`` that forwards
+argv/stdin to the broker, so the raw credential files never exist in the sandbox
+filesystem.
+
+Everything here is Python-3-stdlib only so the same module can run inside the
+minimal broker sidecar image without extra dependencies.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import threading
+from dataclasses import dataclass
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# ── Loopback wire contract ────────────────────────────────────────────────
+
+# The sandbox and the broker sidecar share the Pod network namespace, so the
+# shim reaches the broker on loopback. The port is fixed and injected into the
+# sandbox as DEERFLOW_LARK_BROKER_URL.
+LARK_BROKER_DEFAULT_HOST = "127.0.0.1"
+LARK_BROKER_DEFAULT_PORT = 8788
+LARK_BROKER_URL_ENV = "DEERFLOW_LARK_BROKER_URL"
+LARK_BROKER_EXEC_PATH = "/v1/exec"
+LARK_BROKER_HEALTH_PATH = "/v1/health"
+
+# Guards. Bounded so a compromised sandbox cannot exhaust the broker.
+LARK_BROKER_MAX_REQUEST_BYTES = 1 * 1024 * 1024
+LARK_BROKER_MAX_OUTPUT_BYTES = 4 * 1024 * 1024
+LARK_BROKER_DEFAULT_TIMEOUT_SECONDS = 120
+LARK_BROKER_MAX_CONCURRENCY = 8
+# Per-connection socket timeout. ThreadingHTTPServer spawns a thread per
+# connection, so without this a sandbox could declare a large Content-Length and
+# never send the body, parking a thread forever. Bounds the read so a slow/stuck
+# client releases its thread. (Loopback-only, so this only guards the sandbox
+# against tying up its own broker.)
+LARK_BROKER_SOCKET_TIMEOUT_SECONDS = 30
+
+# Optional env knob (comma-separated) for the subcommand denylist below.
+LARK_BROKER_DENY_SUBCOMMANDS_ENV = "DEERFLOW_LARK_BROKER_DENY_SUBCOMMANDS"
+
+# The runtime layout the sandbox sees is two files in ``bin/``:
+#
+#   bin/lark-cli          the POSIX-sh *launcher* (below) — the executable on PATH
+#   bin/lark-cli-shim.py  the Python *shim* body (below) it execs
+#
+# Pattern A's launcher is pure ``#!/bin/sh`` (it just execs the arch-dispatched
+# binary) and therefore has no runtime deps. The broker shim has to speak HTTP,
+# so it is Python — but shipping it as a bare ``#!/usr/bin/env python3`` script
+# would make every ``lark-cli`` call ENOEXEC/exit-127 on a sandbox image without
+# ``python3`` on PATH, and because broker mode is opt-in that could slip past CI
+# and only surface for an operator. The ``/bin/sh`` launcher resolves a Python 3
+# interpreter itself (using only shell built-ins, so it still works when PATH is
+# empty and the interpreter is pinned) and, if none exists, fails *loudly* with
+# an actionable message instead of an opaque ENOEXEC. ``DEERFLOW_LARK_BROKER_PYTHON``
+# pins a specific interpreter for images that ship Python under a non-standard
+# name.
+#
+# The launcher's path to the shim body is baked in at install time rather than
+# derived from ``$0``: when the sandbox runs ``lark-cli`` off PATH, ``$0`` is the
+# bare command name with no directory, so a sibling lookup would fail. The
+# install dir is a stable, shared mount, so the absolute path is valid in both
+# the init container that writes it and the sandbox that reads it.
+#
+# Both scripts are kept here as the single source of truth and mirrored by the
+# broker image build via ``install_shim``, exactly like ``LARK_CLI_SANDBOX_LAUNCHER_SCRIPT``
+# for Pattern A, so the image copies can never drift from the Gateway's.
+LARK_BROKER_PYTHON_ENV = "DEERFLOW_LARK_BROKER_PYTHON"
+LARK_CLI_BROKER_SHIM_FILENAME = "lark-cli-shim.py"
+_LARK_CLI_BROKER_SHIM_PATH_PLACEHOLDER = "@@LARK_CLI_BROKER_SHIM_PATH@@"
+
+LARK_CLI_BROKER_LAUNCHER_TEMPLATE = (
+    "#!/bin/sh\n"
+    "# DeerFlow lark-cli broker launcher (Pattern B). Resolves a Python 3\n"
+    "# interpreter and execs the forwarding shim. Fails loudly (not with an opaque\n"
+    "# ENOEXEC) when the sandbox image ships no python3. Uses only shell built-ins\n"
+    "# so it still works when PATH is empty and DEERFLOW_LARK_BROKER_PYTHON pins\n"
+    "# the interpreter.\n"
+    "set -eu\n"
+    'shim="' + _LARK_CLI_BROKER_SHIM_PATH_PLACEHOLDER + '"\n'
+    'if [ -n "${DEERFLOW_LARK_BROKER_PYTHON:-}" ]; then\n'
+    '  exec "$DEERFLOW_LARK_BROKER_PYTHON" "$shim" "$@"\n'
+    "fi\n"
+    "for _py in python3 python; do\n"
+    '  if command -v "$_py" >/dev/null 2>&1; then\n'
+    '    exec "$_py" "$shim" "$@"\n'
+    "  fi\n"
+    "done\n"
+    'echo "lark-cli: broker mode needs a Python 3 interpreter but none was found;" >&2\n'
+    'echo "          set DEERFLOW_LARK_BROKER_PYTHON to a python3 path in the sandbox image." >&2\n'
+    "exit 127\n"
+)
+
+
+def render_launcher_script(shim_path: str) -> str:
+    """Render the ``/bin/sh`` launcher with the shim body's absolute path baked in."""
+    return LARK_CLI_BROKER_LAUNCHER_TEMPLATE.replace(_LARK_CLI_BROKER_SHIM_PATH_PLACEHOLDER, shim_path)
+
+
+# The shim reads argv/stdin, POSTs to the broker, and replays the broker's
+# stdout/stderr/exit code. On any transport failure it fails loudly and non-zero
+# so a broker outage never looks like a successful lark-cli run. It is invoked as
+# ``<python> lark-cli-shim.py <args...>`` by the launcher above (so it does not
+# rely on its own shebang being resolvable), and stdin/argv pass straight through.
+LARK_CLI_BROKER_SHIM_SCRIPT = r'''#!/usr/bin/env python3
+"""DeerFlow lark-cli broker shim (Pattern B). Forwards argv/stdin to the broker.
+
+Note: the broker runs lark-cli in the *sidecar's* working directory and cannot
+see the sandbox filesystem, so cwd is intentionally not forwarded. Subcommands
+that read/write files relative to the sandbox cwd are unsupported in broker mode.
+"""
+import base64
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+
+BROKER_URL = os.environ.get("DEERFLOW_LARK_BROKER_URL", "http://127.0.0.1:8788")
+
+
+def _fail(message, code=127):
+    sys.stderr.write("lark-cli: " + message + "\n")
+    sys.exit(code)
+
+
+def main():
+    try:
+        stdin_bytes = b"" if sys.stdin is None or sys.stdin.isatty() else sys.stdin.buffer.read()
+    except Exception:
+        stdin_bytes = b""
+    payload = json.dumps(
+        {
+            "args": sys.argv[1:],
+            "stdin_b64": base64.b64encode(stdin_bytes).decode("ascii"),
+        }
+    ).encode("utf-8")
+    req = urllib.request.Request(
+        BROKER_URL.rstrip("/") + "/v1/exec",
+        data=payload,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=600) as resp:
+            body = json.loads(resp.read().decode("utf-8"))
+    except urllib.error.HTTPError as exc:
+        detail = ""
+        try:
+            detail = json.loads(exc.read().decode("utf-8")).get("error", "")
+        except Exception:
+            detail = ""
+        _fail("broker rejected request (HTTP %d%s)" % (exc.code, ": " + detail if detail else ""))
+    except (urllib.error.URLError, OSError) as exc:
+        _fail("broker unreachable at %s (%s)" % (BROKER_URL, exc))
+    except Exception as exc:  # noqa: BLE001
+        _fail("broker call failed (%s)" % exc)
+    sys.stdout.buffer.write(base64.b64decode(body.get("stdout_b64", "")))
+    sys.stdout.buffer.flush()
+    sys.stderr.buffer.write(base64.b64decode(body.get("stderr_b64", "")))
+    sys.stderr.buffer.flush()
+    sys.exit(int(body.get("exit_code", 1)))
+
+
+if __name__ == "__main__":
+    main()
+'''
+
+
+# ── Broker server ─────────────────────────────────────────────────────────
+
+
+@dataclass(frozen=True)
+class BrokerConfig:
+    """Runtime configuration for the broker sidecar."""
+
+    lark_cli_path: str
+    config_dir: str
+    data_dir: str
+    host: str = LARK_BROKER_DEFAULT_HOST
+    port: int = LARK_BROKER_DEFAULT_PORT
+    timeout_seconds: int = LARK_BROKER_DEFAULT_TIMEOUT_SECONDS
+    # Opt-in denylist of ``lark-cli`` subcommand paths the broker refuses to run
+    # (issue #4338 hardening). Each entry is a space-joined command prefix, e.g.
+    # "config show" or "auth token", matched against the leading non-flag tokens
+    # of the request. Narrows the command surface a prompt-injected agent can
+    # reach — the broker already removes the credential *files*, but the full
+    # command surface stays reachable unless a secret-dumping subcommand is denied
+    # here. Empty by default (no behavior change).
+    deny_subcommands: tuple[tuple[str, ...], ...] = ()
+
+    def credential_env(self) -> dict[str, str]:
+        """Env the broker injects into every lark-cli invocation.
+
+        The client never supplies these — the broker owns the credential paths,
+        so a sandbox process cannot point lark-cli at a different profile.
+        """
+        return {
+            "LARKSUITE_CLI_CONFIG_DIR": self.config_dir,
+            "LARKSUITE_CLI_DATA_DIR": self.data_dir,
+            "LARKSUITE_CLI_NO_UPDATE_NOTIFIER": "1",
+            "LARKSUITE_CLI_NO_SKILLS_NOTIFIER": "1",
+        }
+
+
+def parse_deny_subcommands(raw: str | None) -> tuple[tuple[str, ...], ...]:
+    """Parse the comma-separated denylist env into command-prefix tuples.
+
+    ``"config show, auth token"`` → ``(("config", "show"), ("auth", "token"))``.
+    Blank/whitespace-only entries are dropped.
+    """
+    if not raw:
+        return ()
+    prefixes: list[tuple[str, ...]] = []
+    for entry in raw.split(","):
+        tokens = tuple(entry.split())
+        if tokens:
+            prefixes.append(tokens)
+    return tuple(prefixes)
+
+
+def _denied_subcommand(deny: tuple[tuple[str, ...], ...], args: list[str]) -> tuple[str, ...] | None:
+    """Return the matched denylist prefix if ``args`` is a denied subcommand.
+
+    Matches against the leading non-flag tokens (options and their values are
+    skipped) so ``config --json show`` is still caught by a ``config show`` rule.
+    """
+    if not deny:
+        return None
+    positional = [token for token in args if not token.startswith("-")]
+    for prefix in deny:
+        if positional[: len(prefix)] == list(prefix):
+            return prefix
+    return None
+
+
+@dataclass(frozen=True)
+class ExecResult:
+    exit_code: int
+    stdout: bytes
+    stderr: bytes
+    truncated: bool
+
+
+def run_lark_cli(config: BrokerConfig, args: list[str], stdin: bytes) -> ExecResult:
+    """Run a single ``lark-cli`` invocation with broker-owned credentials.
+
+    ``args`` is passed as an argv list with ``shell=False`` so a sandbox-supplied
+    argument can never be shell-interpreted into a second command. A configured
+    ``deny_subcommands`` prefix is refused before the binary is ever spawned.
+    """
+    denied = _denied_subcommand(config.deny_subcommands, args)
+    if denied is not None:
+        message = f"lark-cli: subcommand '{' '.join(denied)}' is disabled in broker mode\n"
+        return ExecResult(126, b"", message.encode("utf-8"), False)
+    env = {**os.environ, **config.credential_env()}
+    try:
+        completed = subprocess.run(  # noqa: S603 - argv list, shell=False, fixed binary
+            [config.lark_cli_path, *args],
+            input=stdin,
+            capture_output=True,
+            timeout=config.timeout_seconds,
+            check=False,
+            env=env,
+        )
+    except subprocess.TimeoutExpired:
+        return ExecResult(124, b"", b"lark-cli: broker timed out\n", False)
+    except FileNotFoundError:
+        return ExecResult(127, b"", b"lark-cli: binary not found in broker\n", False)
+
+    stdout, out_trunc = _cap(completed.stdout or b"")
+    stderr, err_trunc = _cap(completed.stderr or b"")
+    return ExecResult(completed.returncode, stdout, stderr, out_trunc or err_trunc)
+
+
+def _cap(data: bytes) -> tuple[bytes, bool]:
+    if len(data) <= LARK_BROKER_MAX_OUTPUT_BYTES:
+        return data, False
+    return data[:LARK_BROKER_MAX_OUTPUT_BYTES], True
+
+
+def make_handler(config: BrokerConfig) -> type[BaseHTTPRequestHandler]:
+    """Build a request handler bound to ``config``.
+
+    A bounded semaphore caps concurrency so a flood of sandbox calls cannot spawn
+    unbounded ``lark-cli`` subprocesses.
+    """
+    semaphore = threading.BoundedSemaphore(LARK_BROKER_MAX_CONCURRENCY)
+
+    class Handler(BaseHTTPRequestHandler):
+        # Bound per-connection reads so a client that declares a large
+        # Content-Length and never sends the body cannot park its thread forever
+        # (ThreadingHTTPServer is one-thread-per-connection). stdlib reads this
+        # to arm socket timeouts.
+        timeout = LARK_BROKER_SOCKET_TIMEOUT_SECONDS
+
+        # Quiet: default BaseHTTPRequestHandler logs to stderr per request.
+        def log_message(self, *_args: Any) -> None:  # noqa: D401
+            return
+
+        def _send_json(self, status: int, body: dict[str, Any]) -> None:
+            payload = json.dumps(body).encode("utf-8")
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(payload)))
+            self.end_headers()
+            self.wfile.write(payload)
+
+        def do_GET(self) -> None:  # noqa: N802 - stdlib API
+            if self.path.rstrip("/") == LARK_BROKER_HEALTH_PATH:
+                self._send_json(200, {"ok": True})
+                return
+            self._send_json(404, {"error": "not found"})
+
+        def do_POST(self) -> None:  # noqa: N802 - stdlib API
+            if self.path.rstrip("/") != LARK_BROKER_EXEC_PATH:
+                self._send_json(404, {"error": "not found"})
+                return
+            try:
+                length = int(self.headers.get("Content-Length", "0"))
+            except ValueError:
+                self._send_json(400, {"error": "bad content-length"})
+                return
+            if length <= 0 or length > LARK_BROKER_MAX_REQUEST_BYTES:
+                self._send_json(413, {"error": "request too large"})
+                return
+            try:
+                request = json.loads(self.rfile.read(length).decode("utf-8"))
+                args = request["args"]
+                if not isinstance(args, list) or not all(isinstance(a, str) for a in args):
+                    raise ValueError("args must be a list of strings")
+                stdin = base64.b64decode(request.get("stdin_b64", "") or "")
+            except Exception:  # noqa: BLE001 - untrusted client input
+                self._send_json(400, {"error": "invalid request"})
+                return
+
+            if not semaphore.acquire(blocking=False):
+                self._send_json(503, {"error": "broker busy"})
+                return
+            try:
+                result = run_lark_cli(config, args, stdin)
+            except Exception:  # noqa: BLE001 - keep the wire contract uniform
+                # run_lark_cli already maps the expected failures (timeout,
+                # missing binary) to ExecResults; anything else (OSError,
+                # PermissionError, …) would otherwise close the connection with
+                # no body and surface as an opaque transport error in the shim.
+                # Return a structured 500 so the shim reports a meaningful error.
+                logger.exception("lark-cli exec failed unexpectedly")
+                self._send_json(500, {"error": "broker exec failed"})
+                return
+            finally:
+                semaphore.release()
+
+            self._send_json(
+                200,
+                {
+                    "exit_code": result.exit_code,
+                    "stdout_b64": base64.b64encode(result.stdout).decode("ascii"),
+                    "stderr_b64": base64.b64encode(result.stderr).decode("ascii"),
+                    "truncated": result.truncated,
+                },
+            )
+
+    return Handler
+
+
+def serve(config: BrokerConfig) -> ThreadingHTTPServer:
+    """Start the broker HTTP server bound to loopback and return it."""
+    if not shutil.which(config.lark_cli_path) and not os.path.isfile(config.lark_cli_path):
+        logger.warning("lark-cli not found at %s; broker will report 127 for exec", config.lark_cli_path)
+    server = ThreadingHTTPServer((config.host, config.port), make_handler(config))
+    logger.info("lark-cli broker listening on %s:%d", config.host, config.port)
+    return server
+
+
+def install_shim(dest_dir: str, *, version: str | None = None) -> str:
+    """Write the launcher + shim + runtime marker into the sandbox runtime dir.
+
+    Called by the broker image's ``install-shim`` init-container mode. Produces
+    the same ``bin/lark-cli`` + ``.deerflow-lark-cli-runtime.json`` layout Pattern
+    A stages, but marked ``kind="shim"`` so the runtime validator knows the
+    ``linux-*`` binaries are intentionally absent (the sidecar holds the real
+    binary).
+
+    Two files are written into ``bin/``: the executable-on-PATH ``lark-cli`` is a
+    ``/bin/sh`` *launcher* that resolves a Python 3 interpreter and execs the
+    ``lark-cli-shim.py`` *body* next to it. Splitting them keeps broker mode from
+    silently failing with ENOEXEC on a sandbox image that has no ``python3`` (the
+    launcher fails loudly with an actionable message instead). Both come from the
+    in-process constants so the image copy can never drift from the Gateway's.
+    """
+    dest = os.path.abspath(dest_dir)
+    bin_dir = os.path.join(dest, "bin")
+    os.makedirs(bin_dir, exist_ok=True)
+    shim_body = os.path.join(bin_dir, LARK_CLI_BROKER_SHIM_FILENAME)
+    with open(shim_body, "w", encoding="utf-8") as handle:
+        handle.write(LARK_CLI_BROKER_SHIM_SCRIPT)
+    os.chmod(shim_body, 0o755)
+    launcher = os.path.join(bin_dir, "lark-cli")
+    with open(launcher, "w", encoding="utf-8") as handle:
+        handle.write(render_launcher_script(shim_body))
+    os.chmod(launcher, 0o755)
+    marker = os.path.join(dest, ".deerflow-lark-cli-runtime.json")
+    with open(marker, "w", encoding="utf-8") as handle:
+        json.dump({"version": version or "unknown", "kind": "shim"}, handle)
+    return launcher
+
+
+def _config_from_env() -> BrokerConfig:
+    return BrokerConfig(
+        lark_cli_path=os.environ.get("DEERFLOW_LARK_BROKER_CLI", "lark-cli"),
+        config_dir=os.environ.get("LARKSUITE_CLI_CONFIG_DIR", "/var/lark/config"),
+        data_dir=os.environ.get("LARKSUITE_CLI_DATA_DIR", "/var/lark/data"),
+        host=os.environ.get("DEERFLOW_LARK_BROKER_HOST", LARK_BROKER_DEFAULT_HOST),
+        port=int(os.environ.get("DEERFLOW_LARK_BROKER_PORT", str(LARK_BROKER_DEFAULT_PORT))),
+        timeout_seconds=int(os.environ.get("DEERFLOW_LARK_BROKER_TIMEOUT", str(LARK_BROKER_DEFAULT_TIMEOUT_SECONDS))),
+        deny_subcommands=parse_deny_subcommands(os.environ.get(LARK_BROKER_DENY_SUBCOMMANDS_ENV)),
+    )
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO)
+    argv = sys.argv[1:]
+    if argv and argv[0] == "install-shim":
+        dest = argv[1] if len(argv) > 1 else os.environ.get("LARK_CLI_RUNTIME_DEST", "/mnt/integrations/lark-cli/runtime")
+        launcher = install_shim(dest, version=os.environ.get("LARK_CLI_VERSION"))
+        logger.info("Installed lark-cli broker shim at %s", launcher)
+        return
+    server = serve(_config_from_env())
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        server.shutdown()
+
+
+if __name__ == "__main__":
+    main()

backend/packages/harness/deerflow/integrations/lark_cli.py

@@ -73,6 +73,7 @@ except ImportError:  # pragma: no cover - Windows fallback
 
 from deerflow.config.app_config import AppConfig
 from deerflow.config.paths import Paths, get_paths
+from deerflow.integrations.lark_broker import LARK_BROKER_URL_ENV
 from deerflow.skills.installer import is_executable_binary_prefix, is_symlink_member, is_unsafe_zip_member
 from deerflow.skills.parser import parse_skill_file
 from deerflow.skills.permissions import make_skill_tree_sandbox_readable
@@ -108,6 +109,11 @@ LARK_CLI_SANDBOX_RUNTIME_DIR = "/mnt/integrations/lark-cli/runtime"
 LARK_CLI_LINUX_ARCHES = ("amd64", "arm64")
 LARK_CLI_RUNTIME_MANIFEST_FILE = ".deerflow-lark-cli-runtime.json"
 
+# Pattern B (issue #4338): loopback URL the sandbox shim uses to reach the broker
+# sidecar. LARK_BROKER_URL_ENV is imported from the broker module so the shim,
+# server, and Gateway overlay share one source of truth.
+LARK_BROKER_SANDBOX_URL = "http://127.0.0.1:8788"
+
 # Arch-dispatch launcher for the sandbox runtime layout. Shared by the Gateway
 # writer (`_write_lark_cli_sandbox_launcher`) and the `docker/lark-cli-init`
 # init image so the two producers of `bin/lark-cli` never drift.
@@ -522,13 +528,23 @@ def _lark_cli_managed_path() -> str | None:
     return None
 
 
-def lark_cli_env_overlay(user_id: str, *, sandbox_paths: bool = False) -> dict[str, str]:
+def lark_cli_env_overlay(user_id: str, *, sandbox_paths: bool = False, broker: bool = False) -> dict[str, str]:
     """Environment overlay for lark-cli using DeerFlow-managed credentials.
 
     The directories are per-user so a local trusted-mode login cannot bleed
-    across accounts. Auth Proxy support can later replace these directories for
-    sandbox execution without changing the status API contract.
+    across accounts.
+
+    When ``broker`` is set (Pattern B, issue #4338), the sandbox talks to a
+    broker sidecar that owns the credentials, so the overlay carries only the
+    broker URL and the runtime PATH — never ``LARKSUITE_CLI_CONFIG_DIR`` /
+    ``DATA_DIR``. This keeps the plaintext app secret / OAuth tokens out of the
+    sandbox filesystem entirely. ``broker`` implies ``sandbox_paths``.
     """
+    if broker:
+        return {
+            "PATH": f"{LARK_CLI_SANDBOX_RUNTIME_DIR}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            LARK_BROKER_URL_ENV: LARK_BROKER_SANDBOX_URL,
+        }
     if sandbox_paths:
         config_dir: Path | str = LARK_CLI_SANDBOX_CONFIG_DIR
         data_dir: Path | str = LARK_CLI_SANDBOX_DATA_DIR
@@ -660,7 +676,12 @@ def _resolve_sandbox_runtime_readiness(
     - ``gateway-download``: local AIO — the Gateway stages a ``sandbox-cli`` dir
       and bind-mounts it; ready when that dir validates.
     - ``init-container``: remote provisioner — a lark-cli init image provisions
-      the runtime; ready when the provisioner reports the init image is configured.
+      the runtime binary + plaintext credential mounts (Pattern A); ready when
+      the provisioner reports the init image is configured.
+    - ``broker``: remote provisioner — a lark-cli broker sidecar holds the
+      credentials and the sandbox gets only a shim (Pattern B, issue #4338);
+      ready when the provisioner reports the broker image is configured. Broker
+      supersedes ``init-container`` when both are available.
 
     ``probe`` gates the (best-effort, short-timeout) provisioner capability call.
     """
@@ -670,12 +691,16 @@ def _resolve_sandbox_runtime_readiness(
     if _uses_remote_provisioner(config):
         if not probe:
             return "init-container", False, None
-        init_image = _probe_provisioner_lark_cli_init_image(config)
-        if init_image is None:
-            return "init-container", False, "Could not reach the provisioner to confirm the lark-cli init image."
-        if not init_image:
-            return "init-container", False, "The provisioner has no lark-cli init image configured (LARK_CLI_INIT_IMAGE)."
-        return "init-container", True, None
+        caps = _probe_provisioner_capabilities(config)
+        if caps is None:
+            return "init-container", False, "Could not reach the provisioner to confirm the lark-cli runtime image."
+        # Pattern B (broker) supersedes Pattern A (init-container binary) when
+        # the provisioner has a broker image configured.
+        if caps["lark_cli_broker_image"]:
+            return "broker", True, None
+        if caps["lark_cli_init_image"]:
+            return "init-container", True, None
+        return "init-container", False, "The provisioner has no lark-cli runtime image configured (LARK_CLI_INIT_IMAGE / LARK_CLI_BROKER_IMAGE)."
 
     # Local AIO: Gateway-download runtime dir.
     runtime_dir = lark_cli_managed_sandbox_dir()
@@ -686,6 +711,61 @@ def _resolve_sandbox_runtime_readiness(
     return "gateway-download", True, None
 
 
+LARK_BROKER_MODE_TTL_SECONDS = 60
+# Negative results (broker not active) are cached longer than positive ones: a
+# non-broker remote-provisioner deployment stays non-broker for the life of the
+# process far more often than it flips on, so this keeps the hot bash path from
+# re-probing every minute. A positive result still refreshes on the shorter TTL.
+LARK_BROKER_MODE_NEGATIVE_TTL_SECONDS = 300
+# Tight probe budget on the per-bash-call hot path: unlike the Settings status
+# probe (5s, user is waiting on a page), this runs inline before a sandbox
+# lark-cli command, so a slow/unreachable provisioner must not add seconds of
+# latency to every first-call-per-TTL for non-broker deployments.
+LARK_BROKER_MODE_PROBE_TIMEOUT_SECONDS = 1.5
+# Guards the cache attribute on sandbox_lark_broker_active against concurrent
+# bash invocations so the correctness story doesn't rely on idempotent races.
+_LARK_BROKER_MODE_CACHE_LOCK = threading.Lock()
+
+
+def sandbox_lark_broker_active(config: AppConfig | None = None) -> bool:
+    """Whether sandbox ``lark-cli`` runs in broker mode (Pattern B).
+
+    Cached with a short TTL because it is consulted on every ``lark-cli`` bash
+    call and reads the provisioner capability over HTTP. Broker mode requires a
+    remote provisioner that reports a configured broker image; any other config
+    (local AIO, init-container binary mode, unreachable provisioner) is False, so
+    the caller falls back to the credential-mount overlay.
+
+    The probe uses a tight timeout and negatives are cached longer than positives
+    so a non-broker remote-provisioner deployment does not pay a latency penalty
+    on the bash hot path.
+    """
+    if config is None:
+        try:
+            from deerflow.config.app_config import get_app_config
+
+            config = get_app_config()
+        except Exception:  # noqa: BLE001 - degrade to non-broker overlay
+            return False
+
+    now = time.monotonic()
+    with _LARK_BROKER_MODE_CACHE_LOCK:
+        cached = getattr(sandbox_lark_broker_active, "_cache", None)
+        if cached is not None:
+            ts, value = cached
+            ttl = LARK_BROKER_MODE_TTL_SECONDS if value else LARK_BROKER_MODE_NEGATIVE_TTL_SECONDS
+            if now - ts < ttl:
+                return value
+
+    active = False
+    if _uses_aio_sandbox(config) and _uses_remote_provisioner(config):
+        caps = _probe_provisioner_capabilities(config, timeout=LARK_BROKER_MODE_PROBE_TIMEOUT_SECONDS)
+        active = bool(caps and caps["lark_cli_broker_image"])
+    with _LARK_BROKER_MODE_CACHE_LOCK:
+        sandbox_lark_broker_active._cache = (now, active)  # type: ignore[attr-defined]
+    return active
+
+
 def get_lark_integration_status(
     user_id: str,
     config: AppConfig,
@@ -862,12 +942,14 @@ def _uses_remote_provisioner(config: AppConfig) -> bool:
     return bool(_sandbox_config_value(config, "provisioner_url"))
 
 
-def _probe_provisioner_lark_cli_init_image(config: AppConfig) -> bool | None:
-    """Best-effort read of the provisioner's lark-cli init-image capability.
+def _probe_provisioner_capabilities(config: AppConfig, *, timeout: float = 5.0) -> dict[str, bool] | None:
+    """Best-effort read of the provisioner's lark-cli capabilities.
 
-    Returns True/False when the provisioner answers, or None when it can't be
-    reached. Used only to surface a sandbox-runtime readiness signal; failures
-    degrade to "not ready" rather than raising.
+    Returns the capability dict when the provisioner answers, or None when it
+    can't be reached. Used both for the status readiness signal and to select
+    broker vs. binary mode on the bash hot path; failures degrade to "not
+    ready"/"not broker" rather than raising. ``timeout`` is caller-tunable so the
+    per-bash-call probe can use a tighter budget than the Settings status probe.
     """
     base = _sandbox_config_value(config, "provisioner_url")
     if not base:
@@ -877,9 +959,14 @@ def _probe_provisioner_lark_cli_init_image(config: AppConfig) -> bool | None:
     url = f"{base.rstrip('/')}/api/capabilities"
     try:
         request = urllib.request.Request(url, headers={"User-Agent": "deer-flow", **headers})
-        with urllib.request.urlopen(request, timeout=5) as response:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
             payload = json.loads(response.read().decode("utf-8"))
-        return bool(payload.get("lark_cli_init_image"))
+        if not isinstance(payload, dict):
+            return None
+        return {
+            "lark_cli_init_image": bool(payload.get("lark_cli_init_image")),
+            "lark_cli_broker_image": bool(payload.get("lark_cli_broker_image")),
+        }
     except Exception:
         return None
 

backend/packages/harness/deerflow/models/vllm_provider.py

@@ -15,6 +15,9 @@ This provider preserves ``reasoning`` on:
 from __future__ import annotations
 
 import json
+import threading
+import time
+from collections import OrderedDict
 from collections.abc import Mapping
 from typing import Any, cast
 
@@ -30,10 +33,15 @@ from langchain_core.messages import (
     SystemMessageChunk,
     ToolMessageChunk,
 )
+from langchain_core.messages.ai import UsageMetadata, subtract_usage
 from langchain_core.messages.tool import tool_call_chunk
 from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
 from langchain_openai import ChatOpenAI
 from langchain_openai.chat_models.base import _create_usage_metadata
+from pydantic import Field, PrivateAttr
+
+_CUMULATIVE_USAGE_TRACKER_CAPACITY = 1024
+_CUMULATIVE_USAGE_TRACKER_IDLE_SECONDS = 60 * 60
 
 
 def _normalize_vllm_chat_template_kwargs(payload: dict[str, Any]) -> None:
@@ -156,15 +164,59 @@ def _restore_reasoning_field(payload_msg: dict[str, Any], orig_msg: AIMessage) -
         payload_msg["reasoning"] = reasoning
 
 
+def _get_completion_id(chunk: Mapping[str, Any]) -> str | None:
+    """Return a stable completion id when the provider supplied one."""
+    candidates = [chunk.get("id")]
+    nested_chunk = chunk.get("chunk")
+    if isinstance(nested_chunk, Mapping):
+        candidates.append(nested_chunk.get("id"))
+
+    for candidate in candidates:
+        if isinstance(candidate, str) and candidate.strip():
+            return candidate
+    return None
+
+
 class VllmChatModel(ChatOpenAI):
     """ChatOpenAI variant that preserves vLLM reasoning fields across turns."""
 
     model_config = {"arbitrary_types_allowed": True}
 
+    cumulative_stream_usage: bool = Field(
+        default=False,
+        description=("Treat streaming usage snapshots from vLLM as cumulative per completion and convert them to per-chunk deltas."),
+    )
+    _cumulative_usage_by_completion: OrderedDict[str, tuple[UsageMetadata, float]] = PrivateAttr(default_factory=OrderedDict)
+    _cumulative_usage_lock: Any = PrivateAttr(default_factory=threading.Lock)
+
     @property
     def _llm_type(self) -> str:
         return "vllm-openai-compatible"
 
+    def _usage_delta(self, completion_id: str, usage: UsageMetadata, *, terminal: bool) -> UsageMetadata:
+        """Convert a completion's cumulative usage snapshot into a delta."""
+        with self._cumulative_usage_lock:
+            previous_snapshot = self._cumulative_usage_by_completion.get(completion_id)
+            previous = previous_snapshot[0] if previous_snapshot is not None else None
+            delta = subtract_usage(usage, previous)
+            if terminal:
+                self._cumulative_usage_by_completion.pop(completion_id, None)
+            else:
+                now = time.monotonic()
+                self._cumulative_usage_by_completion[completion_id] = (usage, now)
+                self._cumulative_usage_by_completion.move_to_end(completion_id)
+                while len(self._cumulative_usage_by_completion) > _CUMULATIVE_USAGE_TRACKER_CAPACITY:
+                    oldest_completion_id, (_, updated_at) = next(iter(self._cumulative_usage_by_completion.items()))
+                    if now - updated_at < _CUMULATIVE_USAGE_TRACKER_IDLE_SECONDS:
+                        break
+                    self._cumulative_usage_by_completion.pop(oldest_completion_id, None)
+            return delta
+
+    def _clear_usage_snapshot(self, completion_id: str) -> None:
+        """Forget a completed stream even when its terminal frame has no usage."""
+        with self._cumulative_usage_lock:
+            self._cumulative_usage_by_completion.pop(completion_id, None)
+
     def _get_request_payload(
         self,
         input_: LanguageModelInput,
@@ -224,9 +276,15 @@ class VllmChatModel(ChatOpenAI):
         token_usage = chunk.get("usage")
         choices = chunk.get("choices", []) or chunk.get("chunk", {}).get("choices", [])
         usage_metadata = _create_usage_metadata(token_usage, chunk.get("service_tier")) if token_usage else None
+        completion_id = _get_completion_id(chunk) if self.cumulative_stream_usage else None
 
         if len(choices) == 0:
             generation_chunk = ChatGenerationChunk(message=default_chunk_class(content="", usage_metadata=usage_metadata), generation_info=base_generation_info)
+            if completion_id is not None:
+                if usage_metadata is not None and isinstance(generation_chunk.message, AIMessageChunk):
+                    generation_chunk.message.usage_metadata = self._usage_delta(completion_id, usage_metadata, terminal=True)
+                else:
+                    self._clear_usage_snapshot(completion_id)
             if self.output_version == "v1":
                 generation_chunk.message.content = []
                 generation_chunk.message.response_metadata["output_version"] = "v1"
@@ -252,6 +310,8 @@ class VllmChatModel(ChatOpenAI):
             generation_info["logprobs"] = logprobs
 
         if usage_metadata and isinstance(message_chunk, AIMessageChunk):
+            if completion_id is not None:
+                usage_metadata = self._usage_delta(completion_id, usage_metadata, terminal=False)
             message_chunk.usage_metadata = usage_metadata
 
         message_chunk.response_metadata["model_provider"] = "openai"

backend/packages/harness/deerflow/persistence/migrations/versions/0010_run_cancel_request.py

@@ -0,0 +1,37 @@
+"""durable cross-worker run cancellation requests.
+
+Revision ID: 0010_run_cancel_request
+Revises: 0009_webhook_dedupe
+Create Date: 2026-07-27
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+revision: str = "0010_run_cancel_request"
+down_revision: str | Sequence[str] | None = "0009_webhook_dedupe"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    from deerflow.persistence.migrations._helpers import safe_add_column
+
+    safe_add_column(
+        "runs",
+        sa.Column("cancel_action", sa.String(length=20), nullable=True),
+    )
+    safe_add_column(
+        "runs",
+        sa.Column("cancel_requested_at", sa.DateTime(timezone=True), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    from deerflow.persistence.migrations._helpers import safe_drop_column
+
+    safe_drop_column("runs", "cancel_requested_at")
+    safe_drop_column("runs", "cancel_action")

backend/packages/harness/deerflow/persistence/migrations/versions/0011_feedback_tags.py

@@ -1,7 +1,7 @@
 """feedback tags.
 
-Revision ID: 0010_feedback_tags
-Revises: 0009_webhook_dedupe
+Revision ID: 0011_feedback_tags
+Revises: 0010_run_cancel_request
 Create Date: 2026-07-23
 """
 
@@ -12,8 +12,8 @@ from collections.abc import Sequence
 import sqlalchemy as sa
 from alembic import op
 
-revision: str = "0010_feedback_tags"
-down_revision: str | Sequence[str] | None = "0009_webhook_dedupe"
+revision: str = "0011_feedback_tags"
+down_revision: str | Sequence[str] | None = "0010_run_cancel_request"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 

backend/packages/harness/deerflow/persistence/migrations/versions/0012_feedback_drop_message_id.py

@@ -4,8 +4,8 @@ Feedback is bound to a run, not to a single message: nothing ever wrote or
 read ``message_id`` (no API field, no frontend usage, no query), so the
 column is redundant design and the domain aggregate no longer carries it.
 
-Revision ID: 0011_feedback_drop_message_id
-Revises: 0010_feedback_tags
+Revision ID: 0012_feedback_drop_message_id
+Revises: 0011_feedback_tags
 Create Date: 2026-07-29
 """
 
@@ -17,8 +17,8 @@ import sqlalchemy as sa
 
 from deerflow.persistence.migrations._helpers import safe_add_column, safe_drop_column
 
-revision: str = "0011_feedback_drop_message_id"
-down_revision: str | Sequence[str] | None = "0010_feedback_tags"
+revision: str = "0012_feedback_drop_message_id"
+down_revision: str | Sequence[str] | None = "0011_feedback_tags"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 

backend/packages/harness/deerflow/persistence/run/model.py

@@ -49,6 +49,10 @@ class RunRow(Base):
     # Multi-worker run ownership
     owner_worker_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
     lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    # A non-owning worker records cancellation here; the owner consumes it
+    # while renewing its lease. The first action wins.
+    cancel_action: Mapped[str | None] = mapped_column(String(20), nullable=True)
+    cancel_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
 
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
     updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))

backend/packages/harness/deerflow/persistence/run/sql.py

@@ -11,11 +11,15 @@ import json
 from datetime import UTC, datetime, timedelta
 from typing import Any
 
-from sqlalchemy import or_, select, update
+from sqlalchemy import case, or_, select, update
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
 
 from deerflow.persistence.run.model import RunRow
-from deerflow.runtime.runs.store.base import RunStore
+from deerflow.runtime.runs.store.base import (
+    LeaseRenewal,
+    RunStore,
+    StatusFinalization,
+)
 from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
 from deerflow.utils.time import coerce_iso
 
@@ -77,7 +81,7 @@ class RunRepository(RunStore):
         # Convert datetime to ISO string for consistency with MemoryRunStore.
         # SQLite drops tzinfo on read despite ``DateTime(timezone=True)`` —
         # ``coerce_iso`` normalizes naive datetimes as UTC.
-        for key in ("created_at", "updated_at", "lease_expires_at"):
+        for key in ("created_at", "updated_at", "lease_expires_at", "cancel_requested_at"):
             val = d.get(key)
             if isinstance(val, datetime):
                 d[key] = coerce_iso(val)
@@ -512,6 +516,105 @@ class RunRepository(RunStore):
             await session.commit()
             return result.rowcount != 0
 
+    async def renew_lease(
+        self,
+        run_id: str,
+        *,
+        owner_worker_id: str,
+        lease_expires_at: str,
+    ) -> LeaseRenewal:
+        """Renew the owner lease and read cancellation intent atomically."""
+        lease_dt = datetime.fromisoformat(lease_expires_at)
+        async with self._sf() as session:
+            result = await session.execute(
+                update(RunRow)
+                .where(
+                    RunRow.run_id == run_id,
+                    RunRow.owner_worker_id == owner_worker_id,
+                    RunRow.status.in_(("pending", "running")),
+                )
+                .values(
+                    lease_expires_at=lease_dt,
+                    updated_at=datetime.now(UTC),
+                )
+                .returning(RunRow.run_id, RunRow.cancel_action)
+            )
+            row = result.first()
+            await session.commit()
+        if row is None:
+            return LeaseRenewal(renewed=False)
+        return LeaseRenewal(renewed=True, cancel_action=row.cancel_action)
+
+    async def request_cancel(self, run_id: str, *, action: str) -> str | None:
+        """Atomically persist the first cancellation action on an active run."""
+        if action not in ("interrupt", "rollback"):
+            raise ValueError(f"Unsupported cancellation action: {action}")
+        now = datetime.now(UTC)
+        async with self._sf() as session:
+            result = await session.execute(
+                update(RunRow)
+                .where(
+                    RunRow.run_id == run_id,
+                    RunRow.status.in_(("pending", "running")),
+                )
+                .values(
+                    cancel_action=case(
+                        (RunRow.cancel_action.is_(None), action),
+                        else_=RunRow.cancel_action,
+                    ),
+                    cancel_requested_at=case(
+                        (RunRow.cancel_requested_at.is_(None), now),
+                        else_=RunRow.cancel_requested_at,
+                    ),
+                    updated_at=now,
+                )
+                .returning(RunRow.cancel_action)
+            )
+            row = result.first()
+            await session.commit()
+        return row.cancel_action if row is not None else None
+
+    async def finalize_if_not_cancelled(
+        self,
+        run_id: str,
+        *,
+        status: str,
+        error: str | None = None,
+        stop_reason: str | None = None,
+    ) -> StatusFinalization:
+        """Atomically let completion win only before cancellation."""
+        values: dict[str, Any] = {
+            "status": status,
+            "updated_at": datetime.now(UTC),
+        }
+        if error is not None:
+            values["error"] = error
+        if stop_reason is not None:
+            values["stop_reason"] = stop_reason
+
+        async with self._sf() as session:
+            result = await session.execute(
+                update(RunRow)
+                .where(
+                    RunRow.run_id == run_id,
+                    RunRow.status.in_(("pending", "running")),
+                    RunRow.cancel_action.is_(None),
+                )
+                .values(**values)
+                .returning(RunRow.run_id)
+            )
+            if result.first() is not None:
+                await session.commit()
+                return StatusFinalization(finalized=True)
+
+            current = await session.execute(select(RunRow.cancel_action).where(RunRow.run_id == run_id))
+            cancel_action = current.scalar_one_or_none()
+            await session.commit()
+            return StatusFinalization(
+                finalized=False,
+                cancel_action=cancel_action,
+            )
+
     async def claim_for_takeover(
         self,
         run_id: str,

backend/packages/harness/deerflow/runtime/checkpoint_mode.py

@@ -13,7 +13,7 @@ from __future__ import annotations
 
 from typing import Any
 
-from deerflow.config.database_config import CheckpointChannelMode
+from deerflow.config.database_config import DEFAULT_CHECKPOINT_SNAPSHOT_FREQUENCY, CheckpointChannelMode
 
 INTERNAL_CHECKPOINT_MODE_KEY = "__deerflow_checkpoint_channel_mode"
 CHECKPOINT_MODE_METADATA_KEY = "deerflow_checkpoint_channel_mode"
@@ -28,6 +28,7 @@ class CheckpointModeReconfigurationError(RuntimeError):
 
 
 _frozen_checkpoint_channel_mode: CheckpointChannelMode | None = None
+_frozen_checkpoint_snapshot_frequency: int | None = None
 
 
 def frozen_checkpoint_channel_mode() -> CheckpointChannelMode | None:
@@ -44,6 +45,39 @@ def freeze_checkpoint_channel_mode(mode: CheckpointChannelMode) -> CheckpointCha
     return _frozen_checkpoint_channel_mode
 
 
+def frozen_checkpoint_snapshot_frequency() -> int | None:
+    """Return the process-frozen delta snapshot frequency, if already frozen."""
+    return _frozen_checkpoint_snapshot_frequency
+
+
+def freeze_checkpoint_snapshot_frequency(snapshot_frequency: int) -> int:
+    """Freeze the delta snapshot cadence alongside the channel mode.
+
+    The cadence is compiled into each graph's channel table, so like the
+    mode it is restart-required and must match across every process sharing
+    one checkpoint database. It is deliberately NOT stamped into checkpoint
+    metadata: the mode marker contract (absence = full) and the full ->
+    delta migration semantics are unchanged by the frequency value.
+    """
+    global _frozen_checkpoint_snapshot_frequency
+    if snapshot_frequency <= 0:
+        raise ValueError("snapshot frequency must be positive")
+    if _frozen_checkpoint_snapshot_frequency is None:
+        _frozen_checkpoint_snapshot_frequency = snapshot_frequency
+    elif _frozen_checkpoint_snapshot_frequency != snapshot_frequency:
+        raise CheckpointModeReconfigurationError("checkpoint_delta.snapshot_frequency is restart-required and cannot change in a running process")
+    return _frozen_checkpoint_snapshot_frequency
+
+
+def resolve_checkpoint_snapshot_frequency(snapshot_frequency: int | None = None) -> int:
+    """Resolve the effective snapshot frequency: explicit value, else the
+    process-frozen value, else the config default."""
+    if snapshot_frequency is not None:
+        return snapshot_frequency
+    frozen = _frozen_checkpoint_snapshot_frequency
+    return frozen if frozen is not None else DEFAULT_CHECKPOINT_SNAPSHOT_FREQUENCY
+
+
 def inject_checkpoint_mode(config: dict[str, Any], mode: CheckpointChannelMode) -> None:
     configurable = config.setdefault("configurable", {})
     configurable[INTERNAL_CHECKPOINT_MODE_KEY] = mode

backend/packages/harness/deerflow/runtime/checkpoint_state.py

@@ -33,7 +33,13 @@ def _finish_state_mutation(_state: dict[str, Any]) -> dict[str, Any]:
     return {}
 
 
-def build_state_mutation_graph(as_node: str, mode: CheckpointChannelMode, state_schema: Any | None = None) -> Any:
+def build_state_mutation_graph(
+    as_node: str,
+    mode: CheckpointChannelMode,
+    state_schema: Any | None = None,
+    *,
+    snapshot_frequency: int | None = None,
+) -> Any:
     """Compile a state-only graph whose single writer node finishes immediately.
 
     ``update_state(..., as_node=...)`` requires the node to be registered in
@@ -45,12 +51,15 @@ def build_state_mutation_graph(as_node: str, mode: CheckpointChannelMode, state_
     assistant graph was compiled with) whenever the write carries materialized
     state: the base ThreadState fallback does not know channels contributed by
     custom middleware, and writes to unknown channels are silently discarded.
+    The fallback resolves the delta snapshot cadence explicit arg ->
+    process-frozen -> config default; an explicit ``state_schema`` already
+    carries its cadence in its identity.
     """
     if not as_node:
         raise ValueError("as_node is required for checkpoint state mutation")
     from langgraph.graph import StateGraph
 
-    builder = StateGraph(state_schema if state_schema is not None else get_thread_state_schema(mode))
+    builder = StateGraph(state_schema if state_schema is not None else get_thread_state_schema(mode, snapshot_frequency))
     builder.add_node(as_node, _finish_state_mutation)
     builder.set_entry_point(as_node)
     builder.set_finish_point(as_node)

backend/packages/harness/deerflow/runtime/journal.py

@@ -178,6 +178,12 @@ def build_branch_history_seed_events(
 class RunJournal(BaseCallbackHandler):
     """LangChain callback handler that captures events to RunEventStore."""
 
+    # Subagents may execute on a persistent event loop in another thread. This
+    # handler owns loop-local tasks and a store/pool created for the parent run,
+    # so the isolated-loop context copier must not inherit it. LangGraph's own
+    # stream callbacks remain inheritable and keep child token frames flowing.
+    deerflow_loop_bound = True
+
     # Every callback only updates in-memory run state or schedules async IO.
     # Keeping callbacks on the run's event-loop thread serializes mutations
     # from parallel tool calls and prevents cancelled executor callbacks from

backend/packages/harness/deerflow/runtime/runs/manager.py

@@ -925,6 +925,65 @@ class RunManager:
             )
         return persisted
 
+    async def set_status_if_not_cancelled(
+        self,
+        run_id: str,
+        status: RunStatus,
+        *,
+        error: str | None = None,
+        stop_reason: str | None = None,
+        persist: bool = True,
+    ) -> str | None:
+        """Set a terminal status unless a durable cancellation won first."""
+        if not persist or not self.heartbeat_enabled or self._store is None:
+            await self.set_status(
+                run_id,
+                status,
+                error=error,
+                stop_reason=stop_reason,
+                persist=persist,
+            )
+            return None
+
+        try:
+            result = await self._call_store_with_retry(
+                "finalize_if_not_cancelled",
+                run_id,
+                lambda: self._store.finalize_if_not_cancelled(
+                    run_id,
+                    status=status.value,
+                    error=error,
+                    stop_reason=stop_reason,
+                ),
+            )
+        except Exception:
+            async with self._lock:
+                record = self._runs.get(run_id)
+            if record is not None:
+                await self._mark_ownership_lost(
+                    record,
+                    reason=("The durable store could not confirm whether cancellation or completion won."),
+                    require_active=False,
+                )
+            return None
+
+        if result.cancel_action is not None:
+            async with self._lock:
+                record = self._runs.get(run_id)
+                if record is not None:
+                    record.abort_action = result.cancel_action
+                    record.abort_event.set()
+            return result.cancel_action
+
+        await self.set_status(
+            run_id,
+            status,
+            error=error,
+            stop_reason=stop_reason,
+            persist=not result.finalized,
+        )
+        return None
+
     async def _ensure_delivery_receipt(self, record: RunRecord) -> bool:
         """Idempotently persist a zero-delivery receipt during recovery."""
         if self._event_store is None:
@@ -1037,6 +1096,94 @@ class RunManager:
         await self._persist_model_name(run_id, model_name)
         logger.info("Run %s model_name=%s", run_id, model_name)
 
+    async def _request_durable_cancel(
+        self,
+        run_id: str,
+        *,
+        action: str,
+    ) -> tuple[CancelOutcome, str | None]:
+        """Record cancellation and return the first action that won."""
+        if self._store is None:
+            return CancelOutcome.unknown, None
+        try:
+            winning_action = await self._call_store_with_retry(
+                "request_cancel",
+                run_id,
+                lambda: self._store.request_cancel(run_id, action=action),
+            )
+        except NotImplementedError:
+            # Keep third-party stores that predate durable cancellation on the
+            # old safe behavior instead of pretending the owner was notified.
+            logger.info(
+                "Run store does not support cross-worker cancellation for run %s",
+                run_id,
+            )
+            return CancelOutcome.lease_valid_elsewhere, None
+        except Exception:
+            logger.warning(
+                "Failed to persist cancellation request for run %s",
+                run_id,
+                exc_info=True,
+            )
+            return CancelOutcome.unknown, None
+
+        if winning_action is not None:
+            logger.info(
+                "Run %s cancellation requested (requested=%s,winner=%s)",
+                run_id,
+                action,
+                winning_action,
+            )
+            return CancelOutcome.requested, winning_action
+
+        # Completion may have won the race between the caller's read and the
+        # guarded cancellation UPDATE. Re-read so the API reports that precise
+        # terminal result rather than claiming the request was accepted.
+        try:
+            fresh = await self._store.get(run_id)
+        except Exception:
+            fresh = None
+        if fresh is None:
+            return CancelOutcome.unknown, None
+        if fresh.get("status") not in ("pending", "running"):
+            return CancelOutcome.not_cancellable, None
+        # A legacy/partial store implementation may decline the request while
+        # the owner is still live. Preserve the former lease-conflict signal.
+        return CancelOutcome.lease_valid_elsewhere, None
+
+    async def _request_remote_cancel(
+        self,
+        run_id: str,
+        *,
+        action: str,
+    ) -> CancelOutcome:
+        """Record cancellation for a run whose task belongs to another worker."""
+        outcome, _ = await self._request_durable_cancel(
+            run_id,
+            action=action,
+        )
+        return outcome
+
+    async def _signal_local_cancel(
+        self,
+        run_id: str,
+        *,
+        action: str,
+    ) -> None:
+        """Set process-local abort state without status persistence or cleanup."""
+        async with self._lock:
+            record = self._runs.get(run_id)
+            if record is None or record.status not in (RunStatus.pending, RunStatus.running) or record.abort_event.is_set():
+                return
+
+            record.abort_action = action
+            record.abort_event.set()
+            task_active = record.task is not None and not record.task.done()
+            record.finalizing = task_active
+            if task_active and record.status == RunStatus.running:
+                record.task.cancel()
+        logger.info("Run %s cancellation signalled locally (action=%s)", run_id, action)
+
     async def cancel(self, run_id: str, *, action: str = "interrupt") -> CancelOutcome:
         """Request cancellation of a run.
 
@@ -1051,9 +1198,9 @@ class RunManager:
           ``error``.  The owning worker is assumed dead (its heartbeat
           stopped renewing).
 
-        - **Lease still valid** — returns ``lease_valid_elsewhere`` so
-          the caller can return HTTP 409 + ``Retry-After`` to tell the
-          client when to retry.
+        - **Lease still valid** — durably records the cancellation action.
+          The owner observes it on its next heartbeat and performs the same
+          local abort/finalization path as a directly-routed request.
 
         In single-worker mode (``heartbeat_enabled=False``) store-only
         hydrated runs that aren't in-memory return ``not_active_locally``,
@@ -1075,8 +1222,33 @@ class RunManager:
             if record is not None:
                 if record.status == RunStatus.interrupted:
                     return CancelOutcome.cancelled  # idempotent
-                if record.status not in (RunStatus.pending, RunStatus.running):
+                if record.status not in (RunStatus.pending, RunStatus.running) and (not self.heartbeat_enabled or self._store is None):
                     return CancelOutcome.not_cancellable
+
+        durable_cancel_won = False
+        if record is not None and self.heartbeat_enabled and self._store is not None:
+            outcome, winning_action = await self._request_durable_cancel(
+                run_id,
+                action=action,
+            )
+            if outcome == CancelOutcome.requested:
+                action = winning_action or action
+                durable_cancel_won = True
+            elif outcome == CancelOutcome.unknown:
+                logger.warning(
+                    "Proceeding with local cancellation for run %s after durable cancel persistence failed",
+                    run_id,
+                )
+            elif outcome != CancelOutcome.lease_valid_elsewhere:
+                return outcome
+
+        async with self._lock:
+            record = self._runs.get(run_id)
+            if record is not None:
+                if record.status == RunStatus.interrupted or record.abort_event.is_set():
+                    return CancelOutcome.cancelled
+                if record.status not in (RunStatus.pending, RunStatus.running):
+                    return CancelOutcome.cancelled if durable_cancel_won else CancelOutcome.not_cancellable
                 record.abort_action = action
                 record.abort_event.set()
                 task_active = record.task is not None and not record.task.done()
@@ -1111,6 +1283,9 @@ class RunManager:
             logger.info("Run %s cancelled (action=%s)", run_id, action)
             return CancelOutcome.cancelled
 
+        if durable_cancel_won:
+            return CancelOutcome.cancelled
+
         # ------------------------------------------------------------------
         # Non-local path — no in-memory record, must consult the store.
         # ------------------------------------------------------------------
@@ -1131,6 +1306,8 @@ class RunManager:
             return CancelOutcome.unknown
 
         store_status = row.get("status")
+        if store_status == "interrupted":
+            return CancelOutcome.requested
         if store_status not in ("pending", "running"):
             return CancelOutcome.not_cancellable
 
@@ -1138,7 +1315,7 @@ class RunManager:
         lease_expires_at: str | None = row.get("lease_expires_at")
 
         if not is_lease_expired(lease_expires_at, grace_seconds=grace_seconds):
-            return CancelOutcome.lease_valid_elsewhere
+            return await self._request_remote_cancel(run_id, action=action)
 
         take_over_msg = f"Run reclaimed by worker {self._worker_id}: the owning worker ({row.get('owner_worker_id') or 'unknown'}) stopped renewing its lease and is presumed dead."
         try:
@@ -1160,7 +1337,7 @@ class RunManager:
             return CancelOutcome.taken_over
 
         # The conditional UPDATE matched 0 rows. Two causes:
-        #   (a) the owner renewed the lease → lease_valid_elsewhere.
+        #   (a) the owner renewed the lease → persist a cancellation request.
         #   (b) the row went terminal between our read and the claim
         #       (run finished, or another worker already took it over)
         #       → not_cancellable or taken_over.
@@ -1177,8 +1354,9 @@ class RunManager:
                 logger.info("Run %s takeover lost to another worker already at error", run_id)
                 return CancelOutcome.taken_over
             return CancelOutcome.not_cancellable
-        # Row is still active — lease must have been renewed by the owner.
-        return CancelOutcome.lease_valid_elsewhere
+        # Row is still active — lease was renewed by the owner while the
+        # takeover raced. Notify that owner instead of exposing routing as 409.
+        return await self._request_remote_cancel(run_id, action=action)
 
     def _compute_lease_expires_at(self) -> str | None:
         """Return the lease expiry ISO timestamp for a freshly created run.
@@ -1788,6 +1966,7 @@ class RunManager:
         if self._store is None or self._run_ownership_config is None:
             return
         lease_seconds = self._run_ownership_config.lease_seconds
+        cancellations: list[tuple[str, str]] = []
 
         async with self._lock:
             # Renew any pending/running run owned by this worker unless its
@@ -1815,16 +1994,16 @@ class RunManager:
             new_expiry = (datetime.now(UTC) + timedelta(seconds=lease_seconds)).isoformat()
             try:
                 async with asyncio.timeout(remaining):
-                    updated = await self._call_store_with_retry(
-                        "update_lease",
+                    renewal = await self._call_store_with_retry(
+                        "renew_lease",
                         run_id,
-                        lambda: self._store.update_lease(
+                        lambda: self._store.renew_lease(
                             run_id,
                             owner_worker_id=self._worker_id,
                             lease_expires_at=new_expiry,
                         ),
                     )
-                if updated:
+                if renewal.renewed:
                     if confirmed_deadline <= datetime.now(UTC):
                         await self._mark_ownership_lost(
                             record,
@@ -1838,8 +2017,18 @@ class RunManager:
                     # fields). Re-acquiring ``self._lock`` here would
                     # serialise against unrelated run mutations for no gain.
                     record.lease_expires_at = new_expiry
+                    if renewal.cancel_action is not None:
+                        action = renewal.cancel_action
+                        if action not in ("interrupt", "rollback"):
+                            logger.warning(
+                                "Run %s has invalid durable cancel action %r; using interrupt",
+                                run_id,
+                                action,
+                            )
+                            action = "interrupt"
+                        cancellations.append((run_id, action))
                 else:
-                    # ``update_lease`` returned False — the row was claimed
+                    # ``renew_lease`` returned False — the row was claimed
                     # by another worker (status is no longer pending/running,
                     # or ``owner_worker_id`` changed). Stop the local task so
                     # we don't waste CPU or overwrite the takeover status on
@@ -1870,6 +2059,15 @@ class RunManager:
                         exc_info=True,
                     )
 
+        # Keep cancellation status writes and cleanup out of the sole renewal
+        # loop. After every local lease has had a chance to renew, only signal
+        # the owning worker task; that task performs normal terminal handling.
+        for run_id, action in cancellations:
+            await self._signal_local_cancel(
+                run_id,
+                action=action,
+            )
+
     async def _reconcile_orphans_periodic(self) -> None:
         """Sweep for expired leases owned by dead peers.
 
@@ -2040,6 +2238,7 @@ class CancelOutcome(StrEnum):
     """Result of a :meth:`RunManager.cancel` call."""
 
     cancelled = "cancelled"
+    requested = "requested"
     taken_over = "taken_over"
     lease_valid_elsewhere = "lease_valid_elsewhere"
     not_cancellable = "not_cancellable"

backend/packages/harness/deerflow/runtime/runs/store/__init__.py

@@ -1,4 +1,4 @@
-from deerflow.runtime.runs.store.base import RunStore
+from deerflow.runtime.runs.store.base import LeaseRenewal, RunStore
 from deerflow.runtime.runs.store.memory import MemoryRunStore
 
-__all__ = ["MemoryRunStore", "RunStore"]
+__all__ = ["LeaseRenewal", "MemoryRunStore", "RunStore"]

backend/packages/harness/deerflow/runtime/runs/store/base.py

@@ -21,6 +21,26 @@ class EditReplayVisibility:
     hidden_attempt_run_ids: set[str] = field(default_factory=set)
 
 
+@dataclass(frozen=True)
+class LeaseRenewal:
+    """Result of renewing a run lease.
+
+    ``cancel_action`` carries a durable cancellation request to the owning
+    worker without transferring lease ownership.
+    """
+
+    renewed: bool
+    cancel_action: str | None = None
+
+
+@dataclass(frozen=True)
+class StatusFinalization:
+    """Result of completing a run only if cancellation has not won."""
+
+    finalized: bool
+    cancel_action: str | None = None
+
+
 class RunStore(abc.ABC):
     @abc.abstractmethod
     async def put(
@@ -217,6 +237,57 @@ class RunStore(abc.ABC):
         """Renew the lease on an active run. Returns ``False`` when no row matched."""
         pass
 
+    async def renew_lease(
+        self,
+        run_id: str,
+        *,
+        owner_worker_id: str,
+        lease_expires_at: str,
+    ) -> LeaseRenewal:
+        """Renew ownership and return any durable cancellation request.
+
+        The default wraps the legacy ``update_lease`` method and returns no
+        cancellation action, so third-party stores remain source-compatible
+        without adding a background read. Stores that support multi-process
+        cancellation must override this method to renew and observe the
+        request atomically.
+        """
+        renewed = await self.update_lease(
+            run_id,
+            owner_worker_id=owner_worker_id,
+            lease_expires_at=lease_expires_at,
+        )
+        return LeaseRenewal(renewed=renewed)
+
+    async def request_cancel(self, run_id: str, *, action: str) -> str | None:
+        """Persist the first cancellation action for an active run.
+
+        Implementations must update only ``pending`` or ``running`` rows and
+        return the winning action, or ``None`` when no active row matched.
+        """
+        raise NotImplementedError
+
+    async def finalize_if_not_cancelled(
+        self,
+        run_id: str,
+        *,
+        status: str,
+        error: str | None = None,
+        stop_reason: str | None = None,
+    ) -> StatusFinalization:
+        """Atomically finalize an active run unless cancellation won.
+
+        The compatibility default is safe for stores that do not implement
+        durable cancellation.
+        """
+        updated = await self.update_status(
+            run_id,
+            status,
+            error=error,
+            stop_reason=stop_reason,
+        )
+        return StatusFinalization(finalized=updated is not False)
+
     @abc.abstractmethod
     async def claim_for_takeover(
         self,

backend/packages/harness/deerflow/runtime/runs/store/memory.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 from datetime import UTC, datetime, timedelta
 from typing import Any
 
-from deerflow.runtime.runs.store.base import RunStore
+from deerflow.runtime.runs.store.base import LeaseRenewal, RunStore, StatusFinalization
 
 
 class MemoryRunStore(RunStore):
@@ -52,6 +52,7 @@ class MemoryRunStore(RunStore):
         lease_expires_at=None,
     ):
         now = datetime.now(UTC).isoformat()
+        existing = self._runs.get(run_id)
         self._runs[run_id] = {
             "run_id": run_id,
             "thread_id": thread_id,
@@ -69,6 +70,10 @@ class MemoryRunStore(RunStore):
             "updated_at": now,
             "owner_worker_id": owner_worker_id,
             "lease_expires_at": lease_expires_at,
+            # ``put`` is an idempotent snapshot write. Preserve a cancellation
+            # request that may have raced a retry of an earlier snapshot.
+            "cancel_action": existing.get("cancel_action") if existing else None,
+            "cancel_requested_at": existing.get("cancel_requested_at") if existing else None,
         }
         self._index_run(run_id, thread_id)
 
@@ -254,6 +259,66 @@ class MemoryRunStore(RunStore):
         run["updated_at"] = datetime.now(UTC).isoformat()
         return True
 
+    async def renew_lease(
+        self,
+        run_id: str,
+        *,
+        owner_worker_id: str,
+        lease_expires_at: str,
+    ) -> LeaseRenewal:
+        # Delegate through ``update_lease`` so lightweight subclasses and tests
+        # that override the legacy primitive keep the same behavior.
+        renewed = await self.update_lease(
+            run_id,
+            owner_worker_id=owner_worker_id,
+            lease_expires_at=lease_expires_at,
+        )
+        if not renewed:
+            return LeaseRenewal(renewed=False)
+        run = self._runs.get(run_id)
+        return LeaseRenewal(
+            renewed=True,
+            cancel_action=run.get("cancel_action") if run is not None else None,
+        )
+
+    async def request_cancel(self, run_id: str, *, action: str) -> str | None:
+        if action not in ("interrupt", "rollback"):
+            raise ValueError(f"Unsupported cancellation action: {action}")
+        run = self._runs.get(run_id)
+        if run is None or run["status"] not in ("pending", "running"):
+            return None
+        if run.get("cancel_action") is None:
+            run["cancel_action"] = action
+            run["cancel_requested_at"] = datetime.now(UTC).isoformat()
+        run["updated_at"] = datetime.now(UTC).isoformat()
+        return run["cancel_action"]
+
+    async def finalize_if_not_cancelled(
+        self,
+        run_id: str,
+        *,
+        status: str,
+        error: str | None = None,
+        stop_reason: str | None = None,
+    ) -> StatusFinalization:
+        run = self._runs.get(run_id)
+        if run is None:
+            return StatusFinalization(finalized=False)
+        if run.get("cancel_action") is not None:
+            return StatusFinalization(
+                finalized=False,
+                cancel_action=run["cancel_action"],
+            )
+        if run["status"] not in ("pending", "running"):
+            return StatusFinalization(finalized=False)
+        run["status"] = status
+        if error is not None:
+            run["error"] = error
+        if stop_reason is not None:
+            run["stop_reason"] = stop_reason
+        run["updated_at"] = datetime.now(UTC).isoformat()
+        return StatusFinalization(finalized=True)
+
     async def claim_for_takeover(
         self,
         run_id: str,
@@ -409,6 +474,8 @@ class MemoryRunStore(RunStore):
             "error": None,
             "owner_worker_id": owner_worker_id,
             "lease_expires_at": lease_expires_at,
+            "cancel_action": None,
+            "cancel_requested_at": None,
             "created_at": created_at or now,
             "updated_at": now,
         }

backend/packages/harness/deerflow/runtime/runs/worker.py

@@ -389,6 +389,9 @@ class RunContext:
     thread_store: Any | None = field(default=None)
     app_config: AppConfig | None = field(default=None)
     checkpoint_channel_mode: CheckpointChannelMode = "full"
+    # Delta snapshot cadence frozen at startup; ``None`` means "not frozen in
+    # this process" (embedded/tests) and resolves to the config default.
+    checkpoint_snapshot_frequency: int | None = None
     on_run_completed: Any | None = field(default=None)
 
 
@@ -548,6 +551,50 @@ async def run_agent(
     subagent_events: _SubagentEventBuffer | None = None
     started = False
 
+    async def _finish_cancellation(
+        action: str,
+        *,
+        restore_checkpoint: bool = True,
+    ) -> None:
+        nonlocal checkpoint_rollback_completed
+        await run_manager.set_finalizing(run_id, True)
+        if action == "rollback":
+            await run_manager.set_status(
+                run_id,
+                RunStatus.error,
+                error="Rolled back by user",
+                **terminal_status_kwargs,
+            )
+            if not restore_checkpoint:
+                return
+            try:
+                checkpoint_rollback_completed = await _rollback_to_pre_run_checkpoint(
+                    accessor=accessor,
+                    checkpointer=checkpointer,
+                    thread_id=thread_id,
+                    run_id=run_id,
+                    rollback_point=rollback_point,
+                    snapshot_capture_failed=snapshot_capture_failed,
+                )
+                logger.info(
+                    "Run %s rolled back to pre-run checkpoint %s",
+                    run_id,
+                    pre_run_checkpoint_id,
+                )
+            except Exception:
+                logger.warning(
+                    "Run %s cancellation rollback failed",
+                    run_id,
+                    exc_info=True,
+                )
+        else:
+            await run_manager.set_status(
+                run_id,
+                RunStatus.interrupted,
+                **terminal_status_kwargs,
+            )
+            logger.info("Run %s was cancelled", run_id)
+
     try:
         normalized_stream_modes = normalize_stream_modes(stream_modes)
         requested_modes: set[str] = set(normalized_stream_modes)
@@ -576,6 +623,11 @@ async def run_agent(
 
         start_outcome = await run_manager.try_start(run_id)
         if start_outcome is not RunStartOutcome.started:
+            if record.abort_event.is_set():
+                await _finish_cancellation(
+                    record.abort_action,
+                    restore_checkpoint=False,
+                )
             return
         started = True
 
@@ -881,45 +933,21 @@ async def run_agent(
 
         # 8. Final status
         if record.abort_event.is_set():
-            await run_manager.set_finalizing(run_id, True)
-            action = record.abort_action
-            if action == "rollback":
-                await run_manager.set_status(
-                    run_id,
-                    RunStatus.error,
-                    error="Rolled back by user",
-                    **terminal_status_kwargs,
-                )
-                try:
-                    checkpoint_rollback_completed = await _rollback_to_pre_run_checkpoint(
-                        accessor=accessor,
-                        checkpointer=checkpointer,
-                        thread_id=thread_id,
-                        run_id=run_id,
-                        rollback_point=rollback_point,
-                        snapshot_capture_failed=snapshot_capture_failed,
-                    )
-                    logger.info("Run %s rolled back to pre-run checkpoint %s", run_id, pre_run_checkpoint_id)
-                except Exception:
-                    logger.warning("Failed to rollback checkpoint for run %s", run_id, exc_info=True)
-            else:
-                await run_manager.set_status(
-                    run_id,
-                    RunStatus.interrupted,
-                    **terminal_status_kwargs,
-                )
+            await _finish_cancellation(record.abort_action)
         elif llm_error_fallback_message or (journal is not None and journal.had_llm_error_fallback):
             error_msg = llm_error_fallback_message
             if error_msg is None and journal is not None:
                 error_msg = journal.llm_error_fallback_message
             error_msg = error_msg or "LLM provider failed after retries"
             await _ensure_finalizing_before_edit_failure(run_manager, record)
-            await run_manager.set_status(
+            cancel_action = await run_manager.set_status_if_not_cancelled(
                 run_id,
                 RunStatus.error,
                 error=error_msg,
                 **terminal_status_kwargs,
             )
+            if cancel_action is not None:
+                await _finish_cancellation(cancel_action)
         else:
             runtime_context = runtime.context if isinstance(runtime.context, dict) else None
             # Guard middlewares that hard-stop a run by stripping tool_calls
@@ -947,63 +975,40 @@ async def run_agent(
                 produced_output_paths,
             )
             delivery_error = _delivery_error(delivery_content)
-            await run_manager.set_status(
+            cancel_action = await run_manager.set_status_if_not_cancelled(
                 run_id,
                 RunStatus.error if delivery_error else RunStatus.success,
                 error=delivery_error,
                 stop_reason=stop_reason,
                 **terminal_status_kwargs,
             )
+            if cancel_action is not None:
+                await _finish_cancellation(cancel_action)
 
     except asyncio.CancelledError:
-        await run_manager.set_finalizing(run_id, True)
-        action = record.abort_action
-        if action == "rollback":
-            await run_manager.set_status(
-                run_id,
-                RunStatus.error,
-                error="Rolled back by user",
-                **terminal_status_kwargs,
-            )
-            try:
-                checkpoint_rollback_completed = await _rollback_to_pre_run_checkpoint(
-                    accessor=accessor,
-                    checkpointer=checkpointer,
-                    thread_id=thread_id,
-                    run_id=run_id,
-                    rollback_point=rollback_point,
-                    snapshot_capture_failed=snapshot_capture_failed,
-                )
-                logger.info("Run %s was cancelled and rolled back", run_id)
-            except Exception:
-                logger.warning("Run %s cancellation rollback failed", run_id, exc_info=True)
-        else:
-            await _ensure_finalizing_before_edit_failure(run_manager, record)
-            await run_manager.set_status(
-                run_id,
-                RunStatus.interrupted,
-                **terminal_status_kwargs,
-            )
-            logger.info("Run %s was cancelled", run_id)
+        await _finish_cancellation(record.abort_action)
 
     except Exception as exc:
         error_msg = f"{exc}"
         logger.exception("Run %s failed: %s", run_id, error_msg)
         await _ensure_finalizing_before_edit_failure(run_manager, record)
-        await run_manager.set_status(
+        cancel_action = await run_manager.set_status_if_not_cancelled(
             run_id,
             RunStatus.error,
             error=error_msg,
             **terminal_status_kwargs,
         )
-        await bridge.publish(
-            run_id,
-            "error",
-            {
-                "message": error_msg,
-                "name": type(exc).__name__,
-            },
-        )
+        if cancel_action is not None:
+            await _finish_cancellation(cancel_action)
+        else:
+            await bridge.publish(
+                run_id,
+                "error",
+                {
+                    "message": error_msg,
+                    "name": type(exc).__name__,
+                },
+            )
 
     finally:
         if record.ownership_lost:
@@ -1092,7 +1097,18 @@ async def run_agent(
                 # real worker outcome. Leaving a successful row inflight would
                 # let lease recovery rewrite it as an error with a synthetic
                 # zero receipt.
-                await run_manager.persist_current_status(run_id)
+                if record.abort_event.is_set():
+                    await run_manager.persist_current_status(run_id)
+                else:
+                    cancel_action = await run_manager.set_status_if_not_cancelled(
+                        run_id,
+                        record.status,
+                        error=record.error,
+                        stop_reason=record.stop_reason,
+                    )
+                    if cancel_action is not None:
+                        await _finish_cancellation(cancel_action)
+                        await run_manager.persist_current_status(run_id)
             except Exception:
                 logger.warning("Failed to persist terminal status for run %s after delivery receipt attempts", run_id, exc_info=True)
 

backend/packages/harness/deerflow/runtime/secret_context.py

@@ -31,6 +31,25 @@ ACTIVE_SECRETS_CONTEXT_KEY = "__active_skill_secrets"
 # entire value must be stripped from every observable serialization surface.
 SKILL_TOOL_POLICY_DECISION_CONTEXT_KEY = "__skill_tool_policy_decision"
 
+LEGACY_AUTH_TOKEN_METADATA_KEY = "auth_token"
+
+
+class LegacyRunMetadataSecretError(ValueError):
+    """Raised when a run puts a request credential in persisted metadata."""
+
+
+def validate_run_metadata_secrets(metadata: Any) -> None:
+    """Reject the legacy credential field at run admission."""
+    if isinstance(metadata, dict) and LEGACY_AUTH_TOKEN_METADATA_KEY in metadata:
+        raise LegacyRunMetadataSecretError("Run metadata key 'auth_token' is not allowed; pass request-scoped credentials via config.context.secrets instead.")
+
+
+def redact_metadata_secrets(metadata: Any) -> Any:
+    """Return API-safe metadata without mutating historical storage objects."""
+    if not isinstance(metadata, dict):
+        return metadata
+    return {key: value for key, value in metadata.items() if key != LEGACY_AUTH_TOKEN_METADATA_KEY}
+
 
 def _string_pairs(raw: Any) -> dict[str, str]:
     if not isinstance(raw, dict):
@@ -130,17 +149,23 @@ def redact_secret_context_keys(context: Any) -> Any:
 def redact_config_secrets(config: Any) -> Any:
     """Return a copy of a run config safe to persist or echo back to clients.
 
-    The request config (``body.config``) is stored verbatim on the run record
-    (``runs.kwargs_json``) and echoed by the run API. Strip the secret-bearing
-    keys from its ``context`` so a request-scoped secret is never persisted or
-    returned, while the live config that drives the run (built separately) keeps
-    them. Non-dict / context-less configs pass through unchanged.
+    The request config (``body.config``) would otherwise be stored verbatim on
+    the run record (``runs.kwargs_json``) and echoed by the run API. Strip secret-bearing keys
+    from its ``context`` and legacy credentials from its ``metadata`` so neither
+    protected config surface is persisted or returned, while the live config
+    that drives the run (built separately) keeps them. Ordinary metadata is
+    preserved. Non-dict configs pass through unchanged.
     """
     if not isinstance(config, dict):
         return config
-    context = config.get("context")
-    if not isinstance(context, dict):
-        return config
+
     redacted = dict(config)
-    redacted["context"] = redact_secret_context_keys(context)
+    context = config.get("context")
+    if isinstance(context, dict):
+        redacted["context"] = redact_secret_context_keys(context)
+
+    metadata = config.get("metadata")
+    if isinstance(metadata, dict):
+        redacted["metadata"] = redact_metadata_secrets(metadata)
+
     return redacted

backend/packages/harness/deerflow/runtime/user_context.py

@@ -34,6 +34,7 @@ a background task must *not* see the foreground user, wrap it with
 
 from __future__ import annotations
 
+from collections.abc import Mapping
 from contextvars import ContextVar, Token
 from typing import Final, Protocol, runtime_checkable
 
@@ -109,19 +110,90 @@ def get_effective_user_id() -> str:
     return str(user.id)
 
 
+def _storage_user_id_from_auth_identity(identity: object | None) -> str | None:
+    """Return a stable storage-safe ID for a LangGraph auth identity."""
+    if not isinstance(identity, str) or not identity:
+        return None
+
+    # LangGraph permits arbitrary strings (commonly email addresses) for
+    # BaseUser.identity, while DeerFlow's user directories require a narrower
+    # charset. Keep the normalization at the auth boundary so graph
+    # construction and runtime middleware always select the same bucket.
+    from deerflow.config.paths import make_safe_user_id
+
+    return make_safe_user_id(identity)
+
+
+def _user_id_from_auth_user(user: object | None) -> str | None:
+    if isinstance(user, Mapping):
+        identity = user.get("identity")
+    else:
+        identity = getattr(user, "identity", None)
+    return _storage_user_id_from_auth_identity(identity)
+
+
+def _user_id_from_langgraph_config(config: object | None) -> str | None:
+    if not isinstance(config, Mapping):
+        return None
+    configurable = config.get("configurable")
+    if not isinstance(configurable, Mapping):
+        return None
+
+    user_id = _storage_user_id_from_auth_identity(configurable.get("langgraph_auth_user_id"))
+    if user_id:
+        return user_id
+    return _user_id_from_auth_user(configurable.get("langgraph_auth_user"))
+
+
+def resolve_config_user_id(config: object | None) -> str:
+    """Resolve the effective user from a LangGraph/Gateway run config.
+
+    Server-owned LangGraph authentication fields take precedence over ordinary
+    ``user_id`` values because Agent Server reserves and overwrites the auth
+    fields, while a standalone client may supply regular configurable/context
+    values. Gateway runtime context remains the next source for DeerFlow's
+    embedded run path, followed by the legacy configurable channel and the
+    request ContextVar/default fallback.
+    """
+    langgraph_user_id = _user_id_from_langgraph_config(config)
+    if langgraph_user_id:
+        return langgraph_user_id
+
+    if isinstance(config, Mapping):
+        context = config.get("context")
+        if isinstance(context, Mapping):
+            context_user_id = context.get("user_id")
+            if context_user_id:
+                return str(context_user_id)
+
+        configurable = config.get("configurable")
+        if isinstance(configurable, Mapping):
+            configurable_user_id = configurable.get("user_id")
+            if configurable_user_id:
+                return str(configurable_user_id)
+
+    return get_effective_user_id()
+
+
 def resolve_runtime_user_id(runtime: object | None) -> str:
     """Single source of truth for a tool/middleware's effective user_id.
 
     Resolution order (most authoritative first):
-      1. ``runtime.context["user_id"]`` — set by ``inject_authenticated_user_context``
+      1. ``runtime.server_info.user.identity`` — populated by current LangGraph
+         runtimes from Agent Server's authenticated user. Unlike ordinary run
+         context, this is server-owned.
+      2. ``config["configurable"]["langgraph_auth_user_id"]`` — populated by
+         LangGraph Server from the deployment's ``@auth.authenticate`` result.
+         This supports older runtimes and code paths without ``server_info``.
+      3. ``runtime.context["user_id"]`` — set by ``inject_authenticated_user_context``
          in the gateway from the auth-validated ``request.state.user``. This is
          the only source that survives boundaries where the contextvar may have
          been lost (background tasks scheduled outside the request task,
          worker pools that don't copy_context, future cross-process drivers).
-      2. The ``_current_user`` ContextVar — set by the auth middleware at
+      4. The ``_current_user`` ContextVar — set by the auth middleware at
          request entry. Reliable for in-task work; copied by ``asyncio``
          child tasks and by ``ContextThreadPoolExecutor``.
-      3. ``DEFAULT_USER_ID`` — last-resort fallback so unauthenticated
+      5. ``DEFAULT_USER_ID`` — last-resort fallback so unauthenticated
          CLI / migration / test paths keep working without raising.
 
     Tools that persist user-scoped state (custom agents, memory, uploads)
@@ -129,14 +201,42 @@ def resolve_runtime_user_id(runtime: object | None) -> str:
     benefit from the runtime.context channel that ``setup_agent`` already
     relies on.
     """
+    server_info = getattr(runtime, "server_info", None)
+    server_user_id = _user_id_from_auth_user(getattr(server_info, "user", None))
+    if server_user_id:
+        return server_user_id
+
+    langgraph_user_id = _user_id_from_langgraph_auth()
+    if langgraph_user_id:
+        return langgraph_user_id
+
     context = getattr(runtime, "context", None)
-    if isinstance(context, dict):
+    if isinstance(context, Mapping):
         ctx_user_id = context.get("user_id")
         if ctx_user_id:
             return str(ctx_user_id)
     return get_effective_user_id()
 
 
+def _user_id_from_langgraph_auth() -> str | None:
+    """Return the authenticated LangGraph Server user id, if available.
+
+    LangGraph Server reserves the ``langgraph_auth_user`` and
+    ``langgraph_auth_user_id`` configurable keys and populates them from the
+    deployment's ``@auth.authenticate`` result. ``get_config()`` raises when
+    called outside a runnable context, which simply means this identity channel
+    is unavailable.
+    """
+    try:
+        from langgraph.config import get_config
+
+        config = get_config()
+    except RuntimeError:
+        return None
+
+    return _user_id_from_langgraph_config(config)
+
+
 # ---------------------------------------------------------------------------
 # Sentinel-based user_id resolution
 # ---------------------------------------------------------------------------

backend/packages/harness/deerflow/sandbox/middleware.py

@@ -9,11 +9,12 @@ from langchain.agents.middleware import AgentMiddleware
 from langchain_core.messages import ToolMessage
 from langgraph.prebuilt.tool_node import ToolCallRequest
 from langgraph.runtime import Runtime
-from langgraph.types import Command, Overwrite
+from langgraph.types import Command
 
 from deerflow.agents.thread_state import SandboxStateField, ThreadDataState
 from deerflow.runtime.user_context import resolve_runtime_user_id
 from deerflow.sandbox import get_sandbox_provider
+from deerflow.sandbox.overwrite import unwrap_sandbox
 
 logger = logging.getLogger(__name__)
 
@@ -25,24 +26,6 @@ class SandboxMiddlewareState(AgentState):
     thread_data: NotRequired[ThreadDataState | None]
 
 
-def _unwrap_sandbox(sandbox: object) -> tuple[object, bool]:
-    """Unwrap an ``Overwrite``-wrapped sandbox channel value, if present.
-
-    Fork-restored checkpoints can deliver the sandbox channel still wrapped in
-    ``langgraph.types.Overwrite`` (the rollback restore applies replace-style
-    writes through a state-mutation graph in delta checkpoint mode). Reading
-    ``sandbox["sandbox_id"]`` on the wrapper itself crashes with ``TypeError:
-    'Overwrite' object is not subscriptable``, so unwrap before use.
-
-    Returns ``(value, fork_restored)``. The wrapped form replays the parent
-    thread's sandbox state, so callers must not treat the sandbox as owned by
-    this run (e.g. release it).
-    """
-    if isinstance(sandbox, Overwrite):
-        return sandbox.value, True
-    return sandbox, False
-
-
 class SandboxMiddleware(AgentMiddleware[SandboxMiddlewareState]):
     """Create a sandbox environment and assign it to an agent.
 
@@ -117,7 +100,7 @@ class SandboxMiddleware(AgentMiddleware[SandboxMiddlewareState]):
 
     @override
     def after_agent(self, state: SandboxMiddlewareState, runtime: Runtime) -> dict | None:
-        sandbox, fork_restored = _unwrap_sandbox(state.get("sandbox"))
+        sandbox, fork_restored = unwrap_sandbox(state.get("sandbox"))
         if sandbox is not None:
             sandbox_id = sandbox["sandbox_id"]
             if fork_restored:
@@ -140,7 +123,7 @@ class SandboxMiddleware(AgentMiddleware[SandboxMiddlewareState]):
 
     @override
     async def aafter_agent(self, state: SandboxMiddlewareState, runtime: Runtime) -> dict | None:
-        sandbox, fork_restored = _unwrap_sandbox(state.get("sandbox"))
+        sandbox, fork_restored = unwrap_sandbox(state.get("sandbox"))
         if sandbox is not None:
             sandbox_id = sandbox["sandbox_id"]
             if fork_restored:

backend/packages/harness/deerflow/sandbox/overwrite.py

@@ -0,0 +1,21 @@
+"""Helpers for ``langgraph.types.Overwrite``-wrapped channel values."""
+
+from langgraph.types import Overwrite
+
+
+def unwrap_sandbox(sandbox: object) -> tuple[object, bool]:
+    """Unwrap an ``Overwrite``-wrapped sandbox channel value, if present.
+
+    Fork-restored checkpoints can deliver the sandbox channel still wrapped in
+    ``langgraph.types.Overwrite`` (the rollback restore applies replace-style
+    writes through a state-mutation graph in delta checkpoint mode). Reading
+    ``sandbox["sandbox_id"]`` or ``sandbox.get("sandbox_id")`` on the wrapper
+    itself crashes, so unwrap before use.
+
+    Returns ``(value, fork_restored)``. The wrapped form replays the parent
+    thread's sandbox state, so callers must not treat the sandbox as owned by
+    this run (e.g. release it).
+    """
+    if isinstance(sandbox, Overwrite):
+        return sandbox.value, True
+    return sandbox, False

backend/packages/harness/deerflow/sandbox/tools.py

@@ -23,6 +23,7 @@ from deerflow.sandbox.exceptions import (
     SandboxRuntimeError,
 )
 from deerflow.sandbox.file_operation_lock import get_file_operation_lock
+from deerflow.sandbox.overwrite import unwrap_sandbox
 from deerflow.sandbox.path_patterns import build_output_mask_pattern
 from deerflow.sandbox.sandbox import Sandbox
 from deerflow.sandbox.sandbox_provider import get_sandbox_provider
@@ -1329,7 +1330,9 @@ def is_local_sandbox(runtime: Runtime | None) -> bool:
         return False
     if runtime.state is None:
         return False
-    sandbox_state = runtime.state.get("sandbox")
+    # Read-only classification: the id is only matched, so a fork-restored
+    # wrapper is safe to discard here (nothing gets released on this path).
+    sandbox_state, _ = unwrap_sandbox(runtime.state.get("sandbox"))
     if sandbox_state is None:
         return False
     sandbox_id = sandbox_state.get("sandbox_id")
@@ -1352,7 +1355,9 @@ def sandbox_from_runtime(runtime: Runtime | None = None) -> Sandbox:
         raise SandboxRuntimeError("Tool runtime not available")
     if runtime.state is None:
         raise SandboxRuntimeError("Tool runtime state not available")
-    sandbox_state = runtime.state.get("sandbox")
+    # Read-only lookup: this only resolves the provider entry, and ownership
+    # (release) stays with after_agent's short-circuit on the wrapped state.
+    sandbox_state, _ = unwrap_sandbox(runtime.state.get("sandbox"))
     if sandbox_state is None:
         raise SandboxRuntimeError("Sandbox state not initialized in runtime")
     sandbox_id = sandbox_state.get("sandbox_id")
@@ -1392,7 +1397,10 @@ def ensure_sandbox_initialized(runtime: Runtime | None = None) -> Sandbox:
         raise SandboxRuntimeError("Tool runtime state not available")
 
     # Check if sandbox already exists in state
-    sandbox_state = runtime.state.get("sandbox")
+    # Discarding fork_restored is safe: after_agent short-circuits on the
+    # still-wrapped state before the context-based release branch, so this
+    # reuse path never releases the parent sandbox.
+    sandbox_state, _ = unwrap_sandbox(runtime.state.get("sandbox"))
     if sandbox_state is not None:
         sandbox_id = sandbox_state.get("sandbox_id")
         if sandbox_id is not None:
@@ -1439,7 +1447,9 @@ async def ensure_sandbox_initialized_async(runtime: Runtime | None = None) -> Sa
     if runtime.state is None:
         raise SandboxRuntimeError("Tool runtime state not available")
 
-    sandbox_state = runtime.state.get("sandbox")
+    # Same discard as the sync path above: the reuse path never releases,
+    # because after_agent short-circuits on the still-wrapped state first.
+    sandbox_state, _ = unwrap_sandbox(runtime.state.get("sandbox"))
     if sandbox_state is not None:
         sandbox_id = sandbox_state.get("sandbox_id")
         if sandbox_id is not None:
@@ -1738,13 +1748,18 @@ def _lark_cli_env_from_runtime(runtime: Runtime, command: str, *, sandbox_paths:
     unrelated unauthenticated profile. Keep this scoped to commands that
     actually call ``lark-cli`` so ordinary bash calls do not switch AIO into the
     env-bearing execution path.
+
+    In broker mode (Pattern B, issue #4338) a sidecar owns the credentials, so
+    the overlay carries only the broker URL + runtime PATH — the config/data
+    directories are never injected into the sandbox.
     """
     if not _LARK_CLI_COMMAND_RE.search(command):
         return None
     try:
-        from deerflow.integrations.lark_cli import lark_cli_env_overlay
+        from deerflow.integrations.lark_cli import lark_cli_env_overlay, sandbox_lark_broker_active
 
-        return lark_cli_env_overlay(resolve_runtime_user_id(runtime), sandbox_paths=sandbox_paths)
+        broker = sandbox_paths and sandbox_lark_broker_active()
+        return lark_cli_env_overlay(resolve_runtime_user_id(runtime), sandbox_paths=sandbox_paths, broker=broker)
     except Exception:
         logger.warning("Could not build Lark CLI env overlay; running command without managed auth", exc_info=True)
         return None

backend/packages/harness/deerflow/subagents/config.py

@@ -17,8 +17,10 @@ class SubagentConfig:
         system_prompt: The system prompt that guides the subagent's behavior.
         tools: Optional list of tool names to allow. If None, inherits all tools.
         disallowed_tools: Optional list of tool names to deny.
-        skills: Optional list of skill names to load. If None, inherits all enabled skills.
-                If an empty list, no skills are loaded.
+        skills: Optional list of skill names to make discoverable and activatable.
+                If None, all enabled skills are available. If empty, skills are
+                disabled for this subagent. Skill bodies and their allowed-tools
+                policies take effect only after activation/loading at runtime.
         model: Model to use - 'inherit' uses parent's model.
         max_turns: Maximum agent turns before stopping. Built-in agents use the
             value set here (general-purpose=150, bash=60) unless the global

backend/packages/harness/deerflow/subagents/executor.py

@@ -2,7 +2,6 @@
 
 import asyncio
 import atexit
-import html
 import logging
 import os
 import threading
@@ -18,8 +17,10 @@ from typing import TYPE_CHECKING, Any
 
 from langchain.agents import create_agent
 from langchain.tools import BaseTool
+from langchain_core.callbacks.base import BaseCallbackManager
 from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
 from langchain_core.runnables import RunnableConfig
+from langchain_core.runnables.config import var_child_runnable_config
 from langgraph.errors import GraphRecursionError
 
 from deerflow.agents.thread_state import SandboxState, ThreadDataState, ThreadState
@@ -28,7 +29,6 @@ from deerflow.config import get_app_config
 from deerflow.config.app_config import AppConfig
 from deerflow.models import create_chat_model
 from deerflow.runtime.user_context import DEFAULT_USER_ID
-from deerflow.skills.tool_policy import filter_tools_by_skill_allowed_tools
 from deerflow.skills.types import Skill
 from deerflow.subagents.config import SubagentConfig, resolve_subagent_model_name
 from deerflow.subagents.step_events import capture_new_step_messages
@@ -362,6 +362,44 @@ def _submit_to_isolated_loop_in_context(
     )
 
 
+def _copy_isolated_subagent_context() -> Context:
+    """Copy ambient context without loop-bound parent graph callbacks.
+
+    LangGraph keeps the current runnable config in a ``ContextVar``. Crossing
+    into the persistent subagent loop must retain checkpoint lineage, runtime
+    metadata, user identity, and tracing context. LangGraph merges inherited
+    and explicit callbacks, so merely supplying the subagent collector is
+    insufficient: loop-bound application callbacks such as the parent
+    ``RunJournal`` would still run on the isolated loop. Framework streaming
+    callbacks are intentionally preserved so namespaced child token frames
+    continue to reach the parent stream.
+    """
+    context = copy_context()
+    inherited_config = context.get(var_child_runnable_config)
+    if inherited_config is None or "callbacks" not in inherited_config:
+        return context
+
+    callbacks = inherited_config.get("callbacks")
+    if isinstance(callbacks, BaseCallbackManager):
+        isolated_callbacks = callbacks.copy()
+        isolated_callbacks.handlers = [handler for handler in callbacks.handlers if not getattr(handler, "deerflow_loop_bound", False)]
+        isolated_callbacks.inheritable_handlers = [handler for handler in callbacks.inheritable_handlers if not getattr(handler, "deerflow_loop_bound", False)]
+    elif isinstance(callbacks, (list, tuple)):
+        isolated_callbacks = [handler for handler in callbacks if not getattr(handler, "deerflow_loop_bound", False)]
+    elif getattr(callbacks, "deerflow_loop_bound", False):
+        isolated_callbacks = None
+    else:
+        isolated_callbacks = callbacks
+
+    isolated_config = inherited_config.copy()
+    if isolated_callbacks:
+        isolated_config["callbacks"] = isolated_callbacks
+    else:
+        isolated_config.pop("callbacks", None)
+    context.run(var_child_runnable_config.set, isolated_config)
+    return context
+
+
 def _filter_tools(
     all_tools: list[BaseTool],
     allowed: list[str] | None,
@@ -477,6 +515,10 @@ class SubagentExecutor:
             config.disallowed_tools,
         )
         self.tools = self._base_tools
+        # Populated from the same per-user, config-filtered registry used to
+        # build the prompt. Runtime skill activation/policy middleware receives
+        # this exact set so a subagent cannot activate an undisclosed skill.
+        self._available_skill_names: set[str] = set()
         # Guard middlewares that expose ``consume_stop_reason`` (currently
         # ``TokenBudgetMiddleware`` and ``LoopDetectionMiddleware``), captured in
         # ``_create_agent`` so ``_aexecute`` can read each after the run and
@@ -519,6 +561,8 @@ class SubagentExecutor:
             "lazy_init": True,
             "deferred_setup": deferred_setup,
             "agent_name": self.config.name,
+            "available_skills": self._available_skill_names,
+            "user_id": self.user_id or DEFAULT_USER_ID,
         }
         authz_provider = getattr(self, "_authz_provider", None)
         if authz_provider is not None:
@@ -595,43 +639,6 @@ class SubagentExecutor:
             return [s for s in all_skills if s.name in allowed]
         return all_skills
 
-    def _apply_skill_allowed_tools(self, skills: list[Skill]) -> list[BaseTool]:
-        return filter_tools_by_skill_allowed_tools(self._base_tools, skills)
-
-    async def _load_skill_messages(self, skills: list[Skill]) -> list[SystemMessage]:
-        """Load skill content as conversation items based on config.skills.
-
-        Aligned with Codex's pattern: each subagent loads its own skills
-        per-session and injects them as conversation items (developer messages),
-        not as system prompt text. The config.skills whitelist controls which
-        skills are loaded:
-        - None: load all enabled skills
-        - []: no skills
-        - ["skill-a", "skill-b"]: only these skills
-
-        Returns:
-            List of SystemMessages containing skill content.
-        """
-        if not skills:
-            return []
-
-        # Read each skill's SKILL.md content and create conversation items
-        messages = []
-        for skill in skills:
-            try:
-                content = await asyncio.to_thread(skill.skill_file.read_text, encoding="utf-8")
-                content = content.strip()
-                if content:
-                    # name/body are untrusted (installable ``.skill`` archive); escape
-                    # both so the body cannot forge a framework tag, matching the
-                    # slash-activation sibling (name quote=True attribute, body quote=False).
-                    messages.append(SystemMessage(content=f'<skill name="{html.escape(skill.name, quote=True)}">\n{html.escape(content, quote=False)}\n</skill>'))
-                    logger.info(f"[trace={self.trace_id}] Subagent {self.config.name} loaded skill: {skill.name}")
-            except Exception:
-                logger.debug(f"[trace={self.trace_id}] Failed to read skill {skill.name}", exc_info=True)
-
-        return messages
-
     async def _build_initial_state(self, task: str) -> tuple[dict[str, Any], list[BaseTool], "DeferredToolSetup"]:
         """Build the initial state for agent execution.
 
@@ -640,8 +647,8 @@ class SubagentExecutor:
 
         Returns:
             ``(state, final_tools, deferred_setup)``. ``final_tools`` is the
-            policy-filtered tool list with the ``tool_search`` tool appended when
-            deferral applies; ``deferred_setup`` is consumed by ``_create_agent``
+            authorized tool list with discovery helpers appended when their
+            deferral modes apply; ``deferred_setup`` is consumed by ``_create_agent``
             so the agent build and the injected ``<available-deferred-tools>``
             section share one catalog/hash.
         """
@@ -650,15 +657,26 @@ class SubagentExecutor:
         # re-enter this package during its own initialization.
         from deerflow.tools.builtins.tool_search import assemble_deferred_tools, get_deferred_tools_prompt_section, get_mcp_routing_hints_prompt_section
 
-        # Load skills as conversation items (Codex pattern)
+        # Skills are discoverable metadata until explicitly slash-activated or
+        # loaded through read_file. Their allowed-tools declarations are applied
+        # dynamically by SkillToolPolicyMiddleware, not eagerly here.
         skills = await self._load_skills()
-        filtered_tools = self._apply_skill_allowed_tools(skills)
+        self._available_skill_names = {skill.name for skill in skills}
+
+        resolved_app_config = self.app_config or get_app_config()
+
+        from deerflow.skills.describe import build_skill_search_setup, get_skill_index_prompt_section
+
+        skill_setup = build_skill_search_setup(
+            skills,
+            enabled=resolved_app_config.skills.deferred_discovery,
+            container_base_path=resolved_app_config.skills.container_path,
+        )
 
         # Apply authorization Layer 1: filter tools before deferred assembly
         # so denied tools can never enter the DeferredToolCatalog.
         from deerflow.authz.tool_filter import apply_tool_authorization
 
-        resolved_app_config = self.app_config or get_app_config()
         authz_context = {
             "user_id": self.user_id,
             "user_role": self.user_role,
@@ -668,36 +686,64 @@ class SubagentExecutor:
             "is_internal": self.is_internal,
             "authz_attributes": self.authz_attributes,
         }
-        filtered_tools, self._authz_provider = apply_tool_authorization(
-            filtered_tools,
+        authorization_candidates = [*self._base_tools]
+        if skill_setup.describe_skill_tool is not None:
+            authorization_candidates.append(skill_setup.describe_skill_tool)
+        configured_tool_ids = {id(tool) for tool in self._base_tools}
+        authorized_tools, self._authz_provider = apply_tool_authorization(
+            authorization_candidates,
             context=authz_context,
             app_config=resolved_app_config,
         )
+        configured_tools = [tool for tool in authorized_tools if id(tool) in configured_tool_ids]
+        late_tools = [tool for tool in authorized_tools if id(tool) not in configured_tool_ids]
 
-        # Assemble deferred tool_search AFTER policy filtering (fail-closed),
-        # mirroring the lead path so subagents stop binding full MCP schemas.
+        # Assemble deferred tool_search after the subagent's name allow/deny and
+        # authorization filters, mirroring the lead path so subagents stop
+        # binding full MCP schemas.
         # The generated tool_search helper is intentionally not subject to the
         # subagent's name-level allow/deny (config.tools / disallowed_tools):
-        # its catalog is built from the already-filtered list, so it can never
-        # surface a tool the policy denied. This matches the lead agent.
-        enabled = (self.app_config or get_app_config()).tool_search.enabled
-        final_tools, deferred_setup = assemble_deferred_tools(filtered_tools, enabled=enabled)
-        skill_messages = await self._load_skill_messages(skills)
+        # its catalog is built from that already-filtered list. Active skill
+        # policy is applied later by middleware to both schema visibility and
+        # execution, so promotion cannot widen an active skill's authority.
+        final_tools, deferred_setup = assemble_deferred_tools(
+            configured_tools,
+            enabled=resolved_app_config.tool_search.enabled,
+        )
+        final_tools.extend(late_tools)
 
-        # Combine system_prompt and skills into a single SystemMessage.
+        # Combine the system prompt and skill discovery metadata into a single
+        # SystemMessage. Full SKILL.md bodies are loaded only when activated.
         # Some LLM APIs reject multiple SystemMessages with
         # "System message must be at the beginning."
         system_parts: list[str] = []
         if self.config.system_prompt:
             system_parts.append(self.config.system_prompt)
-        for skill_msg in skill_messages:
-            system_parts.append(skill_msg.content)
+        if skills:
+            if skill_setup.skill_names:
+                skills_section = get_skill_index_prompt_section(
+                    skill_names=skill_setup.skill_names,
+                    container_base_path=resolved_app_config.skills.container_path,
+                )
+            else:
+                # Reuse the lead agent's metadata renderer in legacy discovery
+                # mode so both agent types describe the same skill catalog.
+                from deerflow.agents.lead_agent.prompt import get_skills_prompt_section
+
+                skills_section = await asyncio.to_thread(
+                    get_skills_prompt_section,
+                    self._available_skill_names,
+                    app_config=resolved_app_config,
+                    user_id=self.user_id or DEFAULT_USER_ID,
+                )
+            if skills_section:
+                system_parts.append(skills_section)
         # Name the deferred MCP tools in the prompt; their schemas stay withheld
         # until tool_search promotes them. Empty set -> "" -> appends nothing.
         deferred_section = get_deferred_tools_prompt_section(deferred_names=deferred_setup.deferred_names)
         if deferred_section:
             system_parts.append(deferred_section)
-        mcp_routing_hints_section = get_mcp_routing_hints_prompt_section(filtered_tools, deferred_names=deferred_setup.deferred_names)
+        mcp_routing_hints_section = get_mcp_routing_hints_prompt_section(authorized_tools, deferred_names=deferred_setup.deferred_names)
         if mcp_routing_hints_section:
             system_parts.append(mcp_routing_hints_section)
 
@@ -981,7 +1027,7 @@ class SubagentExecutor:
         from being tied to a short-lived loop that gets closed per execution.
         """
         future: Future[SubagentResult] | None = None
-        parent_context = copy_context()
+        parent_context = _copy_isolated_subagent_context()
         try:
             future = _submit_to_isolated_loop_in_context(
                 parent_context,
@@ -1077,7 +1123,7 @@ class SubagentExecutor:
         with _background_tasks_lock:
             _background_tasks[task_id] = result
 
-        parent_context = copy_context()
+        parent_context = _copy_isolated_subagent_context()
 
         # Submit to scheduler pool
         def run_task():

backend/scripts/benchmark/checkpoint/bench_channels.py

@@ -17,9 +17,10 @@ Examples::
         --backends sqlite --updates 1000 --payload-bytes 128 \
         --repetitions 7 --output snapshot-boundary.jsonl
 
-The controller suppresses matrix pairs whose estimated cumulative full-mode
-message payload exceeds ``--max-estimated-full-bytes``. Both modes are skipped
-as a pair so every emitted full/delta result remains comparable. Use
+The controller suppresses matrix cells whose estimated cumulative full-mode
+message payload exceeds ``--max-estimated-full-bytes``. Full mode and every
+swept delta cadence are skipped together so every emitted result remains
+comparable and subject to the same safety cap. Use
 ``--allow-large-cases`` only on a machine provisioned for the resulting disk
 and memory use.
 """
@@ -37,6 +38,7 @@ import tempfile
 import time
 from collections.abc import Callable
 from dataclasses import asdict, dataclass
+from functools import cache
 from pathlib import Path
 from typing import Annotated, Any, Literal, TypedDict
 
@@ -48,6 +50,7 @@ from langgraph.graph import StateGraph
 from langgraph.graph.message import add_messages
 
 from deerflow.agents.thread_state import merge_message_writes
+from deerflow.config.database_config import DEFAULT_CHECKPOINT_SNAPSHOT_FREQUENCY
 from deerflow.runtime.checkpoint_mode import inject_checkpoint_mode
 from deerflow.runtime.checkpoint_state import CheckpointStateAccessor
 
@@ -70,7 +73,7 @@ Backend = Literal["memory", "sqlite"]
 
 SCHEMA_VERSION = 1
 BENCHMARK_VERSION = 1
-PRODUCTION_SNAPSHOT_FREQUENCY = 1000
+PRODUCTION_SNAPSHOT_FREQUENCY = DEFAULT_CHECKPOINT_SNAPSHOT_FREQUENCY
 DEFAULT_MAX_ESTIMATED_FULL_BYTES = 1024**3
 _MODES: tuple[Mode, ...] = ("full", "delta")
 _BACKENDS: tuple[Backend, ...] = ("memory", "sqlite")
@@ -86,11 +89,13 @@ class _FullBenchmarkState(TypedDict):
     messages: Annotated[list[AnyMessage], add_messages]
 
 
-class _DeltaBenchmarkState(TypedDict):
-    messages: Annotated[
-        list[AnyMessage],
-        DeltaChannel(merge_message_writes, snapshot_frequency=PRODUCTION_SNAPSHOT_FREQUENCY),
-    ]
+@cache
+def _delta_benchmark_state(snapshot_frequency: int) -> type:
+    """Delta benchmark schema for one snapshot cadence (cached per value)."""
+    return TypedDict(
+        f"_DeltaBenchmarkState_f{snapshot_frequency}",
+        {"messages": Annotated[list[AnyMessage], DeltaChannel(merge_message_writes, snapshot_frequency=snapshot_frequency)]},
+    )
 
 
 @dataclass(frozen=True)
@@ -115,8 +120,8 @@ class BenchmarkCase:
             raise ValueError("payload_bytes must be positive")
         if self.repetition < 0:
             raise ValueError("repetition must be non-negative")
-        if self.snapshot_frequency != PRODUCTION_SNAPSHOT_FREQUENCY:
-            raise ValueError(f"Phase 1 supports only production snapshot_frequency={PRODUCTION_SNAPSHOT_FREQUENCY}")
+        if self.snapshot_frequency <= 0:
+            raise ValueError("snapshot_frequency must be positive")
         if self.scenario != "append":
             raise ValueError(f"unsupported scenario: {self.scenario!r}")
 
@@ -129,8 +134,15 @@ def _expand_cases(
     payload_bytes: list[int],
     repetitions: int,
     seed: int,
+    snapshot_frequencies: list[int] | None = None,
 ) -> list[BenchmarkCase]:
-    """Build a matrix with alternating mode order to reduce order bias."""
+    """Build a matrix with alternating mode order to reduce order bias.
+
+    ``snapshot_frequencies`` sweeps delta cases across cadences. Full-mode
+    cases ignore the cadence and run once per cell at the production default,
+    so a sweep costs no duplicate full-mode measurements.
+    """
+    frequencies = snapshot_frequencies or [PRODUCTION_SNAPSHOT_FREQUENCY]
     cases: list[BenchmarkCase] = []
     for repetition in range(repetitions):
         for backend in backends:
@@ -140,16 +152,19 @@ def _expand_cases(
                     if (repetition + update_index) % 2 == 1:
                         ordered_modes.reverse()
                     for mode in ordered_modes:
-                        cases.append(
-                            BenchmarkCase(
-                                mode=mode,  # type: ignore[arg-type]
-                                backend=backend,  # type: ignore[arg-type]
-                                update_count=update_count,
-                                payload_bytes=payload,
-                                repetition=repetition,
-                                seed=seed,
+                        mode_frequencies = frequencies if mode == "delta" else [PRODUCTION_SNAPSHOT_FREQUENCY]
+                        for snapshot_frequency in mode_frequencies:
+                            cases.append(
+                                BenchmarkCase(
+                                    mode=mode,  # type: ignore[arg-type]
+                                    backend=backend,  # type: ignore[arg-type]
+                                    update_count=update_count,
+                                    payload_bytes=payload,
+                                    repetition=repetition,
+                                    seed=seed,
+                                    snapshot_frequency=snapshot_frequency,
+                                )
                             )
-                        )
     return cases
 
 
@@ -161,7 +176,10 @@ def _estimated_full_payload_bytes(case: BenchmarkCase) -> int:
 def _filter_oversized_pairs(cases: list[BenchmarkCase], *, max_bytes: int | None) -> tuple[list[BenchmarkCase], list[BenchmarkCase]]:
     if max_bytes is None:
         return cases, []
-    group_fields = ("backend", "scenario", "snapshot_frequency", "update_count", "payload_bytes", "repetition")
+    # Cadence is a delta-only sweep dimension: full mode runs once per benchmark
+    # cell at the production default. Excluding it ensures an oversized full
+    # case suppresses every delta cadence in that same cell.
+    group_fields = ("backend", "scenario", "update_count", "payload_bytes", "repetition")
     oversized_keys = {tuple(getattr(case, field) for field in group_fields) for case in cases if case.mode == "full" and _estimated_full_payload_bytes(case) > max_bytes}
     kept = [case for case in cases if tuple(getattr(case, field) for field in group_fields) not in oversized_keys]
     skipped = [case for case in cases if tuple(getattr(case, field) for field in group_fields) in oversized_keys]
@@ -180,8 +198,8 @@ def _noop(_state: dict[str, Any]) -> dict[str, Any]:
     return {}
 
 
-def _build_graph(mode: Mode, saver: Any) -> Any:
-    schema = _DeltaBenchmarkState if mode == "delta" else _FullBenchmarkState
+def _build_graph(mode: Mode, saver: Any, snapshot_frequency: int) -> Any:
+    schema = _delta_benchmark_state(snapshot_frequency) if mode == "delta" else _FullBenchmarkState
     builder = StateGraph(schema)
     builder.add_node("noop", _noop)
     builder.set_entry_point("noop")
@@ -283,7 +301,7 @@ def _sqlite_storage_stats(saver: SqliteSaver, thread_id: str) -> dict[str, int]:
 
 
 def _write_and_read(case: BenchmarkCase, saver: Any, messages: list[BaseMessage]) -> tuple[dict[str, Any], list[AnyMessage]]:
-    graph = _build_graph(case.mode, saver)
+    graph = _build_graph(case.mode, saver, case.snapshot_frequency)
     accessor = CheckpointStateAccessor.bind(graph, saver, mode=case.mode)
     config = _config(case)
     update_latencies: list[float] = []
@@ -312,7 +330,7 @@ def _write_and_read(case: BenchmarkCase, saver: Any, messages: list[BaseMessage]
 
 
 def _cold_read(case: BenchmarkCase, saver: Any) -> tuple[float, list[AnyMessage]]:
-    graph = _build_graph(case.mode, saver)
+    graph = _build_graph(case.mode, saver, case.snapshot_frequency)
     accessor = CheckpointStateAccessor.bind(graph, saver, mode=case.mode)
     gc.collect()
     start = time.perf_counter()
@@ -455,7 +473,7 @@ def _failure_row(case: BenchmarkCase, error: str) -> dict[str, Any]:
 
 
 def _profile_filename(case: BenchmarkCase) -> str:
-    return f"{case.backend}-{case.mode}-updates-{case.update_count}-payload-{case.payload_bytes}-rep-{case.repetition}.prof"
+    return f"{case.backend}-{case.mode}-freq-{case.snapshot_frequency}-updates-{case.update_count}-payload-{case.payload_bytes}-rep-{case.repetition}.prof"
 
 
 def _run_child_case(case: BenchmarkCase, *, timeout_seconds: float, git_sha: str, profile_dir: Path | None = None) -> dict[str, Any]:
@@ -477,6 +495,16 @@ def _build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--backends", default="memory,sqlite", help="Comma-separated backends (default: memory,sqlite)")
     parser.add_argument("--updates", default="10,100", help="Comma-separated message update counts (default: 10,100)")
     parser.add_argument("--payload-bytes", default="128", help="Comma-separated exact message content sizes (default: 128)")
+    parser.add_argument(
+        "--snapshot-frequencies",
+        default=str(PRODUCTION_SNAPSHOT_FREQUENCY),
+        help=(
+            "Comma-separated DeltaChannel snapshot cadences for delta cases "
+            f"(default: {PRODUCTION_SNAPSHOT_FREQUENCY}). Full-mode cases ignore "
+            "the cadence and run once per cell at the production default. "
+            "Sweep 100,250,500,1000 to compare cadence tradeoffs."
+        ),
+    )
     parser.add_argument("--repetitions", type=int, default=3)
     parser.add_argument("--seed", type=int, default=1)
     parser.add_argument("--timeout-seconds", type=float, default=900)
@@ -521,6 +549,7 @@ def main(argv: list[str] | None = None) -> int:
         backends = _parse_choice_csv(args.backends, option="--backends", choices=_BACKENDS)
         updates = _parse_positive_int_csv(args.updates, option="--updates")
         payload_bytes = _parse_positive_int_csv(args.payload_bytes, option="--payload-bytes")
+        snapshot_frequencies = _parse_positive_int_csv(args.snapshot_frequencies, option="--snapshot-frequencies")
         if args.repetitions <= 0:
             raise ValueError("--repetitions must be positive")
         if args.timeout_seconds <= 0:
@@ -537,6 +566,7 @@ def main(argv: list[str] | None = None) -> int:
         payload_bytes=payload_bytes,
         repetitions=args.repetitions,
         seed=args.seed,
+        snapshot_frequencies=snapshot_frequencies,
     )
     cases, skipped = _filter_oversized_pairs(cases, max_bytes=None if args.allow_large_cases else args.max_estimated_full_bytes)
     if skipped:
@@ -552,8 +582,9 @@ def main(argv: list[str] | None = None) -> int:
     git_sha = _resolve_git_sha()
     rows: list[dict[str, Any]] = []
     for index, case in enumerate(cases, start=1):
+        cadence = f" freq={case.snapshot_frequency}" if case.mode == "delta" else ""
         print(
-            f"[{index}/{len(cases)}] {case.backend} {case.mode} updates={case.update_count} payload={case.payload_bytes} repetition={case.repetition}",
+            f"[{index}/{len(cases)}] {case.backend} {case.mode}{cadence} updates={case.update_count} payload={case.payload_bytes} repetition={case.repetition}",
             file=sys.stderr,
         )
         rows.append(_run_child_case(case, timeout_seconds=args.timeout_seconds, git_sha=git_sha, profile_dir=args.profile_dir))

backend/scripts/benchmark/checkpoint/bench_production.py

@@ -28,6 +28,8 @@ from dataclasses import asdict, dataclass
 from pathlib import Path
 from typing import Literal
 
+from deerflow.config.database_config import DEFAULT_CHECKPOINT_SNAPSHOT_FREQUENCY
+
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 import checkpoint_bench_common as common  # noqa: E402
 
@@ -404,7 +406,7 @@ async def _run_phase(case: ProductionCase, saver: Any, timing: _TimingSaver, wor
                 "backend": "sqlite",
                 "sqlite_dir": str(work_dir),
                 "checkpoint_channel_mode": case.mode,
-                "checkpoint_delta_snapshot_frequency": case.snapshot_frequency or 1000,
+                "checkpoint_delta": {"snapshot_frequency": case.snapshot_frequency or DEFAULT_CHECKPOINT_SNAPSHOT_FREQUENCY},
             },
         }
     )

backend/tests/blocking_io/test_dynamic_context_middleware.py

@@ -49,11 +49,11 @@ async def test_abefore_agent_does_not_block_event_loop() -> None:
     # event-loop blocking visible to the Blockbuster gate.
     original_build = mw._build_full_reminder
 
-    def slow_build_reminder():
+    def slow_build_reminder(runtime=None):
         import time
 
         time.sleep(0.05)  # 50ms sync sleep — blocks the thread it runs on
-        return original_build()
+        return original_build(runtime)
 
     with (
         mock.patch.object(mw, "_build_full_reminder", slow_build_reminder),
@@ -114,7 +114,7 @@ async def test_abefore_agent_returns_none_on_timeout() -> None:
     finished = threading.Event()
     journal = mock.MagicMock()
 
-    def blocking_inject(state):
+    def blocking_inject(state, runtime=None):
         started.set()
         release.wait(timeout=2)
         try:
@@ -159,7 +159,7 @@ async def test_abefore_agent_records_checkpointed_memory_on_timeout() -> None:
     journal = mock.MagicMock()
     memory_content = "<memory>checkpoint context</memory>"
 
-    def blocking_inject(state):
+    def blocking_inject(state, runtime=None):
         started.set()
         release.wait(timeout=2)
         try:

backend/tests/blocking_io/test_openviking_memory_backend.py

@@ -0,0 +1,91 @@
+"""Regression anchors: OpenViking async memory methods must not block the loop."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pytest
+from langchain_core.messages import AIMessage, HumanMessage
+
+from deerflow.agents.memory.backends.openviking.models import OpenVikingCommitResult, OpenVikingSearchHit
+from deerflow.agents.memory.backends.openviking.openviking_manager import OpenVikingMemoryManager
+
+
+class _BlockingProbeClient:
+    """Perform real file IO so Blockbuster can detect missing async offload."""
+
+    def __init__(self, probe_path: Path):
+        self._probe_path = probe_path
+
+    def _probe(self) -> None:
+        self._probe_path.write_text("probe", encoding="utf-8")
+
+    def ensure_session(self, identity, session_id) -> None:
+        self._probe()
+
+    def add_messages(self, identity, session_id, messages) -> int:
+        self._probe()
+        return len(messages)
+
+    def commit_session(self, identity, session_id) -> OpenVikingCommitResult:
+        self._probe()
+        return OpenVikingCommitResult(status="accepted", task_id="task-1", archive_uri=None, archived=True)
+
+    def search(
+        self,
+        identity,
+        query: str,
+        *,
+        top_k: int,
+        category: str | None = None,
+        session_id: str | None = None,
+    ) -> list[OpenVikingSearchHit]:
+        self._probe()
+        return [
+            OpenVikingSearchHit(
+                uri="viking://user/memories/preferences/test.md",
+                context_type="memory",
+                category="preferences",
+                score=0.9,
+                abstract="Prefers concise answers.",
+                overview=None,
+                match_reason="",
+            )
+        ]
+
+    def close(self) -> None:
+        pass
+
+
+def _manager(tmp_path: Path) -> OpenVikingMemoryManager:
+    manager = OpenVikingMemoryManager.from_config(
+        {
+            "base_url": "http://openviking:1933",
+            "storage_path": str(tmp_path),
+            "auth_mode": "trusted",
+            "account": "deerflow",
+            "startup_policy": "warn",
+        }
+    )
+    manager._client = _BlockingProbeClient(tmp_path / "probe.txt")  # type: ignore[assignment]
+    return manager
+
+
+@pytest.mark.asyncio
+async def test_async_openviking_operations_do_not_block_event_loop(tmp_path: Path) -> None:
+    manager = _manager(tmp_path)
+    messages: list[Any] = [HumanMessage("hello", id="h1"), AIMessage("hi", id="a1")]
+
+    await manager.aadd("thread-1", messages, user_id="alice")
+    assert await manager.aget_context("alice") == "- [preferences] Prefers concise answers."
+    assert await manager.asearch("answer style", user_id="alice") == [
+        {
+            "id": "viking://user/memories/preferences/test.md",
+            "content": "Prefers concise answers.",
+            "category": "preferences",
+            "confidence": 0.9,
+            "source": "viking://user/memories/preferences/test.md",
+            "score": 0.9,
+        }
+    ]

backend/tests/conftest.py

@@ -101,7 +101,8 @@ def _reset_skill_storage_singleton():
 def _reset_frozen_checkpoint_channel_mode(monkeypatch):
     """Reset the process-global frozen checkpoint channel mode between tests.
 
-    Production treats ``checkpoint_channel_mode`` as restart-required: the
+    Production treats ``checkpoint_channel_mode`` (and the delta
+    ``snapshot_frequency`` frozen alongside it) as restart-required: the
     first client/app freezes it for the process. The test suite builds many
     clients and apps with different modes in one process, so the freeze must
     not leak across tests. Mirrors the per-test ``monkeypatch.setattr``
@@ -110,20 +111,7 @@ def _reset_frozen_checkpoint_channel_mode(monkeypatch):
     from deerflow.runtime import checkpoint_mode
 
     monkeypatch.setattr(checkpoint_mode, "_frozen_checkpoint_channel_mode", None)
-    yield
-
-
-@pytest.fixture(autouse=True)
-def _reset_frozen_delta_snapshot_frequency(monkeypatch):
-    """Reset the process-global frozen delta snapshot frequency between tests.
-
-    Same restart-required semantics as the checkpoint channel mode freeze
-    above: ``make_lead_agent`` freezes it from the app config, and the suite
-    builds many agents with different configs in one process.
-    """
-    from deerflow.agents import thread_state
-
-    monkeypatch.setattr(thread_state, "_frozen_delta_snapshot_frequency", None)
+    monkeypatch.setattr(checkpoint_mode, "_frozen_checkpoint_snapshot_frequency", None)
     yield
 
 

backend/tests/test_aio_sandbox_provider.py

@@ -11,6 +11,7 @@ from unittest.mock import MagicMock, patch
 import pytest
 
 from deerflow.config.paths import Paths, join_host_path
+from deerflow.config.sandbox_config import SandboxConfig
 from deerflow.runtime.user_context import reset_current_user, set_current_user
 
 _LEGACY_COLLIDING_IDENTITIES = (
@@ -18,6 +19,48 @@ _LEGACY_COLLIDING_IDENTITIES = (
     ("user-94361", "thread-94361"),
 )
 
+# ── thread-data mount configuration ─────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    ("sandbox_overrides", "expected"),
+    [
+        ({}, None),
+        ({"thread_data_mounts": True}, True),
+        ({"thread_data_mounts": False}, False),
+    ],
+)
+def test_load_config_preserves_thread_data_mounts_override(sandbox_overrides, expected, monkeypatch):
+    aio_mod = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox_provider")
+    sandbox_config = SandboxConfig(
+        use="deerflow.community.aio_sandbox:AioSandboxProvider",
+        **sandbox_overrides,
+    )
+    app_config = SimpleNamespace(sandbox=sandbox_config, stream_bridge=None)
+    monkeypatch.setattr(aio_mod, "get_app_config", lambda: app_config)
+    provider = aio_mod.AioSandboxProvider.__new__(aio_mod.AioSandboxProvider)
+
+    assert provider._load_config()["thread_data_mounts"] is expected
+
+
+@pytest.mark.parametrize(
+    ("backend_is_local", "override", "expected"),
+    [
+        (True, None, True),
+        (False, None, False),
+        (True, False, False),
+        (False, True, True),
+    ],
+)
+def test_thread_data_mounts_override_precedes_backend_detection(backend_is_local, override, expected):
+    aio_mod = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox_provider")
+    provider = aio_mod.AioSandboxProvider.__new__(aio_mod.AioSandboxProvider)
+    provider._config = {} if override is None else {"thread_data_mounts": override}
+    provider._backend = object.__new__(aio_mod.LocalContainerBackend) if backend_is_local else object()
+
+    assert provider.uses_thread_data_mounts is expected
+
+
 # ── ensure_thread_dirs ───────────────────────────────────────────────────────
 
 
@@ -518,6 +561,7 @@ def test_remote_backend_create_forwards_effective_user_id(monkeypatch):
         "user_id": "user-7",
         "include_legacy_skills": True,
         "provision_lark_cli_runtime": False,
+        "provision_lark_cli_broker": False,
     }
 
 
@@ -562,18 +606,52 @@ def test_create_sandbox_requests_runtime_when_lark_installed(tmp_path, monkeypat
 
     captured: dict = {}
 
-    def _create(thread_id, sandbox_id, *, extra_mounts=None, user_id=None, provision_lark_cli_runtime=False):
+    def _create(thread_id, sandbox_id, *, extra_mounts=None, user_id=None, provision_lark_cli_runtime=False, provision_lark_cli_broker=False):
         captured["provision_lark_cli_runtime"] = provision_lark_cli_runtime
+        captured["provision_lark_cli_broker"] = provision_lark_cli_broker
         return aio_mod.SandboxInfo(sandbox_id=sandbox_id, sandbox_url="http://sandbox")
 
     provider._backend = SimpleNamespace(create=_create, destroy=MagicMock(), discover=MagicMock(return_value=None))
     monkeypatch.setattr(aio_mod, "wait_for_sandbox_ready", lambda *_a, **_k: True)
     monkeypatch.setattr(provider, "_get_extra_mounts", lambda *_a, **_k: [])
     monkeypatch.setattr(aio_mod.AioSandboxProvider, "_lark_integration_active", staticmethod(lambda user_id=None: True))
+    monkeypatch.setattr(aio_mod.AioSandboxProvider, "_lark_broker_active", staticmethod(lambda user_id=None: False))
     monkeypatch.setattr(provider, "_register_created_sandbox", lambda *a, **k: "sandbox-lark")
 
     provider._create_sandbox("thread-lark", "sandbox-lark", user_id="alice")
     assert captured["provision_lark_cli_runtime"] is True
+    assert captured["provision_lark_cli_broker"] is False
+
+
+def test_create_sandbox_requests_broker_when_active(tmp_path, monkeypatch):
+    """Broker mode (Pattern B) is requested when the provisioner reports it."""
+    aio_mod = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox_provider")
+    provider = _make_provider(tmp_path)
+    provider._config = {"replicas": 3}
+    provider._thread_locks = {}
+    provider._warm_pool = {}
+    provider._sandbox_infos = {}
+    provider._thread_sandboxes = {}
+    provider._last_activity = {}
+    provider._lock = aio_mod.threading.Lock()
+
+    captured: dict = {}
+
+    def _create(thread_id, sandbox_id, *, extra_mounts=None, user_id=None, provision_lark_cli_runtime=False, provision_lark_cli_broker=False):
+        captured["provision_lark_cli_runtime"] = provision_lark_cli_runtime
+        captured["provision_lark_cli_broker"] = provision_lark_cli_broker
+        return aio_mod.SandboxInfo(sandbox_id=sandbox_id, sandbox_url="http://sandbox")
+
+    provider._backend = SimpleNamespace(create=_create, destroy=MagicMock(), discover=MagicMock(return_value=None))
+    monkeypatch.setattr(aio_mod, "wait_for_sandbox_ready", lambda *_a, **_k: True)
+    monkeypatch.setattr(provider, "_get_extra_mounts", lambda *_a, **_k: [])
+    monkeypatch.setattr(aio_mod.AioSandboxProvider, "_lark_integration_active", staticmethod(lambda user_id=None: True))
+    monkeypatch.setattr(aio_mod.AioSandboxProvider, "_lark_broker_active", staticmethod(lambda user_id=None: True))
+    monkeypatch.setattr(provider, "_register_created_sandbox", lambda *a, **k: "sandbox-broker")
+
+    provider._create_sandbox("thread-broker", "sandbox-broker", user_id="alice")
+    assert captured["provision_lark_cli_runtime"] is True
+    assert captured["provision_lark_cli_broker"] is True
 
 
 def test_create_sandbox_skips_runtime_when_lark_absent(tmp_path, monkeypatch):
@@ -590,18 +668,21 @@ def test_create_sandbox_skips_runtime_when_lark_absent(tmp_path, monkeypatch):
 
     captured: dict = {}
 
-    def _create(thread_id, sandbox_id, *, extra_mounts=None, user_id=None, provision_lark_cli_runtime=False):
+    def _create(thread_id, sandbox_id, *, extra_mounts=None, user_id=None, provision_lark_cli_runtime=False, provision_lark_cli_broker=False):
         captured["provision_lark_cli_runtime"] = provision_lark_cli_runtime
+        captured["provision_lark_cli_broker"] = provision_lark_cli_broker
         return aio_mod.SandboxInfo(sandbox_id=sandbox_id, sandbox_url="http://sandbox")
 
     provider._backend = SimpleNamespace(create=_create, destroy=MagicMock(), discover=MagicMock(return_value=None))
     monkeypatch.setattr(aio_mod, "wait_for_sandbox_ready", lambda *_a, **_k: True)
     monkeypatch.setattr(provider, "_get_extra_mounts", lambda *_a, **_k: [])
     monkeypatch.setattr(aio_mod.AioSandboxProvider, "_lark_integration_active", staticmethod(lambda user_id=None: False))
+    monkeypatch.setattr(aio_mod.AioSandboxProvider, "_lark_broker_active", staticmethod(lambda user_id=None: False))
     monkeypatch.setattr(provider, "_register_created_sandbox", lambda *a, **k: "sandbox-nolark")
 
     provider._create_sandbox("thread-nolark", "sandbox-nolark", user_id="alice")
     assert captured["provision_lark_cli_runtime"] is False
+    assert captured["provision_lark_cli_broker"] is False
 
 
 # ── Sandbox client teardown (#2872) ──────────────────────────────────────────

backend/tests/test_app_config_reload.py

@@ -119,6 +119,70 @@ def test_checkpoint_channel_mode_rejects_unknown_value() -> None:
         DatabaseConfig(checkpoint_channel_mode="auto")
 
 
+def test_checkpoint_delta_defaults_to_production_cadence() -> None:
+    assert DatabaseConfig().checkpoint_delta.snapshot_frequency == 10
+
+
+def test_checkpoint_delta_accepts_custom_snapshot_frequency() -> None:
+    assert DatabaseConfig(checkpoint_delta={"snapshot_frequency": 250}).checkpoint_delta.snapshot_frequency == 250
+
+
+@pytest.mark.parametrize("value", [0, -1])
+def test_checkpoint_delta_rejects_non_positive_snapshot_frequency(value: int) -> None:
+    with pytest.raises(ValidationError):
+        DatabaseConfig(checkpoint_delta={"snapshot_frequency": value})
+
+
+def test_legacy_snapshot_frequency_maps_to_nested_key(caplog: pytest.LogCaptureFixture) -> None:
+    with caplog.at_level("WARNING"):
+        config = DatabaseConfig(checkpoint_delta_snapshot_frequency=1000)
+    assert config.checkpoint_delta.snapshot_frequency == 1000
+    assert "checkpoint_delta_snapshot_frequency is deprecated" in caplog.text
+
+
+def test_nested_snapshot_frequency_wins_over_legacy_key(caplog: pytest.LogCaptureFixture) -> None:
+    with caplog.at_level("WARNING"):
+        config = DatabaseConfig(
+            checkpoint_delta_snapshot_frequency=1000,
+            checkpoint_delta={"snapshot_frequency": 250},
+        )
+    assert config.checkpoint_delta.snapshot_frequency == 250
+    assert "the nested key wins" in caplog.text
+
+
+@pytest.mark.parametrize("value", [0, -1])
+def test_legacy_snapshot_frequency_rejects_non_positive_value(value: int) -> None:
+    with pytest.raises(ValidationError):
+        DatabaseConfig(checkpoint_delta_snapshot_frequency=value)
+
+
+def test_checkpoint_graph_cache_defaults() -> None:
+    assert DatabaseConfig().checkpoint_graph_cache.accessor_graph_max == 64
+
+
+def test_checkpoint_graph_cache_accepts_custom_cap() -> None:
+    assert DatabaseConfig(checkpoint_graph_cache={"accessor_graph_max": 8}).checkpoint_graph_cache.accessor_graph_max == 8
+
+
+@pytest.mark.parametrize("value", [0, -1])
+def test_checkpoint_graph_cache_rejects_non_positive_cap(value: int) -> None:
+    with pytest.raises(ValidationError):
+        DatabaseConfig(checkpoint_graph_cache={"accessor_graph_max": value})
+
+
+def test_resolve_checkpoint_graph_cache_max_tolerates_stub_configs() -> None:
+    from types import SimpleNamespace
+    from unittest.mock import MagicMock
+
+    from deerflow.config.database_config import resolve_checkpoint_graph_cache_max
+
+    assert resolve_checkpoint_graph_cache_max(None, "accessor_graph_max", 64) == 64
+    assert resolve_checkpoint_graph_cache_max(SimpleNamespace(), "accessor_graph_max", 64) == 64
+    assert resolve_checkpoint_graph_cache_max(MagicMock(), "accessor_graph_max", 64) == 64
+    stub = SimpleNamespace(checkpoint_graph_cache=SimpleNamespace(accessor_graph_max=3))
+    assert resolve_checkpoint_graph_cache_max(stub, "accessor_graph_max", 64) == 3
+
+
 def test_config_example_does_not_enable_empty_extensions_block_by_default():
     config_example_path = Path(__file__).resolve().parents[2] / "config.example.yaml"
 

backend/tests/test_bench_checkpoint_channels.py

@@ -86,6 +86,28 @@ def test_oversized_filter_skips_full_and_delta_as_a_comparable_pair() -> None:
     assert {(case.update_count, case.mode) for case in skipped} == {(100, "full"), (100, "delta")}
 
 
+def test_oversized_filter_applies_full_cap_to_every_swept_delta_cadence() -> None:
+    cases = bench._expand_cases(
+        modes=["full", "delta"],
+        backends=["memory"],
+        update_counts=[100],
+        payload_bytes=[128],
+        repetitions=1,
+        seed=1,
+        snapshot_frequencies=[1, 250, 1000],
+    )
+
+    kept, skipped = bench._filter_oversized_pairs(cases, max_bytes=100_000)
+
+    assert kept == []
+    assert {(case.mode, case.snapshot_frequency) for case in skipped} == {
+        ("full", bench.PRODUCTION_SNAPSHOT_FREQUENCY),
+        ("delta", 1),
+        ("delta", 250),
+        ("delta", 1000),
+    }
+
+
 def test_oversized_filter_does_not_suppress_a_delta_only_diagnostic() -> None:
     case = bench.BenchmarkCase(
         mode="delta",
@@ -106,6 +128,55 @@ def test_help_explains_delta_only_runs_bypass_full_payload_cap() -> None:
     assert "delta-only" in bench._build_parser().format_help()
 
 
+def test_expand_cases_sweeps_delta_frequencies_without_duplicating_full_cases() -> None:
+    cases = bench._expand_cases(
+        modes=["full", "delta"],
+        backends=["memory"],
+        update_counts=[10],
+        payload_bytes=[128],
+        repetitions=1,
+        seed=1,
+        snapshot_frequencies=[100, 250, 500, 1000],
+    )
+
+    full_cases = [case for case in cases if case.mode == "full"]
+    delta_cases = [case for case in cases if case.mode == "delta"]
+    assert len(full_cases) == 1
+    assert full_cases[0].snapshot_frequency == bench.PRODUCTION_SNAPSHOT_FREQUENCY
+    assert sorted(case.snapshot_frequency for case in delta_cases) == [100, 250, 500, 1000]
+
+
+def test_case_rejects_non_positive_snapshot_frequency() -> None:
+    with pytest.raises(ValueError, match="snapshot_frequency"):
+        bench.BenchmarkCase(
+            mode="delta",
+            backend="memory",
+            update_count=4,
+            payload_bytes=64,
+            repetition=0,
+            seed=1,
+            snapshot_frequency=0,
+        )
+
+
+def test_memory_smoke_case_materializes_at_low_snapshot_frequency(tmp_path: Path) -> None:
+    case = bench.BenchmarkCase(
+        mode="delta",
+        backend="memory",
+        update_count=6,
+        payload_bytes=64,
+        repetition=0,
+        seed=1,
+        snapshot_frequency=2,
+    )
+
+    row = bench._run_case(case, work_dir=tmp_path)
+
+    assert row["success"] is True
+    assert row["snapshot_frequency"] == 2
+    assert row["actual_message_count"] == 6
+
+
 def test_cross_mode_validation_rejects_materialized_state_mismatch() -> None:
     rows = [
         {

backend/tests/test_bench_checkpoint_production.py

@@ -152,13 +152,12 @@ def test_timing_saver_records_cumulative_ms() -> None:
 
 
 def _reset_checkpoint_freezes(monkeypatch: pytest.MonkeyPatch) -> None:
-    from deerflow.agents import thread_state
     from deerflow.config.app_config import reset_app_config
     from deerflow.runtime import checkpoint_mode
 
     reset_app_config()
     monkeypatch.setattr(checkpoint_mode, "_frozen_checkpoint_channel_mode", None)
-    monkeypatch.setattr(thread_state, "_frozen_delta_snapshot_frequency", None)
+    monkeypatch.setattr(checkpoint_mode, "_frozen_checkpoint_snapshot_frequency", None)
 
 
 @pytest.mark.parametrize("mode,frequency", [("full", None), ("delta", 10)])
@@ -212,7 +211,7 @@ def test_run_case_fails_when_warm_cache_is_not_hit(tmp_path, monkeypatch) -> Non
     monkeypatch.setattr(
         gateway_services,
         "_state_accessor_graph",
-        lambda agent_factory, assistant_id, mode, config: agent_factory(config=config),
+        lambda agent_factory, assistant_id, mode, snapshot_frequency, config: agent_factory(config=config),
     )
     case = bench.ProductionCase(
         mode="full",

backend/tests/test_check_script.py

@@ -1,20 +1,27 @@
 from __future__ import annotations
 
 import importlib.util
+import subprocess
 from pathlib import Path
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
 CHECK_SCRIPT_PATH = REPO_ROOT / "scripts" / "check.py"
+PNPM_SCRIPT_PATH = REPO_ROOT / "scripts" / "pnpm.py"
 
 
-spec = importlib.util.spec_from_file_location("deerflow_check_script", CHECK_SCRIPT_PATH)
-assert spec is not None
-assert spec.loader is not None
-check_script = importlib.util.module_from_spec(spec)
-spec.loader.exec_module(check_script)
+def _load_script(path: Path, name: str):
+    assert path.exists(), f"{path} must exist"
+    spec = importlib.util.spec_from_file_location(name, path)
+    assert spec is not None
+    assert spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
 
 
 def test_find_pnpm_command_prefers_resolved_executable(monkeypatch):
+    pnpm_script = _load_script(PNPM_SCRIPT_PATH, "deerflow_pnpm_script_direct")
+
     def fake_which(name: str) -> str | None:
         if name == "pnpm":
             return r"C:\Users\tester\AppData\Roaming\npm\pnpm.CMD"
@@ -22,26 +29,30 @@ def test_find_pnpm_command_prefers_resolved_executable(monkeypatch):
             return r"C:\Users\tester\AppData\Roaming\npm\pnpm.cmd"
         return None
 
-    monkeypatch.setattr(check_script.shutil, "which", fake_which)
+    monkeypatch.setattr(pnpm_script.shutil, "which", fake_which)
 
-    assert check_script.find_pnpm_command() == [r"C:\Users\tester\AppData\Roaming\npm\pnpm.CMD"]
+    assert pnpm_script.find_pnpm_command() == [r"C:\Users\tester\AppData\Roaming\npm\pnpm.CMD"]
 
 
 def test_find_pnpm_command_falls_back_to_corepack(monkeypatch):
+    pnpm_script = _load_script(PNPM_SCRIPT_PATH, "deerflow_pnpm_script_corepack")
+
     def fake_which(name: str) -> str | None:
         if name == "corepack":
             return r"C:\Program Files\nodejs\corepack.exe"
         return None
 
-    monkeypatch.setattr(check_script.shutil, "which", fake_which)
+    monkeypatch.setattr(pnpm_script.shutil, "which", fake_which)
 
-    assert check_script.find_pnpm_command() == [
+    assert pnpm_script.find_pnpm_command() == [
         r"C:\Program Files\nodejs\corepack.exe",
         "pnpm",
     ]
 
 
 def test_find_pnpm_command_falls_back_to_corepack_cmd(monkeypatch):
+    pnpm_script = _load_script(PNPM_SCRIPT_PATH, "deerflow_pnpm_script_corepack_cmd")
+
     def fake_which(name: str) -> str | None:
         if name == "corepack":
             return None
@@ -49,9 +60,77 @@ def test_find_pnpm_command_falls_back_to_corepack_cmd(monkeypatch):
             return r"C:\Program Files\nodejs\corepack.cmd"
         return None
 
-    monkeypatch.setattr(check_script.shutil, "which", fake_which)
+    monkeypatch.setattr(pnpm_script.shutil, "which", fake_which)
 
-    assert check_script.find_pnpm_command() == [
+    assert pnpm_script.find_pnpm_command() == [
         r"C:\Program Files\nodejs\corepack.cmd",
         "pnpm",
     ]
+
+
+def test_check_script_uses_shared_pnpm_runner():
+    check_script = CHECK_SCRIPT_PATH.read_text(encoding="utf-8")
+
+    assert 'Path(__file__).with_name("pnpm.py")' in check_script
+
+
+def test_check_script_preserves_runner_failure_diagnostics(monkeypatch):
+    check_script = _load_script(CHECK_SCRIPT_PATH, "deerflow_check_script_failure")
+    call_kwargs = {}
+
+    def fake_run(*args, **kwargs):
+        call_kwargs.update(kwargs)
+        return subprocess.CompletedProcess(
+            args=["python", "pnpm.py", "-v"],
+            returncode=42,
+            stdout="partial pnpm output\n",
+            stderr="Error: pnpm command failed with exit status 42.\n",
+        )
+
+    monkeypatch.setattr(check_script.subprocess, "run", fake_run)
+
+    assert check_script.run_pnpm_version() == (
+        None,
+        False,
+        "Error: pnpm command failed with exit status 42.\npartial pnpm output",
+    )
+    assert call_kwargs["cwd"] == REPO_ROOT / "frontend"
+
+
+def test_check_script_preserves_corepack_resolution_hint(monkeypatch):
+    check_script = _load_script(CHECK_SCRIPT_PATH, "deerflow_check_script_corepack")
+
+    def fake_run(*args, **kwargs):
+        return subprocess.CompletedProcess(
+            args=["python", "pnpm.py", "-v"],
+            returncode=0,
+            stdout="10.26.2\n",
+            stderr="Using pnpm via Corepack.\n",
+        )
+
+    monkeypatch.setattr(check_script.subprocess, "run", fake_run)
+
+    assert check_script.run_pnpm_version() == ("10.26.2", True, None)
+
+
+def test_check_status_labels_corepack_fallback(monkeypatch, capsys):
+    check_script = _load_script(CHECK_SCRIPT_PATH, "deerflow_check_script_status")
+
+    monkeypatch.setattr(check_script.shutil, "which", lambda name: f"/fake/{name}")
+    monkeypatch.setattr(
+        check_script,
+        "run_command",
+        lambda command: {
+            "node": "v22.0.0",
+            "uv": "uv 0.11.31",
+            "nginx": "nginx/1.31.3",
+        }[command[0]],
+    )
+    monkeypatch.setattr(
+        check_script,
+        "run_pnpm_version",
+        lambda: ("10.26.2", True, None),
+    )
+
+    assert check_script.main() == 0
+    assert "OK pnpm 10.26.2 (via Corepack)" in capsys.readouterr().out

backend/tests/test_checkpoint_lineage.py

@@ -0,0 +1,149 @@
+"""Replay-base resolution rules shared by regenerate, edit replay and branching.
+
+The load-bearing rule here is that a replay base must be a checkpoint the
+thread was actually at rest in. A mid-run checkpoint still owns the interrupted
+node's pending writes, so resuming from it replays them.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
+
+from app.gateway.checkpoint_lineage import (
+    CheckpointParentMissingError,
+    find_checkpoint_before_message,
+    find_checkpoint_before_message_chronologically,
+)
+
+THREAD_ID = "thread-1"
+
+
+def _snapshot(checkpoint_id: str, messages: list[object], *, next_tasks: tuple[str, ...] = (), parent_id: str | None = None):
+    parent_config = None
+    if parent_id is not None:
+        parent_config = {"configurable": {"thread_id": THREAD_ID, "checkpoint_ns": "", "checkpoint_id": parent_id}}
+    return SimpleNamespace(
+        values={"messages": messages},
+        config={
+            "configurable": {
+                "thread_id": THREAD_ID,
+                "checkpoint_ns": "",
+                "checkpoint_id": checkpoint_id,
+                "checkpoint_map": None,
+            }
+        },
+        metadata={},
+        parent_config=parent_config,
+        next=next_tasks,
+    )
+
+
+class _Accessor:
+    """Minimal accessor over a fixed snapshot list, keyed by checkpoint id."""
+
+    def __init__(self, snapshots: list[object]) -> None:
+        self.snapshots = snapshots
+
+    async def aget(self, config):
+        checkpoint_id = config.get("configurable", {}).get("checkpoint_id")
+        return next(
+            (item for item in self.snapshots if item.config["configurable"]["checkpoint_id"] == checkpoint_id),
+            SimpleNamespace(values={}, config={}, metadata=None, parent_config=None, next=()),
+        )
+
+
+def _first_turn_history() -> list[object]:
+    """Newest-first history of a thread whose only turn is still its first.
+
+    ``DynamicContextMiddleware`` swaps the first user message's id mid-run:
+    the injected reminder takes ``{id}`` and the real user message becomes
+    ``{id}__user``. So the pre-injection checkpoints hold the very same prompt
+    under an id the replay-base lookup does not recognise (#4531).
+    """
+    system = SystemMessage(id="h1", content="<system-reminder>date</system-reminder>")
+    swapped_human = HumanMessage(id="h1__user", content="question")
+    raw_human = HumanMessage(id="h1", content="question")
+    ai = AIMessage(id="ai-1", content="answer")
+    return [
+        _snapshot("ckpt-head", [system, swapped_human, ai], parent_id="ckpt-after-inject"),
+        _snapshot("ckpt-after-inject", [system, swapped_human], next_tasks=("LoopDetectionMiddleware.before_agent",), parent_id="ckpt-mid"),
+        _snapshot("ckpt-mid", [raw_human], next_tasks=("DynamicContextMiddleware.before_agent",), parent_id="ckpt-input"),
+        _snapshot("ckpt-input", [], next_tasks=("__start__",), parent_id="ckpt-empty"),
+        _snapshot("ckpt-empty", []),
+    ]
+
+
+def test_chronological_scan_skips_checkpoints_with_pending_tasks():
+    history = _first_turn_history()
+
+    base, found = find_checkpoint_before_message_chronologically(history, "h1__user")
+
+    assert found is True
+    assert base is not None
+    assert base.config["configurable"]["checkpoint_id"] == "ckpt-empty"
+
+
+def test_lineage_walk_skips_checkpoints_with_pending_tasks():
+    history = _first_turn_history()
+    accessor = _Accessor(history)
+
+    base = asyncio.run(find_checkpoint_before_message(accessor, history[0], "h1__user", max_depth=10))
+
+    assert base.config["configurable"]["checkpoint_id"] == "ckpt-empty"
+
+
+def test_chronological_scan_prefers_the_previous_turn_boundary():
+    """A later turn resolves to the previous run's settled tail, not its input checkpoint."""
+    human1 = HumanMessage(id="h1__user", content="first")
+    ai1 = AIMessage(id="ai-1", content="first answer")
+    human2 = HumanMessage(id="h2", content="second")
+    history = [
+        _snapshot("ckpt-turn2-head", [human1, ai1, human2, AIMessage(id="ai-2", content="second answer")]),
+        _snapshot("ckpt-turn2-mid", [human1, ai1, human2], next_tasks=("model",)),
+        _snapshot("ckpt-turn2-input", [human1, ai1], next_tasks=("__start__",)),
+        _snapshot("ckpt-turn1-tail", [human1, ai1]),
+    ]
+
+    base, found = find_checkpoint_before_message_chronologically(history, "h2")
+
+    assert found is True
+    assert base.config["configurable"]["checkpoint_id"] == "ckpt-turn1-tail"
+
+
+def test_lineage_walk_reports_missing_parent_when_no_settled_ancestor_exists():
+    """Fail closed rather than fork a mid-run checkpoint."""
+    human = HumanMessage(id="h1__user", content="question")
+    history = [
+        _snapshot("ckpt-head", [human, AIMessage(id="ai-1", content="answer")], parent_id="ckpt-mid"),
+        _snapshot("ckpt-mid", [HumanMessage(id="h1", content="question")], next_tasks=("DynamicContextMiddleware.before_agent",)),
+    ]
+    accessor = _Accessor(history)
+
+    with pytest.raises(CheckpointParentMissingError):
+        asyncio.run(find_checkpoint_before_message(accessor, history[0], "h1__user", max_depth=10))
+
+
+def test_unknown_pending_tasks_do_not_block_selection():
+    """Raw full-mode reads cannot derive tasks; absence of evidence stays permissive."""
+    human = HumanMessage(id="h1", content="question")
+    history = [
+        SimpleNamespace(
+            values={"messages": [human]},
+            config={"configurable": {"thread_id": THREAD_ID, "checkpoint_ns": "", "checkpoint_id": "ckpt-head"}},
+            metadata={},
+        ),
+        SimpleNamespace(
+            values={"messages": []},
+            config={"configurable": {"thread_id": THREAD_ID, "checkpoint_ns": "", "checkpoint_id": "ckpt-base"}},
+            metadata={},
+        ),
+    ]
+
+    base, found = find_checkpoint_before_message_chronologically(history, "h1")
+
+    assert found is True
+    assert base.config["configurable"]["checkpoint_id"] == "ckpt-base"

backend/tests/test_checkpoint_mode.py

@@ -23,6 +23,41 @@ def test_process_mode_change_requires_restart(monkeypatch: pytest.MonkeyPatch) -
         checkpoint_mode.freeze_checkpoint_channel_mode("delta")
 
 
+def test_process_snapshot_frequency_change_requires_restart(monkeypatch: pytest.MonkeyPatch) -> None:
+    from deerflow.runtime import checkpoint_mode
+
+    monkeypatch.setattr(checkpoint_mode, "_frozen_checkpoint_snapshot_frequency", None)
+    assert checkpoint_mode.freeze_checkpoint_snapshot_frequency(250) == 250
+    assert checkpoint_mode.frozen_checkpoint_snapshot_frequency() == 250
+    assert checkpoint_mode.freeze_checkpoint_snapshot_frequency(250) == 250
+    with pytest.raises(checkpoint_mode.CheckpointModeReconfigurationError, match="restart"):
+        checkpoint_mode.freeze_checkpoint_snapshot_frequency(500)
+
+
+@pytest.mark.parametrize("snapshot_frequency", [0, -1])
+def test_process_snapshot_frequency_must_be_positive(
+    monkeypatch: pytest.MonkeyPatch,
+    snapshot_frequency: int,
+) -> None:
+    from deerflow.runtime import checkpoint_mode
+
+    monkeypatch.setattr(checkpoint_mode, "_frozen_checkpoint_snapshot_frequency", None)
+    with pytest.raises(ValueError, match="snapshot frequency must be positive"):
+        checkpoint_mode.freeze_checkpoint_snapshot_frequency(snapshot_frequency)
+    assert checkpoint_mode.frozen_checkpoint_snapshot_frequency() is None
+
+
+def test_resolve_snapshot_frequency_prefers_explicit_then_frozen_then_default(monkeypatch: pytest.MonkeyPatch) -> None:
+    from deerflow.runtime import checkpoint_mode
+
+    monkeypatch.setattr(checkpoint_mode, "_frozen_checkpoint_snapshot_frequency", None)
+    assert checkpoint_mode.resolve_checkpoint_snapshot_frequency() == 10
+    assert checkpoint_mode.resolve_checkpoint_snapshot_frequency(100) == 100
+    monkeypatch.setattr(checkpoint_mode, "_frozen_checkpoint_snapshot_frequency", 250)
+    assert checkpoint_mode.resolve_checkpoint_snapshot_frequency() == 250
+    assert checkpoint_mode.resolve_checkpoint_snapshot_frequency(100) == 100
+
+
 def _config() -> dict:
     return {"configurable": {"thread_id": "thread-1", "checkpoint_ns": ""}}
 
@@ -153,6 +188,49 @@ def test_yaml_mode_change_is_rejected_when_graph_is_reconstructed(tmp_path, monk
         reset_app_config()
 
 
+def test_yaml_snapshot_frequency_change_is_rejected_when_graph_is_reconstructed(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
+    from deerflow.agents.lead_agent import agent as lead_agent
+    from deerflow.config.app_config import reset_app_config
+    from deerflow.runtime import checkpoint_mode
+
+    config_path = tmp_path / "config.yaml"
+
+    def write_config(snapshot_frequency: int) -> None:
+        config_path.write_text(
+            "\n".join(
+                (
+                    "sandbox:",
+                    "  use: deerflow.sandbox.local.provider:LocalSandboxProvider",
+                    "database:",
+                    "  checkpoint_channel_mode: delta",
+                    "  checkpoint_delta:",
+                    f"    snapshot_frequency: {snapshot_frequency}",
+                )
+            )
+            + "\n",
+            encoding="utf-8",
+        )
+
+    write_config(250)
+    monkeypatch.setenv("DEER_FLOW_CONFIG_PATH", str(config_path))
+    monkeypatch.setattr(checkpoint_mode, "_frozen_checkpoint_channel_mode", None)
+    monkeypatch.setattr(checkpoint_mode, "_frozen_checkpoint_snapshot_frequency", None)
+    monkeypatch.setattr(lead_agent, "_make_lead_agent", lambda config, *, app_config: object())
+    reset_app_config()
+    try:
+        lead_agent.make_lead_agent({"configurable": {}})
+        assert checkpoint_mode.frozen_checkpoint_snapshot_frequency() == 250
+
+        write_config(500)
+        future_mtime = config_path.stat().st_mtime + 5
+        os.utime(config_path, (future_mtime, future_mtime))
+
+        with pytest.raises(checkpoint_mode.CheckpointModeReconfigurationError, match="restart"):
+            lead_agent.make_lead_agent({"configurable": {}})
+    finally:
+        reset_app_config()
+
+
 @pytest.mark.asyncio
 async def test_gateway_runtime_rejects_mode_different_from_frozen_process(
     monkeypatch: pytest.MonkeyPatch,
@@ -252,14 +330,14 @@ def test_direct_langgraph_request_cannot_select_delta_in_full_process(
 def test_make_lead_agent_freezes_delta_snapshot_frequency_from_app_config(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    from deerflow.agents import thread_state
     from deerflow.agents.lead_agent import agent as lead_agent
     from deerflow.config.app_config import AppConfig
+    from deerflow.runtime import checkpoint_mode
 
     app_config = AppConfig.model_validate(
         {
             "sandbox": {"use": "deerflow.sandbox.local.provider:LocalSandboxProvider"},
-            "database": {"checkpoint_delta_snapshot_frequency": 7},
+            "database": {"checkpoint_delta": {"snapshot_frequency": 7}},
         }
     )
     monkeypatch.setattr(lead_agent, "get_app_config", lambda: app_config)
@@ -271,7 +349,7 @@ def test_make_lead_agent_freezes_delta_snapshot_frequency_from_app_config(
 
     lead_agent.make_lead_agent({"configurable": {}})
 
-    assert thread_state._frozen_delta_snapshot_frequency == 7
+    assert checkpoint_mode.frozen_checkpoint_snapshot_frequency() == 7
 
 
 def test_gateway_runtime_app_config_can_supply_its_frozen_internal_mode(

backend/tests/test_checkpoint_state.py

@@ -82,6 +82,19 @@ def _assert_delta_config_is_copied(original: dict[str, Any], forwarded: dict[str
     assert forwarded["metadata"][CHECKPOINT_MODE_METADATA_KEY] == "delta"
 
 
+def test_mutation_graph_bakes_snapshot_frequency_into_fallback_schema() -> None:
+    from langgraph.channels import DeltaChannel
+
+    from deerflow.runtime.checkpoint_state import build_state_mutation_graph
+
+    default_graph = build_state_mutation_graph("compact", "delta")
+    assert isinstance(default_graph.channels["messages"], DeltaChannel)
+    assert default_graph.channels["messages"].snapshot_frequency == 10
+
+    low_cadence = build_state_mutation_graph("compact", "delta", snapshot_frequency=250)
+    assert low_cadence.channels["messages"].snapshot_frequency == 250
+
+
 def test_sync_accessor_binds_persistence_guards_operations_and_preserves_input() -> None:
     graph = FakeGraph()
     saver = FakeCheckpointer()

backend/tests/test_checkpointer.py

@@ -1015,7 +1015,7 @@ class TestClientCheckpointerFallback:
         model_mock = MagicMock()
         config_mock = MagicMock()
         config_mock.models = [model_mock]
-        config_mock.database.checkpoint_delta_snapshot_frequency = 1000
+        config_mock.database.checkpoint_delta.snapshot_frequency = 10
         config_mock.get_model_config.return_value = MagicMock(supports_vision=False)
         config_mock.checkpointer = None
 
@@ -1055,7 +1055,7 @@ class TestClientCheckpointerFallback:
         model_mock = MagicMock()
         config_mock = MagicMock()
         config_mock.models = [model_mock]
-        config_mock.database.checkpoint_delta_snapshot_frequency = 1000
+        config_mock.database.checkpoint_delta.snapshot_frequency = 10
         config_mock.get_model_config.return_value = MagicMock(supports_vision=False)
         config_mock.checkpointer = None
 

backend/tests/test_clarification_middleware.py

@@ -173,6 +173,38 @@ class TestHumanInputPayload:
             {"id": "option-4", "label": "None", "value": "None"},
         ]
 
+    def test_payload_flattens_xml_parsed_dict_options(self, middleware):
+        payload = middleware._build_human_input_payload(
+            {
+                "question": "How should the document structure change?",
+                "clarification_type": "approach_choice",
+                "options": {
+                    "item": {
+                        "item": "Move the system section earlier</item>",
+                        "$text": "Merge it into the shared patterns section</item>",
+                    },
+                    "$text": "Add a standalone section</item>",
+                },
+            },
+            tool_call_id="call-abc",
+            request_id="clarification:call-abc",
+        )
+
+        assert payload["options"] == [
+            {"id": "option-1", "label": "Move the system section earlier", "value": "Move the system section earlier"},
+            {
+                "id": "option-2",
+                "label": "Merge it into the shared patterns section",
+                "value": "Merge it into the shared patterns section",
+            },
+            {"id": "option-3", "label": "Add a standalone section", "value": "Add a standalone section"},
+        ]
+
+    def test_dict_options_recursively_flatten_string_and_number_values(self, middleware):
+        options = {"item": [["First</item>"], {"$text": 2}], "$text": None}
+
+        assert middleware._normalize_options(options) == ["First", "2"]
+
     def test_payload_with_plain_string_option(self, middleware):
         payload = middleware._build_human_input_payload(
             {

backend/tests/test_client.py

@@ -52,7 +52,7 @@ def mock_app_config():
     config.skills.container_path = "/mnt/skills"
     config.tool_search.enabled = False
     config.database.checkpoint_channel_mode = "full"
-    config.database.checkpoint_delta_snapshot_frequency = 1000
+    config.database.checkpoint_delta.snapshot_frequency = 10
     config.authorization = AuthorizationConfig(enabled=False)
     return config
 
@@ -154,7 +154,7 @@ class TestClientInit:
         from deerflow.agents import thread_state
 
         mock_app_config.database.checkpoint_channel_mode = "delta"
-        mock_app_config.database.checkpoint_delta_snapshot_frequency = 7
+        mock_app_config.database.checkpoint_delta.snapshot_frequency = 7
         with patch("deerflow.client.get_app_config", return_value=mock_app_config):
             DeerFlowClient()
 
@@ -1334,7 +1334,7 @@ class TestEnsureAgent:
         """_ensure_agent does not recreate if config key unchanged."""
         mock_agent = MagicMock()
         client._agent = mock_agent
-        client._agent_config_key = (None, True, False, False, None, None, None, None, "full", None)
+        client._agent_config_key = (None, True, False, False, None, None, None, None, "full", 10, None)
 
         config = client._get_runnable_config("t1")
         client._ensure_agent(config)

backend/tests/test_deermem_self_contained.py

@@ -133,6 +133,46 @@ def test_zero_config_defaults_run_non_llm_ops(deermem_data_dir):
     assert dm.get_memory(user_id="u")["facts"][0]["content"] == "x"
 
 
+def test_get_context_injects_facts_only_in_middleware_mode(deermem_data_dir):
+    backend_config = {
+        "storage_path": str(deermem_data_dir),
+        "retrieval_adapter": "",
+        "token_counting": "char",
+    }
+    middleware = DeerMem(backend_config=backend_config, mode="middleware")
+    middleware.import_memory(
+        {
+            "user": {
+                "workContext": {"summary": "Works on DeerFlow memory."},
+            },
+            "history": {
+                "recentMonths": {"summary": "Recently redesigned storage."},
+            },
+            "facts": [
+                {
+                    "id": "fact_tool_only",
+                    "content": "Use FTS5 for active fact recall.",
+                    "category": "constraint",
+                    "confidence": 0.9,
+                    "source": "manual",
+                }
+            ],
+        },
+        user_id="u",
+    )
+
+    middleware_context = middleware.get_context(user_id="u")
+    tool_context = DeerMem(backend_config=backend_config, mode="tool").get_context(user_id="u")
+
+    assert "Works on DeerFlow memory." in middleware_context
+    assert "Recently redesigned storage." in middleware_context
+    assert "Use FTS5 for active fact recall." in middleware_context
+    assert "Works on DeerFlow memory." in tool_context
+    assert "Recently redesigned storage." in tool_context
+    assert "Use FTS5 for active fact recall." not in tool_context
+    assert "Facts:" not in tool_context
+
+
 def test_import_without_agent_name_persists_facts_in_default_markdown_bucket(deermem_data_dir):
     dm = DeerMem(backend_config=None)
     dm.import_memory(

backend/tests/test_delta_channel_state.py

@@ -13,7 +13,6 @@ from langgraph.checkpoint.memory import InMemorySaver
 from langgraph.graph import StateGraph
 from langgraph.graph.message import REMOVE_ALL_MESSAGES, add_messages
 
-from deerflow.agents import thread_state
 from deerflow.agents.thread_state import (
     DeltaThreadState,
     ThreadState,
@@ -22,7 +21,6 @@ from deerflow.agents.thread_state import (
     merge_message_writes,
     normalize_middleware_state_schemas,
 )
-from deerflow.runtime.checkpoint_mode import CheckpointModeReconfigurationError
 
 
 def _fold(state: list, writes: list) -> list:
@@ -283,26 +281,56 @@ def test_mode_selects_expected_state_schema() -> None:
     assert any(isinstance(item, DeltaChannel) for item in message_hint.__metadata__)
 
 
+def _delta_channel(schema: type) -> DeltaChannel:
+    hint = get_type_hints(schema, include_extras=True)["messages"]
+    return next(item for item in hint.__metadata__ if isinstance(item, DeltaChannel))
+
+
+def test_snapshot_frequency_parametrizes_delta_schema() -> None:
+    schema = get_thread_state_schema("delta", 250)
+    assert _delta_channel(schema).snapshot_frequency == 250
+    # Cached per frequency, and the default keeps DeltaThreadState identity.
+    assert get_thread_state_schema("delta", 250) is schema
+    assert get_thread_state_schema("delta") is DeltaThreadState
+    assert get_thread_state_schema("delta", 10) is DeltaThreadState
+
+
+def test_frozen_snapshot_frequency_drives_default_delta_schema(monkeypatch: pytest.MonkeyPatch) -> None:
+    from deerflow.runtime import checkpoint_mode
+
+    monkeypatch.setattr(checkpoint_mode, "_frozen_checkpoint_snapshot_frequency", 500)
+    assert _delta_channel(get_thread_state_schema("delta")).snapshot_frequency == 500
+    # An explicit argument always wins over the frozen value.
+    assert _delta_channel(get_thread_state_schema("delta", 250)).snapshot_frequency == 250
+
+
+def test_delta_adaptation_cache_keys_on_snapshot_frequency() -> None:
+    slow = adapt_state_schema_for_mode(AgentState, "delta", 250)
+    fast = adapt_state_schema_for_mode(AgentState, "delta", 500)
+    assert slow is not fast
+    assert _delta_channel(slow).snapshot_frequency == 250
+    assert _delta_channel(fast).snapshot_frequency == 500
+    assert adapt_state_schema_for_mode(AgentState, "delta", 250) is slow
+
+
+def test_compiled_delta_graph_bakes_configured_snapshot_frequency() -> None:
+    graph = create_agent(
+        model=_FakeModel(responses=[AIMessage(id="response", content="done")]),
+        tools=None,
+        middleware=[],
+        state_schema=get_thread_state_schema("delta", 2),
+    )
+    channel = graph.channels["messages"]
+    assert isinstance(channel, DeltaChannel)
+    assert channel.snapshot_frequency == 2
+
+
 def test_delta_adaptation_replaces_agent_state_message_reducer() -> None:
     adapted = adapt_state_schema_for_mode(AgentState, "delta")
     hint = get_type_hints(adapted, include_extras=True)["messages"]
     assert any(isinstance(item, DeltaChannel) for item in hint.__metadata__)
 
 
-def test_delta_adaptation_cache_varies_by_resolved_snapshot_frequency(monkeypatch) -> None:
-    monkeypatch.setattr(thread_state, "_frozen_delta_snapshot_frequency", None)
-    thread_state._adapt_state_schema_for_mode.cache_clear()
-    default_adapted = adapt_state_schema_for_mode(AgentState, "delta")
-
-    thread_state.freeze_delta_snapshot_frequency(7)
-    configured_adapted = adapt_state_schema_for_mode(AgentState, "delta")
-
-    assert configured_adapted is not default_adapted
-    hint = get_type_hints(configured_adapted, include_extras=True)["messages"]
-    channel = next(item for item in hint.__metadata__ if isinstance(item, DeltaChannel))
-    assert channel.snapshot_frequency == 7
-
-
 def test_agents_package_exports_delta_thread_state() -> None:
     from deerflow.agents import DeltaThreadState as ExportedDeltaThreadState
 
@@ -385,31 +413,3 @@ def test_production_message_forms_keep_assigned_ids_across_delta_replay(write: o
 
     assert first[0].id is not None
     assert second[0].id == first[0].id
-
-
-def test_delta_snapshot_frequency_defaults_to_1000(monkeypatch) -> None:
-    monkeypatch.setattr(thread_state, "_frozen_delta_snapshot_frequency", None)
-    assert thread_state.resolved_delta_snapshot_frequency() == 1000
-
-
-def test_freeze_delta_snapshot_frequency_is_process_frozen(monkeypatch) -> None:
-    monkeypatch.setattr(thread_state, "_frozen_delta_snapshot_frequency", None)
-    assert thread_state.freeze_delta_snapshot_frequency(50) == 50
-    assert thread_state.freeze_delta_snapshot_frequency(50) == 50
-    with pytest.raises(CheckpointModeReconfigurationError):
-        thread_state.freeze_delta_snapshot_frequency(10)
-
-
-def test_get_thread_state_schema_honors_frozen_frequency(monkeypatch) -> None:
-    monkeypatch.setattr(thread_state, "_frozen_delta_snapshot_frequency", None)
-    thread_state.freeze_delta_snapshot_frequency(7)
-    hint = get_type_hints(thread_state.get_thread_state_schema("delta"), include_extras=True)["messages"]
-    channel = next(item for item in hint.__metadata__ if isinstance(item, DeltaChannel))
-    assert channel.snapshot_frequency == 7
-
-
-def test_database_config_default_snapshot_frequency() -> None:
-    from deerflow.config.database_config import DatabaseConfig
-
-    assert DatabaseConfig().checkpoint_delta_snapshot_frequency == 1000
-    assert DatabaseConfig(checkpoint_delta_snapshot_frequency=25).checkpoint_delta_snapshot_frequency == 25

backend/tests/test_doctor.py

@@ -22,6 +22,50 @@ class TestCheckPython:
         assert result.status == "ok"
 
 
+# ---------------------------------------------------------------------------
+# check_pnpm
+# ---------------------------------------------------------------------------
+
+
+class TestCheckPnpm:
+    def test_uses_shared_runner_from_frontend(self, monkeypatch):
+        captured = {}
+
+        def fake_run(cmd, **kwargs):
+            captured["cmd"] = cmd
+            captured["kwargs"] = kwargs
+            return doctor.subprocess.CompletedProcess(cmd, 0, stdout="10.26.2\n", stderr="")
+
+        monkeypatch.setattr(doctor.subprocess, "run", fake_run)
+
+        result = doctor.check_pnpm()
+
+        expected_runner = doctor.Path(doctor.__file__).with_name("pnpm.py")
+        assert result.status == "ok"
+        assert result.detail == "10.26.2"
+        assert captured["cmd"] == [sys.executable, str(expected_runner), "-v"]
+        assert captured["kwargs"]["cwd"] == expected_runner.parent.parent / "frontend"
+        assert captured["kwargs"]["shell"] is False
+        assert captured["kwargs"]["check"] is False
+
+    def test_runner_failure_is_reported_as_failure(self, monkeypatch):
+        def fake_run(cmd, **kwargs):
+            return doctor.subprocess.CompletedProcess(
+                cmd,
+                42,
+                stdout="",
+                stderr="Error: pnpm command failed with exit status 42.\n",
+            )
+
+        monkeypatch.setattr(doctor.subprocess, "run", fake_run)
+
+        result = doctor.check_pnpm()
+
+        assert result.status == "fail"
+        assert "exit status 42" in result.detail
+        assert result.fix is not None
+
+
 # ---------------------------------------------------------------------------
 # check_config_exists
 # ---------------------------------------------------------------------------

backend/tests/test_dynamic_context_middleware.py

@@ -23,10 +23,12 @@ def _make_middleware(**kwargs) -> DynamicContextMiddleware:
     return DynamicContextMiddleware(**kwargs)
 
 
-def _fake_runtime(journal=None, *, pre_existing_message_ids=()):
+def _fake_runtime(journal=None, *, pre_existing_message_ids=(), user_id: str | None = None):
     context = {"__run_journal": journal} if journal is not None else {}
     if pre_existing_message_ids:
         context[CURRENT_RUN_PRE_EXISTING_MESSAGE_IDS_KEY] = frozenset(pre_existing_message_ids)
+    if user_id is not None:
+        context["user_id"] = user_id
     return SimpleNamespace(context=context)
 
 
@@ -116,6 +118,27 @@ def test_memory_included_when_present():
     assert msgs[2].content == "Hi"
 
 
+def test_memory_lookup_uses_runtime_user_id():
+    mw = _make_middleware()
+    state = {"messages": [HumanMessage(content="Hi", id="msg-1")]}
+
+    with (
+        mock.patch(
+            "deerflow.agents.lead_agent.prompt._get_memory_context",
+            return_value="",
+        ) as get_memory_context,
+        mock.patch("deerflow.agents.middlewares.dynamic_context_middleware.datetime") as mock_dt,
+    ):
+        mock_dt.now.return_value.strftime.return_value = "2026-05-08, Friday"
+        mw.before_agent(state, _fake_runtime(user_id="runtime-user"))
+
+    get_memory_context.assert_called_once_with(
+        None,
+        app_config=None,
+        user_id="runtime-user",
+    )
+
+
 def test_first_run_records_exact_effective_memory():
     journal = mock.MagicMock()
     mw = _make_middleware()