fix(mcp): preserve server binding history across gateway workers (#6617)

* fix(mcp): preserve server binding history across gateway workers

A peer Gateway worker could keep a stale pooled session after another worker deleted and identically re-added a stdio server, because the final effective MCP slice matched the revision it had already applied.

Record a per-server lifecycle generation in the shared extensions config and consume it in the reader, discovery claim, and committed handoff so an observed generation change force-rebinds only the affected server. Legacy configs derive a stable in-memory token; deletions leave permanent tombstones.

* fix(mcp): fail closed on an invalid mcpLifecycle instead of retrying

An unverifiable mcpLifecycle made initialize_mcp_tools() unpublishable, so get_cached_mcp_tools() retried discovery forever and could keep respawning stdio servers. Abort discovery on an invalid ledger after conservatively retiring local MCP state; a fixed config recovers on the next call.

Also documents the rolling-upgrade prerequisite: cross-worker ABA detection requires configuration writes to go through a lifecycle-aware writer.

* refactor(mcp): share stdio fingerprinting across lifecycle paths

Reuse lifecycle._stdio_fingerprints from the applied-revision builder so the connection-identity rule has a single implementation, and move the unusable-server diagnostic into the shared helper.

Add a regression proving the Gateway Skills writer preserves mcpLifecycle byte-for-value, and document that DeerFlowClient.update_mcp_config can raise RuntimeError after the config was committed.

* docs(mcp): condense lifecycle guidance under the AGENTS chain limit

The effective mcp/AGENTS.md chain was 15 bytes under the 98,304-byte CI hard limit, so the standalone cross-worker lifecycle bullet pushed it over. Fold the mcpLifecycle and rolling-upgrade facts into the existing selective-reconciliation bullet and condense it, keeping the chain under the limit while preserving the contract.

* docs: trim inherited agent guidance below CI size limit

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
spud 2026-10-11 11:45:28 +08:00 committed by GitHub
parent e5d9a5f42d
commit 75ce6f4920
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 1561 additions and 81 deletions

View File

@ -1,4 +1,5 @@
import asyncio
import copy
import json
import logging
import os
@ -37,6 +38,7 @@ from deerflow.mcp.cache import (
publish_mcp_tools_cache_reset,
reset_mcp_tools_cache, # noqa: F401 - retained module attribute for compatibility/tests
)
from deerflow.mcp.lifecycle import LIFECYCLE_KEY, plan_mcp_lifecycle
from deerflow.mcp.tasks.runtime import McpTaskConfigurationError, validate_mcp_task_config_snapshot
from deerflow.utils.file_io import await_drained
@ -1227,10 +1229,15 @@ def _commit_mcp_config_write(
config_path: Path,
raw_data: dict,
*,
previous_raw: dict,
check_installation_ids: bool = True,
) -> McpReconciliationPending:
"""Persist one validated candidate and install its committed binding fence.
``previous_raw`` is the raw pre-mutation snapshot and is required: a
controlled MCP write must never be able to silently skip the lifecycle
generation update by omitting it.
Must be called while the extensions-config write and file locks are held.
The parsed candidate and the bytes on disk therefore describe the same
transition even if another writer is waiting to re-add an identical server.
@ -1239,6 +1246,9 @@ def _commit_mcp_config_write(
raw_data,
check_installation_ids=check_installation_ids,
)
raw_data[LIFECYCLE_KEY] = plan_mcp_lifecycle(previous_raw, raw_data, candidate_config=candidate)
# Re-parse so the committed fence sees the ledger we just wrote.
candidate = validate_raw_extensions_config(raw_data)
atomic_write_extensions_config(config_path, raw_data)
try:
reload_extensions_config()
@ -1330,7 +1340,7 @@ def _apply_mcp_config_update(body: McpConfigUpdateRequest) -> dict:
raw_skills = {name: {"enabled": skill.enabled} for name, skill in current_config.skills.items()}
config_data["skills"] = raw_skills
pending = _commit_mcp_config_write(config_path, config_data)
pending = _commit_mcp_config_write(config_path, config_data, previous_raw=copy.deepcopy(raw_data))
try:
logger.info(f"MCP configuration updated and saved to: {config_path}")
@ -1356,6 +1366,7 @@ def _apply_mcp_server_state_update(body: McpServerStateUpdateRequest) -> dict:
)
raw_data = _load_raw_extensions_config(config_path, create=False)
previous_raw = copy.deepcopy(raw_data)
raw_servers = _raw_mcp_servers(raw_data)
if body.server_name not in raw_servers:
raise HTTPException(
@ -1373,7 +1384,7 @@ def _apply_mcp_server_state_update(body: McpServerStateUpdateRequest) -> dict:
)
raw_server["enabled"] = body.enabled
pending = _commit_mcp_config_write(config_path, raw_data)
pending = _commit_mcp_config_write(config_path, raw_data, previous_raw=previous_raw)
try:
logger.info("MCP server %s enabled state updated to %s", body.server_name, body.enabled)
@ -1458,6 +1469,7 @@ def _apply_mcp_servers_create(body: McpConfigUpdateRequest) -> dict:
config_path = _mcp_config_path(create=True)
with extensions_config_write_lock, extensions_config_file_lock(config_path):
raw_data = _load_raw_extensions_config(config_path, create=True)
previous_raw = copy.deepcopy(raw_data)
raw_servers = _raw_mcp_servers(raw_data)
duplicate = next((name for name in body.mcp_servers if name in raw_servers), None)
if duplicate is not None:
@ -1471,7 +1483,7 @@ def _apply_mcp_servers_create(body: McpConfigUpdateRequest) -> dict:
raw_servers[name] = incoming.model_dump()
raw_data["mcpServers"] = raw_servers
_ensure_skills_key(raw_data)
pending = _commit_mcp_config_write(config_path, raw_data)
pending = _commit_mcp_config_write(config_path, raw_data, previous_raw=previous_raw)
try:
logger.info("Added MCP servers: %s", ", ".join(body.mcp_servers))
@ -1485,6 +1497,7 @@ def _apply_mcp_server_config_update(body: McpServerConfigUpdateRequest) -> dict:
config_path = _mcp_config_path(create=False)
with extensions_config_write_lock, extensions_config_file_lock(config_path):
raw_data = _load_raw_extensions_config(config_path, create=False)
previous_raw = copy.deepcopy(raw_data)
raw_servers = _raw_mcp_servers(raw_data)
if body.server_name not in raw_servers:
raise HTTPException(
@ -1501,7 +1514,7 @@ def _apply_mcp_server_config_update(body: McpServerConfigUpdateRequest) -> dict:
_ensure_no_masked_secrets(merged)
raw_servers[body.server_name] = merged.model_dump()
raw_data["mcpServers"] = raw_servers
pending = _commit_mcp_config_write(config_path, raw_data)
pending = _commit_mcp_config_write(config_path, raw_data, previous_raw=previous_raw)
try:
logger.info("Updated MCP server: %s", body.server_name)
@ -1515,6 +1528,7 @@ def _apply_mcp_server_delete(server_name: str) -> dict:
config_path = _mcp_config_path(create=False)
with extensions_config_write_lock, extensions_config_file_lock(config_path):
raw_data = _load_raw_extensions_config(config_path, create=False)
previous_raw = copy.deepcopy(raw_data)
raw_servers = _raw_mcp_servers(raw_data)
if server_name not in raw_servers:
raise HTTPException(
@ -1529,6 +1543,7 @@ def _apply_mcp_server_delete(server_name: str) -> dict:
pending = _commit_mcp_config_write(
config_path,
raw_data,
previous_raw=previous_raw,
check_installation_ids=False,
)

View File

@ -53,6 +53,7 @@ from deerflow.config.extensions_config import (
)
from deerflow.config.paths import get_paths
from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig
from deerflow.mcp.lifecycle import LIFECYCLE_KEY, plan_mcp_lifecycle
from deerflow.mcp_scope import THREAD_INCARNATION_CONTEXT_KEY
from deerflow.models import create_chat_model
from deerflow.models.reasoning import reasoning_capabilities_payload, resolve_reasoning_contract
@ -1455,23 +1456,58 @@ class DeerFlowClient:
Raises:
ValueError: If the resulting config would not load; nothing is written.
OSError: If the config file cannot be written.
RuntimeError: If the configuration was saved but local MCP reconciliation or
teardown failed. The configuration may already have changed on disk, so
callers must not assume the write failed and blindly retry.
"""
from deerflow.mcp.cache import (
fail_mcp_reconciliation,
finish_mcp_reconciliation,
prepare_mcp_reconciliation,
)
config_path = ExtensionsConfig.resolve_config_path()
if config_path is None:
raise FileNotFoundError("Cannot locate extensions_config.json. Set DEER_FLOW_EXTENSIONS_CONFIG_PATH or ensure it exists in the project root.")
pending = None
committed_failure = None
reloaded = None
with extensions_config_write_lock, extensions_config_file_lock(config_path):
# The singleton is process-local, so re-read the shared file under
# the cross-process lock before merging the replacement MCP map.
# Read it raw so sibling keys keep their $VAR placeholders.
config_data = read_raw_extensions_config(config_path)
config_data["mcpServers"] = mcp_servers
previous_raw = read_raw_extensions_config(config_path)
config_data = copy.deepcopy(previous_raw)
config_data["mcpServers"] = copy.deepcopy(mcp_servers)
config_data[LIFECYCLE_KEY] = plan_mcp_lifecycle(previous_raw, config_data)
validate_raw_extensions_config(config_data)
candidate = validate_raw_extensions_config(config_data)
self._atomic_write_json(config_path, config_data)
reloaded = reload_extensions_config()
try:
reloaded = reload_extensions_config()
pending = prepare_mcp_reconciliation(candidate, config_path=config_path)
except Exception as exc:
# The file is already committed, so detach local state under the
# cache condition here and finish teardown after the locks are
# released; never report the write as a clean success.
committed_failure = (fail_mcp_reconciliation(exc), exc)
self.reset_agent()
try:
if committed_failure is not None:
failed_pending, _failure = committed_failure
finish_mcp_reconciliation(failed_pending)
else:
finish_mcp_reconciliation(pending)
finally:
# The config file is already committed, so an agent built from the
# previous MCP tool set must not survive this call even when local
# reconciliation or teardown raised.
self.reset_agent()
if committed_failure is not None:
_failed_pending, failure = committed_failure
raise RuntimeError("MCP configuration was saved, but local cache reconciliation failed; retry or restart DeerFlow before relying on the changed server.") from failure
return {"mcp_servers": {name: server.model_dump() for name, server in reloaded.mcp_servers.items()}}
# ------------------------------------------------------------------

View File

@ -38,7 +38,7 @@ The optional background-task `status` filter flows through `McpTaskSubmitter.lis
transport after its loop is gone. Normal `asyncio.run()` drains its owners.
- **Persistent stdio session capacity**: `MCPSessionPool.MAX_SESSIONS` is a hard cap on the live LRU registry. Capacity is enforced both before creating a session and when an in-flight session is promoted, because different keys can finish initialization concurrently after all observing spare capacity. Promotion-time victims are signalled and drained by separately tracked teardown tasks on their owning loops; the newly promoted owner must never await a victim, so a blocked victim exit cannot prevent its own closure or disconnect recovery.
- **Persistent stdio binding fence**: pooled session registry keys remain `(server_name, scope_key, owning_loop, ownership_domain)`. Separately, each `(ownership_domain, server_name)` has an opaque `ServerBinding` capability derived from a SHA-256 digest of the base stdio process identity (`transport`, `command`, `args`, configured `cwd`, configured `env`; the resolved values themselves are never stored in the binding repr). Discovery captures the exact pool plus binding before its first await, and durable stdio calls bind before per-call workspace `cwd`/`TMPDIR` augmentation. `get_session()` rejects bindings that are already superseded at admission, owner promotion, or the final post-await check. That final check is a check-before-return guard, not a lease across subsequent `call_tool()` use: a reconciliation racing after it detaches and closes the owner, and the caller then fails through the existing transport-disconnect handling. Binding reconciliation consumes already-classified active fingerprints/removals, atomically installs epochs/tombstones and detaches matching owners under the pool lock, and keeps those detached owners in the pool's teardown reaper so their `__aexit__` stays observable even when the caller drops the returned `PreparedRetirement`; configuration diffing remains a higher-layer responsibility.
- **Process-local selective reconciliation**: `mcp/cache.py` keeps an *applied reconciliation baseline* (`_applied_mcp_revision`) apart from the published tool-cache baseline: the effective MCP snapshot, the normalized interceptor chain, and a `normalized_connection_fingerprint()` per enabled stdio server derived from `build_server_params()` (not a whole-config `model_dump()`, so `description`/`routing`/`tools`/`tool_name_prefix` are not connection changes); the snapshot is excluded from `repr` because it can embed resolved `$VAR` values. A claim *installs* the revision it records, so the baseline means those deployment epochs are in place: a stale binding left by a durable-task caller or an earlier failed attempt is repaired before `ensure_binding()` runs, which fails closed on a mismatch, and with no previous baseline the same lock tombstones the unlisted deployment servers the pool retains (`retire_unlisted=True`). Both epoch-installing decisions consult the frozen durable-task snapshot first: a rejected revision takes the conservative reset, and a claim fails as `get_mcp_tools()` would without touching the pool. It is recorded when discovery starts, so a failed/cancelled discovery that already seeded a binding can still retire it, and consecutive observations keep advancing epochs while the cache is empty. Classification and application both happen under `_init_condition` in one synchronous section, so epoch installation and generation invalidation cannot interleave or run out of order. An unchanged effective slice adopts the new path/signature and keeps the published tools; a changed interceptor chain, an unreadable/unverifiable revision, a shared-reset-marker change, a frozen-task rejection, or an unbuildable baseline keeps the conservative whole-pool reset; otherwise `reconcile_bindings(active, removed, domain="deployment")` runs without replacing the pool singleton, then the published tools are dropped and `_cache_generation` is bumped so an in-flight discovery cannot publish. Unchanged stdio servers keep their binding, session and owner; changed or removed ones advance an epoch or get a tombstone. Teardown stays with the pool's reaper outside every lock, even if the claim then fails. Claim and transition are deployment-scoped, so a deployment revision never retires personal state; the conservative fallbacks are whole-pool resets that do. A claim that observes a new shared-reset generation retires local state before adopting the marker whenever published tools, an applied baseline, or deployment-domain pooled state exist; an empty or personal-only process adopts it. Explicit `reset_mcp_tools_cache()` / `POST /api/mcp/cache/reset` remain full resets. Gateway config writers instead call `prepare_mcp_reconciliation()` with the exact committed candidate and post-write path/signature while still holding the config locks; `finish_mcp_reconciliation()` tears down any retired pool after those locks are released, and post-lock response construction/logging is wrapped in `finally` so a later failure cannot skip that teardown. This preserves delete→identical-re-add as two transitions and covers durable-only bindings that predate a published tool cache. If the committed handoff fails, `fail_mcp_reconciliation()` detaches local state conservatively and the API reports an explicit 500 instead of claiming success.
- **Process-local selective reconciliation**: `mcp/cache.py` keeps an *applied reconciliation baseline* (`_applied_mcp_revision`) apart from the published tool-cache baseline: the effective MCP snapshot, the normalized interceptor chain, and a `normalized_connection_fingerprint()` per enabled stdio server derived from `build_server_params()` (not a whole-config `model_dump()`, so presentation-only fields are not connection changes). A claim *installs* the revision it records, so the baseline means those deployment epochs are in place: a stale binding left by a durable caller or an earlier failed attempt is repaired before `ensure_binding()` runs, which fails closed on a mismatch, and with no previous baseline the same lock tombstones the unlisted deployment servers the pool retains (`retire_unlisted=True`). Both epoch-installing decisions consult the frozen durable-task snapshot first: a rejected revision takes the conservative reset, and a claim fails as `get_mcp_tools()` would without touching the pool. The revision is recorded when discovery starts, so even a failed/cancelled discovery can retire a binding it seeded. Classification and application run in one `_init_condition` section, so epoch installation and generation invalidation cannot interleave. An unchanged effective slice adopts the new path/signature and keeps the published tools; a changed interceptor chain, an unreadable/unverifiable revision, a shared-reset-marker change, a frozen-task rejection, or an unbuildable baseline keeps the conservative whole-pool reset; otherwise `reconcile_bindings(active, removed, domain="deployment")` runs without replacing the pool, then the published tools are dropped and `_cache_generation` is bumped so an in-flight discovery cannot publish. Unchanged stdio servers keep their binding, session and owner; changed or removed ones advance an epoch or get a tombstone, and teardown stays with the pool's reaper outside every lock. Claim and transition are deployment-scoped, so a deployment revision never retires personal state; the conservative whole-pool resets do. A new shared-reset generation retires local state first whenever published tools, an applied baseline, or deployment pooled state exist; an empty/personal-only process adopts it. Explicit `reset_mcp_tools_cache()` / `POST /api/mcp/cache/reset` remain full resets. Gateway config writers call `prepare_mcp_reconciliation()` with the exact committed candidate under the config locks, then `finish_mcp_reconciliation()` outside them; post-lock response wrapping in `finally` keeps teardown from being skipped. This preserves delete→identical-re-add as two transitions and covers durable-only bindings that predate a published tool cache; if the committed handoff fails, `fail_mcp_reconciliation()` detaches local state and the API reports an explicit 500. The shared `mcpLifecycle` ledger (`mcp/lifecycle.py`) extends this cross-worker: readers compare `_AppliedMcpRevision.lifecycle_tokens` before the effective-snapshot shortcut, reader/claim/committed force-rebind changed servers, and every config write must go through a lifecycle-aware writer — during a rolling upgrade, restrict writes to upgraded workers until legacy writers are retired (a legacy delete→identical-re-add does not advance the ledger).
- **Cache invalidation**: Staleness is detected in two stages. First, the resolved config path and a `(mtime, size, sha256)` content signature are compared against the values recorded when the cache was published, not a strict mtime `>` comparison — this catches same-second edits, mtime that stays put or moves backward (`git checkout`, `cp -p` / backup restore, `tar` / `rsync`, object-store / network mounts), and a switch to a different config file with an equal-or-older mtime. The signature helper (`config/file_signature.py::get_config_signature`) is shared with `config/app_config.py::get_app_config()` for the sibling runtime-editable config file, rather than each maintaining its own copy. Second, because the file also carries `skills` and middleware settings, a changed signature alone is not enough: `mcp/cache.py::_effective_mcp_config_snapshot()` serializes the effective MCP slice (enabled servers' parsed config in declaration order plus `mcpInterceptors`; the raw `transport` alias is dropped so equivalent `type`/`transport` spellings compare equal) and the cache is retired only when that slice changed. A skills-only or middleware-only edit therefore adopts the new signature and keeps the cached tools and pooled sessions, while a real MCP change is classified against the process-local applied-reconciliation baseline and applied selectively (see the selective-reconciliation bullet above) instead of unconditionally replacing the pool. The snapshot may embed resolved credentials, so it stays in process memory and is never logged or persisted. `_is_cache_stale()` adopts a new signature when the MCP slice is unchanged, so it is not a read-only predicate; production callers run it under `_init_condition`. `ExtensionsConfig.resolve_config_path()` raises `FileNotFoundError` for an explicit `config_path`/`DEER_FLOW_EXTENSIONS_CONFIG_PATH` that points at a missing file — an operator-asserted path going missing is a real misconfiguration, so this is intentionally loud for callers that load the config for actual use (e.g. `from_file()` via `get_mcp_tools()`); only the fallback search mode returns `None`. The MCP cache's own path resolution (`mcp/cache.py::_resolve_config_path`) is narrower: it catches that specific `FileNotFoundError` locally and treats it the same as "unconfigured", so this staleness check degrades to "not stale" instead of propagating an exception when a previously-valid explicit/env-var config disappears mid-run. Two no-config states are distinguished: `_initialized_without_config` makes a cache published while no config existed stale as soon as one appears, while a config deleted after a successful load keeps the last-known-good fail-soft behavior and stays "not stale", while a file that is present but whose content cannot be hashed is treated as stale instead, because the cache cannot prove the revision it published is still the current one. `initialize_mcp_tools()` reads one `ExtensionsConfig` instance, hands that exact object to `get_mcp_tools(extensions_config=...)`, and publishes only when the effective MCP snapshot of the current config still equals the snapshot of the revision that produced the tools; otherwise it resets the tool cache through the same session-pool retirement path as normal stale invalidation before waiters retry. This closes the discovery window — tools built from a superseded MCP revision, or from a resolved path whose signature cannot be re-read, are never published — while a skills-only edit during discovery still publishes because the MCP slice is unchanged.
- **Transports**: stdio (command-based), SSE, HTTP

View File

@ -15,6 +15,13 @@ from deerflow.config.file_signature import ConfigSignature as _ConfigSignature
from deerflow.config.file_signature import get_config_signature as _get_config_signature
from deerflow.config.shared_reset_marker import SharedResetMarker
from deerflow.mcp.config_normalization import normalize_mcp_interceptor_paths, normalize_mcp_server_config
from deerflow.mcp.lifecycle import (
InvalidMcpLifecycle,
_stdio_fingerprints,
effective_tokens,
lifecycle_delta,
parse_mcp_lifecycle,
)
if TYPE_CHECKING:
from deerflow.mcp.session_pool import MCPSessionPool
@ -46,6 +53,7 @@ class _AppliedMcpRevision:
effective_snapshot: str = field(repr=False)
interceptors: str
stdio_connections: Mapping[str, str]
lifecycle_tokens: Mapping[str, str]
# Reconciliation kinds produced by ``_plan_mcp_reconciliation_locked``. The
@ -71,6 +79,7 @@ class _McpReconciliation:
path: Path | None = None
signature: _ConfigSignature | None = None
retire_unlisted: bool = False
force_rebind: frozenset[str] = frozenset()
@dataclass(frozen=True, slots=True)
@ -244,29 +253,13 @@ def _mcp_revision_from_config(config) -> _AppliedMcpRevision:
server — so presentation-only fields (``description``/``routing``/``tools``/
``tool_name_prefix``) cannot masquerade as a connection change.
"""
from deerflow.mcp.client import build_server_params
from deerflow.mcp.session_pool import normalized_connection_fingerprint
stdio_connections: dict[str, str] = {}
for server_name, server in config.get_enabled_mcp_servers().items():
try:
connection = build_server_params(server_name, server)
except Exception:
# ``build_servers_config`` drops a server whose parameters cannot be
# built, so it is absent from discovery too. Omitting it here keeps
# the connection-identity baseline aligned with what discovery can
# actually install instead of raising into the hot staleness check.
logger.debug(
"MCP server '%s' has unusable parameters; omitting it from the connection-identity baseline",
server_name,
)
continue
if connection.get("transport", "stdio") == "stdio":
stdio_connections[server_name] = normalized_connection_fingerprint(connection)
stdio_connections = _stdio_fingerprints(config)
ledger = parse_mcp_lifecycle(config.model_extra or {})
return _AppliedMcpRevision(
effective_snapshot=_effective_mcp_config_snapshot(config),
interceptors=_interceptors_snapshot(config),
stdio_connections=stdio_connections,
lifecycle_tokens=effective_tokens(config, ledger),
)
@ -347,7 +340,12 @@ def _read_stable_mcp_revision(
config = _load_stable_mcp_config(config_path, expected_signature)
if config is None:
return None
return config, _mcp_revision_from_config(config)
try:
revision = _mcp_revision_from_config(config)
except InvalidMcpLifecycle:
logger.warning("Extensions config mcpLifecycle is present but invalid; treating the MCP cache as stale")
return None
return config, revision
def _frozen_task_snapshot_rejects(config):
@ -408,11 +406,16 @@ def _install_claimed_revision_locked(pool, revision: _AppliedMcpRevision, *, pre
the claim must tombstone, under the same lock that installs the epochs.
"""
active, removed = _deployment_binding_delta(previous, revision)
if previous is None:
force_rebind: frozenset[str] = frozenset(active)
else:
force_rebind = lifecycle_delta(previous.lifecycle_tokens, revision.lifecycle_tokens, active)
pool.reconcile_bindings(
active,
removed,
domain="deployment",
retire_unlisted=previous is None,
force_rebind=force_rebind,
)
@ -436,7 +439,10 @@ def _has_mcp_reconciliation_state() -> bool:
all reasons to keep watching the config. A process that never initialized
MCP pays no config-read cost (``refresh_mcp_cache_if_active`` contract).
"""
return _cache_initialized or _applied_mcp_revision is not None or _initializing_generation is not None
from deerflow.mcp.session_pool import current_session_pool
pool = current_session_pool()
return _cache_initialized or _applied_mcp_revision is not None or _initializing_generation is not None or (pool is not None and bool(pool.retained_server_names(domain="deployment")))
def _plan_mcp_reconciliation_locked() -> _McpReconciliation | None:
@ -455,7 +461,7 @@ def _plan_mcp_reconciliation_locked() -> _McpReconciliation | None:
classification and application, so two readers can never apply revisions
out of order).
"""
global _config_path, _config_signature
global _applied_mcp_revision, _config_path, _config_signature
if not _has_mcp_reconciliation_state():
return None
@ -504,39 +510,52 @@ def _plan_mcp_reconciliation_locked() -> _McpReconciliation | None:
candidate_config, revision = loaded
applied = _applied_mcp_revision
if applied is not None and revision.effective_snapshot == applied.effective_snapshot:
logger.info("Extensions config changed but the effective MCP configuration did not; keeping cached MCP tools and sessions")
force_rebind = lifecycle_delta(applied.lifecycle_tokens, revision.lifecycle_tokens, revision.stdio_connections) if applied is not None else frozenset()
if applied is not None and revision.effective_snapshot == applied.effective_snapshot and not force_rebind:
logger.info("Extensions config changed but the effective MCP configuration and active lifecycle generations did not; keeping cached MCP tools and sessions")
# Snapshot equality implies equal stdio_connections, so no active epoch
# changes; but a tombstone-only history advance must still be recorded.
# The published tool cache is deliberately left untouched.
_applied_mcp_revision = revision
_config_path, _config_signature = current_path, current_signature
return None
if applied is not None and revision.interceptors == applied.interceptors:
rejection = _frozen_task_snapshot_rejects(candidate_config)
if rejection is not None:
# Installing this revision's epochs would strand the durable callers
# that still run against the frozen startup configuration, so keep
# the conservative fallback: replace the pool (where those callers
# rebind from the configuration they were started with) and never
# install the rejected epoch.
logger.info(
"MCP configuration revision is rejected by the frozen durable-task snapshot (%s); resetting instead of installing binding epochs",
type(rejection).__name__,
)
return _McpReconciliation(_RECONCILE_FULL)
if applied is not None and revision.interceptors != applied.interceptors:
logger.info("MCP global interceptor chain changed; cache is stale")
return _McpReconciliation(_RECONCILE_FULL)
rejection = _frozen_task_snapshot_rejects(candidate_config)
if rejection is not None:
# Installing this revision's epochs would strand the durable callers
# that still run against the frozen startup configuration, so keep
# the conservative fallback and never install the rejected epoch.
logger.info(
"MCP configuration revision is rejected by the frozen durable-task snapshot (%s); resetting instead of installing binding epochs",
type(rejection).__name__,
)
return _McpReconciliation(_RECONCILE_FULL)
if applied is None:
# No trusted baseline: conservatively re-epoch every active deployment
# binding (same fingerprint included) and tombstone unlisted deployment
# state, without replacing the pool or touching personal state.
logger.info("MCP configuration changed without an applicable selective baseline; conservatively rebinding deployment state")
return _McpReconciliation(
_RECONCILE_SELECTIVE,
revision=revision,
path=current_path,
signature=current_signature,
retire_unlisted=True,
force_rebind=frozenset(revision.stdio_connections),
)
# Either there is no applied baseline to diff against, or the global
# interceptor chain changed. Both keep the pre-PR-C conservative behavior:
# replace the pool instead of guessing which sessions may be reused.
logger.info(
"MCP configuration changed without an applicable selective baseline (interceptors %s); cache is stale",
"changed" if applied is not None else "unknown",
return _McpReconciliation(
_RECONCILE_SELECTIVE,
revision=revision,
path=current_path,
signature=current_signature,
force_rebind=force_rebind,
)
return _McpReconciliation(_RECONCILE_FULL)
def _invalidate_published_tools_locked() -> None:
@ -582,10 +601,13 @@ def _apply_mcp_reconciliation_locked(plan: _McpReconciliation):
re_epoching or "none",
sorted(removed) or "none",
)
if plan.retire_unlisted:
pool.reconcile_bindings(active, removed, domain="deployment", retire_unlisted=True)
else:
pool.reconcile_bindings(active, removed, domain="deployment")
pool.reconcile_bindings(
active,
removed,
domain="deployment",
retire_unlisted=plan.retire_unlisted,
force_rebind=plan.force_rebind,
)
_applied_mcp_revision = revision
_config_path, _config_signature = plan.path, plan.signature
@ -620,13 +642,24 @@ def _plan_committed_mcp_reconciliation_locked(
pool = get_session_pool()
has_local_state = _cache_initialized or _applied_mcp_revision is not None or _initializing_generation is not None or bool(pool.retained_server_names(domain="deployment"))
if not has_local_state:
# No visible local state, but a durable-task caller can bind between this
# observation and the record below (``ensure_binding`` takes only the pool
# lock). Install this revision's deployment epochs under the pool lock
# first, so an insertion in that window is re-epoched or tombstoned rather
# than recorded as applied without ever being fenced.
_install_claimed_revision_locked(pool, revision, previous=None)
_applied_mcp_revision = revision
_config_path, _config_signature = path, signature
_initialized_without_config = path is None
return None
applied = _applied_mcp_revision
if applied is not None and revision.effective_snapshot == applied.effective_snapshot:
force_rebind = lifecycle_delta(applied.lifecycle_tokens, revision.lifecycle_tokens, revision.stdio_connections) if applied is not None else frozenset()
if applied is not None and revision.effective_snapshot == applied.effective_snapshot and not force_rebind:
# Equal snapshot implies equal active connections, so only the consumed
# token map (e.g. tombstones) can have moved; record it without touching
# the pool or the published cache.
_applied_mcp_revision = revision
_config_path, _config_signature = path, signature
_initialized_without_config = path is None
return None
@ -634,12 +667,24 @@ def _plan_committed_mcp_reconciliation_locked(
if applied is not None and revision.interceptors != applied.interceptors:
return _McpReconciliation(_RECONCILE_FULL)
if applied is None:
# Mirrors the reader's no-baseline rule: conservatively re-epoch active
# deployment bindings and tombstone unlisted ones, personal untouched.
return _McpReconciliation(
_RECONCILE_SELECTIVE,
revision=revision,
path=path,
signature=signature,
retire_unlisted=True,
force_rebind=frozenset(revision.stdio_connections),
)
return _McpReconciliation(
_RECONCILE_SELECTIVE,
revision=revision,
path=path,
signature=signature,
retire_unlisted=applied is None,
force_rebind=force_rebind,
)
@ -753,9 +798,11 @@ async def initialize_mcp_tools() -> list[BaseTool]:
loaded_tools = None
loaded_snapshot = None
loaded_lifecycle = None
post_path = None
post_sig = None
post_snapshot = None
post_lifecycle = None
loaded_reset_signature = None
post_reset_signature = None
init_succeeded = False
@ -777,6 +824,18 @@ async def initialize_mcp_tools() -> list[BaseTool]:
)
raise RuntimeError("Extensions config could not be loaded for MCP tool discovery") from None
loaded_snapshot = _effective_mcp_config_snapshot(loaded_config)
try:
loaded_lifecycle = effective_tokens(loaded_config, parse_mcp_lifecycle(loaded_config.model_extra or {}))
except InvalidMcpLifecycle:
# An unverifiable ledger can never be published. Returning an empty
# result would make the lazy caller retry forever and respawn stdio
# servers, so retire any local state and abort discovery instead.
with _init_condition:
retired_for_invalid = _reset_mcp_tools_cache_state_and_retire_pool_locked()
if retired_for_invalid is not None:
retired_for_invalid.close_all_sync()
logger.warning("Extensions config mcpLifecycle is invalid; aborting MCP tool discovery")
raise RuntimeError("Extensions config contains an invalid mcpLifecycle; fix or remove it before using MCP tools") from None
loaded_reset_signature = _current_cache_reset_marker_signature(_resolve_config_path())
# Claim the exact pool this generation owns under the same lock the
# reset path takes, so verify-generation + capture-pool is one atomic
@ -865,7 +924,11 @@ async def initialize_mcp_tools() -> list[BaseTool]:
post_path, post_sig = _current_config_state()
post_reset_signature = _current_cache_reset_marker_signature(post_path)
if post_path is not None and post_sig is not None:
post_snapshot = _read_stable_mcp_snapshot(post_path, post_sig)
post_loaded = _read_stable_mcp_revision(post_path, post_sig)
if post_loaded is not None:
_post_config, post_revision = post_loaded
post_snapshot = post_revision.effective_snapshot
post_lifecycle = post_revision.lifecycle_tokens
elif post_path is not None:
# The path resolved but its signature could not be read. Publishing
# here would record an unpinned cache that later checks could never
@ -885,6 +948,7 @@ async def initialize_mcp_tools() -> list[BaseTool]:
fallback_snapshot = None
recheck_path, recheck_sig = _current_config_state()
post_snapshot = fallback_snapshot if recheck_path is None and recheck_sig is None else None
post_lifecycle = {}
init_succeeded = True
finally:
if not init_succeeded:
@ -900,7 +964,7 @@ async def initialize_mcp_tools() -> list[BaseTool]:
logger.info("MCP cache was reset during initialization; discarding stale result")
return []
publish = loaded_snapshot is not None and post_snapshot is not None and loaded_snapshot == post_snapshot and loaded_reset_signature == post_reset_signature
publish = loaded_snapshot is not None and post_snapshot is not None and loaded_snapshot == post_snapshot and loaded_lifecycle == post_lifecycle and loaded_reset_signature == post_reset_signature
if not publish:
logger.warning("MCP config or shared reset generation changed during initialization; discarding stale result")
# Reconcile against the revision observed now instead of forcing

View File

@ -0,0 +1,207 @@
"""Shared per-server MCP lifecycle generations (``mcpLifecycle`` v1).
A Gateway worker installs binding epochs only for configuration transitions it
personally observes. A second worker sharing the same ``extensions_config.json``
can therefore miss a ``delete(A) -> identical re-add(A)`` when the final effective
MCP slice equals the revision it already applied. This module maintains the small
shared ledger that records a per-server *connection lifecycle* generation so a peer
can detect that history even when the fingerprint is unchanged.
The ledger stores only opaque tokens. It never contains connection configuration
or resolved credentials, and it is deliberately invisible to
``_effective_mcp_config_snapshot`` so the token difference is observable on its own.
Schema (top level of ``extensions_config.json``):
"mcpLifecycle": {"version": 1, "servers": {"A": "<32 lowercase hex>"}}
Tri-state parsing: the key is either absent (legacy, supported), valid, or
invalid. Invalid means unverifiable, and callers must fail closed (full reset for
readers, conservative migration for writers) rather than treat it as unchanged.
"""
from __future__ import annotations
import hashlib
import logging
import re
import secrets
from collections.abc import Collection, Mapping
from typing import Any
from pydantic import BaseModel, ConfigDict, field_validator
logger = logging.getLogger(__name__)
LIFECYCLE_KEY = "mcpLifecycle"
LIFECYCLE_VERSION = 1
_TOKEN_RE = re.compile(r"[0-9a-f]{32}")
_LEGACY_PREFIX = "mcp-lifecycle-legacy:v1:"
class InvalidMcpLifecycle(ValueError):
"""Raised when ``mcpLifecycle`` is present but cannot be trusted."""
class McpLifecycleLedger(BaseModel):
"""Strict model for one ``mcpLifecycle`` block.
Pydantic's default coercion would accept ``version=True`` for an ``int`` field,
so the version and every token are validated with explicit ``type(...)`` checks.
"""
model_config = ConfigDict(extra="forbid")
version: int
servers: dict[str, str]
@field_validator("version", mode="before")
@classmethod
def _strict_version(cls, value: Any) -> Any:
if type(value) is not int or value != LIFECYCLE_VERSION:
raise ValueError("unsupported mcpLifecycle version")
return value
@field_validator("servers", mode="before")
@classmethod
def _strict_servers(cls, value: Any) -> Any:
if type(value) is not dict:
raise ValueError("mcpLifecycle.servers must be an object")
for name, token in value.items():
if type(name) is not str or not name:
raise ValueError("mcpLifecycle server name must be a non-empty string")
if type(token) is not str or _TOKEN_RE.fullmatch(token) is None:
raise ValueError("mcpLifecycle token must be 32 lowercase hex characters")
return value
def new_token() -> str:
"""Return a fresh opaque lifecycle token (128 random bits, lowercase hex)."""
return secrets.token_hex(16)
def legacy_token(server_name: str, fingerprint: str) -> str:
"""Derive the stable in-memory token for a server with no ledger entry.
Depends only on the server name and the secret-safe connection fingerprint, so
every worker derives the same value before the first controlled write and no
unchanged legacy server is retired.
"""
material = f"{_LEGACY_PREFIX}{server_name}:{fingerprint}"
return hashlib.sha256(material.encode("utf-8")).hexdigest()
def parse_mcp_lifecycle(raw_config: Mapping[str, Any] | None) -> McpLifecycleLedger | None:
"""Return the ledger, ``None`` when absent, or raise ``InvalidMcpLifecycle``.
Absent (legacy) is a supported state. Present-but-invalid is not.
"""
if raw_config is None or LIFECYCLE_KEY not in raw_config:
return None
try:
return McpLifecycleLedger.model_validate(raw_config[LIFECYCLE_KEY])
except Exception as exc: # noqa: BLE001 - any parse failure is unverifiable
raise InvalidMcpLifecycle("mcpLifecycle is present but invalid") from exc
def _stdio_fingerprints(config) -> dict[str, str]:
"""Return the enabled stdio server names mapped to connection fingerprints."""
from deerflow.mcp.client import build_server_params
from deerflow.mcp.session_pool import normalized_connection_fingerprint
fingerprints: dict[str, str] = {}
for server_name, server in config.get_enabled_mcp_servers().items():
try:
connection = build_server_params(server_name, server)
except Exception: # noqa: BLE001 - unusable servers are absent from discovery too
# ``build_servers_config`` drops a server whose parameters cannot be
# built, so it is absent from discovery too. Omitting it keeps the
# connection-identity baseline aligned with what discovery installs.
logger.debug(
"MCP server '%s' has unusable parameters; omitting it from the connection-identity baseline",
server_name,
)
continue
if connection.get("transport", "stdio") != "stdio":
continue
fingerprints[server_name] = normalized_connection_fingerprint(connection)
return fingerprints
def effective_tokens(config, ledger: McpLifecycleLedger | None) -> dict[str, str]:
"""Return the effective per-server token map for one parsed config.
Includes explicit ledger entries (tombstones included) plus legacy-derived
tokens for active stdio servers that have no ledger entry yet.
"""
tokens: dict[str, str] = dict(ledger.servers) if ledger is not None else {}
for server_name, fingerprint in _stdio_fingerprints(config).items():
tokens.setdefault(server_name, legacy_token(server_name, fingerprint))
return tokens
def lifecycle_delta(
previous_tokens: Mapping[str, str],
candidate_tokens: Mapping[str, str],
active_names: Collection[str],
) -> frozenset[str]:
"""Return active servers whose effective lifecycle token changed.
These are the servers that must be force-rebound even when their connection
fingerprint is unchanged. A name absent from ``previous_tokens`` counts as
changed so a newly tracked server is installed conservatively.
"""
return frozenset(name for name in active_names if candidate_tokens.get(name) != previous_tokens.get(name))
def plan_mcp_lifecycle(
previous_raw: Mapping[str, Any] | None,
candidate_raw: Mapping[str, Any],
*,
candidate_config=None,
) -> dict[str, Any]:
"""Return the full ``mcpLifecycle`` block to embed in ``candidate_raw``.
The slot of a server is its enabled stdio fingerprint, so any identity change,
delete, re-add or brand-new name is one lifecycle transition. ``previous_raw``
must be the raw config read *before* the mutation being committed, and must not
alias ``candidate_raw``.
A previous config whose server schema cannot be parsed is a conservative
migration: every candidate stdio server gets a fresh token, and tombstones are
kept only when the old ledger itself is valid. Unreadable JSON is the caller's
responsibility to abort before reaching this function.
"""
from deerflow.config.extensions_config import validate_raw_extensions_config
if candidate_config is None:
candidate_config = validate_raw_extensions_config(dict(candidate_raw))
new_fingerprints = _stdio_fingerprints(candidate_config)
previous_provable = previous_raw is not None
previous_fingerprints: dict[str, str] = {}
previous_ledger: McpLifecycleLedger | None = None
if previous_raw is not None:
try:
previous_fingerprints = _stdio_fingerprints(validate_raw_extensions_config(dict(previous_raw)))
except Exception: # noqa: BLE001 - schema-invalid previous config
previous_provable = False
try:
previous_ledger = parse_mcp_lifecycle(previous_raw)
except InvalidMcpLifecycle:
previous_provable = False
previous_ledger = None
ledger = dict(previous_ledger.servers) if previous_ledger is not None else {}
if not previous_provable:
for server_name in new_fingerprints:
ledger[server_name] = new_token()
return {"version": LIFECYCLE_VERSION, "servers": ledger}
for server_name in set(previous_fingerprints) | set(new_fingerprints) | set(ledger):
if previous_fingerprints.get(server_name) != new_fingerprints.get(server_name):
# identity change, delete or re-add: always a fresh generation
ledger[server_name] = new_token()
return {"version": LIFECYCLE_VERSION, "servers": ledger}

View File

@ -277,6 +277,7 @@ class MCPSessionPool:
*,
domain: MCPPoolDomain = "deployment",
retire_unlisted: bool = False,
force_rebind: Collection[str] = (),
) -> PreparedRetirement:
"""Apply already-classified binding changes and detach only changed owners.
@ -291,10 +292,21 @@ class MCPSessionPool:
baseline of their own — a restarted process, where a durable-task caller's
binding is the only record that a server exists — because enumerating
those names separately would race a concurrent ``ensure_binding()``.
``force_rebind`` names servers whose *shared lifecycle version* changed
even though ``active`` still reports the same connection fingerprint —
the cross-worker ``delete(A) -> identical re-add(A)`` case, where the
final effective configuration is indistinguishable from the applied one.
A forced server is re-epoched and has its owners detached in this same
critical section, exactly like a fingerprint change. Names are scoped to
``domain``: forcing a ``deployment`` server never touches a same-named
``personal`` binding. Callers must pass a name only once per observed
generation; passing it again would re-epoch an already-stable server.
"""
entries: list[tuple[ClientSession, asyncio.AbstractEventLoop, asyncio.Task[Any], asyncio.Event]] = []
inflight: list[tuple[asyncio.AbstractEventLoop, asyncio.Future[ClientSession], asyncio.Task[Any], asyncio.Event]] = []
changed: set[str] = set()
forced = set(force_rebind)
with self._lock:
if retire_unlisted:
@ -307,7 +319,7 @@ class MCPSessionPool:
binding_key = (domain, server_name)
self._binding_lifecycle_servers.add(binding_key)
current = self._bindings.get(binding_key)
if current is not None and current.fingerprint == fingerprint:
if current is not None and current.fingerprint == fingerprint and server_name not in forced:
continue
self._install_binding_locked(binding_key, fingerprint)
changed.add(server_name)
@ -979,6 +991,18 @@ def get_session_pool() -> MCPSessionPool:
return _pool
def current_session_pool() -> MCPSessionPool | None:
"""Return the live pool singleton without creating one.
``_pool`` is published and replaced under ``_pool_lock``. Callers that only
need to observe retained state (for example the MCP cache staleness check)
must not read the private global unsynchronized, and must not create a pool
as a side effect of a read.
"""
with _pool_lock:
return _pool
def reset_session_pool() -> MCPSessionPool | None:
"""Reset the singleton and return the retired pool, if any.

View File

@ -249,7 +249,12 @@ def test_delete_recovers_multiple_legacy_identity_collisions(capability_client):
before_raw = json.loads(path.read_text())
assert client.delete(f"/api/mcp/config/servers/{removed}").status_code == 200
del before_raw["mcpServers"][removed]
assert json.loads(path.read_text()) == before_raw
written = json.loads(path.read_text())
assert written["mcpServers"] == before_raw["mcpServers"]
assert written.get("skills") == before_raw.get("skills")
# The controlled writer commits the shared lifecycle ledger in the same write; every
# server here is HTTP, so there are no stdio entries and it stays empty.
assert written.get("mcpLifecycle") == {"version": 1, "servers": {}}
items = client.get("/api/capabilities/installations/mcp").json()["items"]
ambiguous = [item for item in items if not item["selectable"]]
assert len(ambiguous) == {"unrelated": 4, "legacy-peer": 2, "pair-two-b": 0}[removed]

View File

@ -17,6 +17,7 @@ import pytest
from _thread_checkpoint_helpers import INDEXED_SAVER_KINDS, SAVER_KINDS, make_saver, put_goal_write, put_thread
from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage, SystemMessage, ToolMessage # noqa: F401
from langchain_core.tools import StructuredTool
from test_mcp_cache_reconciliation import reconciler # noqa: F401 - fixture re-export
from app.gateway.routers.mcp import McpConfigResponse
from app.gateway.routers.memory import MemoryConfigResponse, MemoryStatusResponse
@ -2500,6 +2501,73 @@ class TestGoalManagement:
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_update_mcp_config_installs_the_local_committed_fence(
client,
reconciler, # noqa: F811 - fixture imported from the sibling suite
monkeypatch,
tmp_path,
):
"""A client write must retire the writer process's own binding, not just publish the token."""
import test_mcp_cache_reconciliation as harness
from deerflow.mcp import session_pool as session_pool_module
from deerflow.mcp.session_pool import MCPSessionPool
cfg = tmp_path / "extensions_config.json"
pool = MCPSessionPool()
session_pool_module._pool = pool
log = harness._session_log()
servers = {"A": harness._stdio("cmd-A")}
await harness._initialize(monkeypatch, cfg, servers, log)
binding_before = harness._binding(pool, "A")
# Delete A, then identically re-add it through the client writer. The
# committing process must install the new epochs under the same lock.
client.update_mcp_config({})
client.update_mcp_config({"A": {"enabled": True, "type": "stdio", "command": "cmd-A"}})
await harness._wait_until(lambda: log["exited"].get("cmd-A") == 1)
assert harness._binding(pool, "A") is not binding_before
@pytest.mark.asyncio
async def test_update_mcp_config_post_commit_failure_still_resets_agent(
client,
reconciler, # noqa: F811 - fixture imported from the sibling suite
monkeypatch,
tmp_path,
):
"""A committed write whose local handoff fails must not keep the old cached agent."""
import test_mcp_cache_reconciliation as harness
from deerflow.mcp import session_pool as session_pool_module
from deerflow.mcp.session_pool import MCPSessionPool
cfg = tmp_path / "extensions_config.json"
pool = MCPSessionPool()
session_pool_module._pool = pool
log = harness._session_log()
await harness._initialize(monkeypatch, cfg, {"A": harness._stdio("cmd-A")}, log)
client._agent = "cached-agent"
client._agent_config_key = ("old",)
def _boom(*_args, **_kwargs):
raise RuntimeError("prepare failed")
monkeypatch.setattr("deerflow.mcp.cache.prepare_mcp_reconciliation", _boom)
with pytest.raises(RuntimeError, match="saved, but local cache reconciliation failed"):
client.update_mcp_config({"A": {"enabled": True, "type": "stdio", "command": "cmd-A2"}})
# The config was committed...
assert json.loads(cfg.read_text(encoding="utf-8"))["mcpServers"]["A"]["command"] == "cmd-A2"
# ...so the old agent must not be reusable, and the local pool is retired.
assert client._agent is None
assert session_pool_module.get_session_pool() is not pool
class TestMcpConfig:
def test_get_mcp_config(self, client):
server = MagicMock()
@ -2546,7 +2614,7 @@ class TestMcpConfig:
finally:
tmp_path.unlink()
def test_update_mcp_config_preserves_raw_sibling_keys(self, client, tmp_path, monkeypatch):
def test_update_mcp_config_preserves_raw_sibling_keys(self, client, reconciler, tmp_path, monkeypatch): # noqa: F811 - imported fixture
"""Only ``mcpServers`` is replaced; every other key keeps its on-disk ``$VAR`` form."""
monkeypatch.setenv("DEERFLOW_TEST_GH_TOKEN", "ghp_live_secret_value")
config_file = tmp_path / "extensions_config.json"
@ -2567,13 +2635,37 @@ class TestMcpConfig:
):
client.update_mcp_config({"new": {"type": "stdio", "command": "uvx", "env": {"TOKEN": "$DEERFLOW_TEST_GH_TOKEN"}}})
written_text = config_file.read_text(encoding="utf-8")
assert json.loads(written_text) == {
"mcpServers": {"new": {"type": "stdio", "command": "uvx", "env": {"TOKEN": "$DEERFLOW_TEST_GH_TOKEN"}}},
"mcpInterceptors": {"auth": "$DEERFLOW_TEST_GH_TOKEN"},
"skills": {"kept": {"enabled": False}},
}
assert "ghp_live_secret_value" not in written_text
written = json.loads(config_file.read_text(encoding="utf-8"))
assert written["mcpServers"] == {"new": {"type": "stdio", "command": "uvx", "env": {"TOKEN": "$DEERFLOW_TEST_GH_TOKEN"}}}
assert written["mcpInterceptors"] == {"auth": "$DEERFLOW_TEST_GH_TOKEN"}
assert written["skills"] == {"kept": {"enabled": False}}
# The controlled writer also commits the shared lifecycle ledger: "old"
# was removed (tombstone) and "new" was added, both with fresh tokens.
lifecycle = written["mcpLifecycle"]
assert lifecycle["version"] == 1
assert set(lifecycle["servers"]) == {"old", "new"}
assert all(len(token) == 32 and all(c in "0123456789abcdef" for c in token) for token in lifecycle["servers"].values())
# Raw $VAR placeholders are never expanded into the file.
assert "ghp_live_secret_value" not in config_file.read_text(encoding="utf-8")
def test_update_mcp_config_delete_then_identical_readd_advances_generation(self, client, reconciler, tmp_path, monkeypatch): # noqa: F811 - imported fixture
"""The client writer must not coalesce delete -> identical re-add."""
config_file = tmp_path / "extensions_config.json"
config_file.write_text(
json.dumps({"mcpServers": {"A": {"enabled": True, "type": "stdio", "command": "cmd-A"}}, "skills": {}}),
encoding="utf-8",
)
with (
patch("deerflow.client.ExtensionsConfig.resolve_config_path", return_value=config_file),
patch("deerflow.client.reload_extensions_config", return_value=ExtensionsConfig()),
):
client.update_mcp_config({})
after_delete = json.loads(config_file.read_text(encoding="utf-8"))["mcpLifecycle"]["servers"]["A"]
client.update_mcp_config({"A": {"enabled": True, "type": "stdio", "command": "cmd-A"}})
after_readd = json.loads(config_file.read_text(encoding="utf-8"))["mcpLifecycle"]["servers"]["A"]
assert after_readd != after_delete
def test_update_mcp_config_rejects_invalid_candidate_without_writing(self, client, tmp_path):
config_file = tmp_path / "extensions_config.json"

View File

@ -21,6 +21,7 @@ from pathlib import Path
import pytest
from dotenv import load_dotenv
from test_mcp_cache_reconciliation import reconciler # noqa: F401 - fixture re-export
from deerflow.client import DeerFlowClient, StreamEvent
from deerflow.config.app_config import AppConfig
@ -698,7 +699,7 @@ class TestConfigManagement:
assert "mcp_servers" in result
assert isinstance(result["mcp_servers"], dict)
def test_update_mcp_config_writes_and_invalidates(self, e2e_env, tmp_path, monkeypatch):
def test_update_mcp_config_writes_and_invalidates(self, e2e_env, reconciler, tmp_path, monkeypatch): # noqa: F811 - imported fixture
"""update_mcp_config() writes extensions_config.json and invalidates the agent."""
# Set up a writable extensions_config.json
config_file = tmp_path / "extensions_config.json"

View File

@ -104,6 +104,62 @@ async def test_reconcile_detaches_only_changed_server_and_preserves_registry_key
assert cm_a.exit_task is cm_a.enter_task
@pytest.mark.asyncio
async def test_reconcile_force_rebind_reepochs_same_fingerprint_and_preserves_others():
"""A shared generation change re-epochs A even though its fingerprint is unchanged.
``force_rebind`` lets a caller that already knows a
server's shared lifecycle version changed install a fresh binding in the
same non-awaiting critical section, without a second teardown owner and
without touching any other domain or server.
"""
pool = MCPSessionPool()
loop = asyncio.get_running_loop()
a = pool.ensure_binding("A", "same-fp")
b = pool.ensure_binding("B", "b-fp")
personal_a = pool.ensure_binding("A", "same-fp", domain="personal")
cm_a = _GatedSessionCm()
cm_b = _GatedSessionCm()
cm_personal = _GatedSessionCm()
with patch("langchain_mcp_adapters.sessions.create_session", side_effect=[cm_a, cm_b, cm_personal]):
session_a = await pool.get_session("A", "u:t", _CONNECTION, binding=a)
session_b = await pool.get_session("B", "u:t", _CONNECTION, binding=b)
session_personal = await pool.get_session("A", "u:p", _CONNECTION, binding=personal_a)
prepared = pool.reconcile_bindings({"A": "same-fp", "B": "b-fp"}, (), force_rebind={"A"})
current_a = pool._bindings[("deployment", "A")]
assert current_a is not a
assert current_a.fingerprint == "same-fp"
assert current_a.epoch > a.epoch
# Only the forced server changed; its sibling and the personal same-name
# binding keep their exact capability objects.
assert pool._bindings[("deployment", "B")] is b
assert pool._bindings[("personal", "A")] is personal_a
# Only the forced server's established session is detached and signalled.
assert [entry[0] for entry in prepared.entries] == [session_a]
assert prepared.inflight == ()
assert pool._entries[("B", "u:t", loop, "deployment")][0] is session_b
assert pool._entries[("A", "u:p", loop, "personal")][0] is session_personal
assert cm_b.closed is False
assert cm_personal.closed is False
# The old capability can no longer create or return the session.
with pytest.raises(StaleMCPBindingError):
await pool.get_session("A", "u:t", _CONNECTION, binding=a)
await asyncio.wait_for(prepared.entries[0][2], timeout=1)
assert cm_a.closed is True
# Re-consuming an already-applied generation is a no-op: A stays stable.
again = pool.reconcile_bindings({"A": "same-fp", "B": "b-fp"}, ())
assert again.entries == ()
assert pool._bindings[("deployment", "A")] is current_a
@pytest.mark.asyncio
async def test_stale_binding_fails_before_session_creation():
pool = MCPSessionPool()

View File

@ -98,6 +98,11 @@ def cache_globals():
cache_module._init_condition = threading.Condition(cache_module._init_lock)
cache_module._initializing_generation = None
cache_module._cache_generation = 0
# ``_has_mcp_reconciliation_state`` now also counts durable-only deployment
# bindings, so "never initialized" requires a fresh pool singleton too.
from deerflow.mcp.session_pool import reset_session_pool
reset_session_pool()
try:
yield
@ -1126,7 +1131,7 @@ def test_initialization_without_a_readable_snapshot_discards_result(cache_global
return ["loaded-tools"]
monkeypatch.setattr("deerflow.mcp.tools.get_mcp_tools", _fake_get_mcp_tools)
monkeypatch.setattr(cache_module, "_read_stable_mcp_snapshot", lambda path, signature: None)
monkeypatch.setattr(cache_module, "_read_stable_mcp_revision", lambda path, signature: None)
result = asyncio.run(cache_module.initialize_mcp_tools())

View File

@ -1239,11 +1239,11 @@ async def test_failed_reconciliation_does_not_commit_a_new_baseline(reconciler,
real_reconcile = pool.reconcile_bindings
fail_next = {"armed": True}
def _maybe_fail(active, removed, *, domain="deployment"):
def _maybe_fail(active, removed, *, domain="deployment", retire_unlisted=False, force_rebind=()):
if fail_next["armed"]:
fail_next["armed"] = False
raise RuntimeError("reconcile failed")
return real_reconcile(active, removed, domain=domain)
return real_reconcile(active, removed, domain=domain, retire_unlisted=retire_unlisted, force_rebind=force_rebind)
monkeypatch.setattr(pool, "reconcile_bindings", _maybe_fail)
_write_config(cfg, {"A": _stdio("cmd-A2")})
@ -1399,11 +1399,11 @@ async def test_concurrent_reconcilers_serialize_read_and_apply(reconciler, monke
release = threading.Event()
calls: list[str] = []
def _slow_reconcile(active, removed, *, domain="deployment"):
def _slow_reconcile(active, removed, *, domain="deployment", retire_unlisted=False, force_rebind=()):
calls.append("enter")
entered.set()
release.wait(timeout=5)
result = real_reconcile(active, removed, domain=domain)
result = real_reconcile(active, removed, domain=domain, retire_unlisted=retire_unlisted, force_rebind=force_rebind)
calls.append("exit")
return result
@ -2072,7 +2072,7 @@ async def test_first_claim_atomically_tombstones_a_concurrently_retained_server(
real_reconcile = pool.reconcile_bindings
interleaved = {"done": False, "binding": None}
def _reconcile_with_racing_retainer(active, removed, *, domain="deployment", retire_unlisted=False):
def _reconcile_with_racing_retainer(active, removed, *, domain="deployment", retire_unlisted=False, force_rebind=()):
if not interleaved["done"]:
interleaved["done"] = True
# The durable caller slips in after the cache listed the retained
@ -2085,6 +2085,8 @@ async def test_first_claim_atomically_tombstones_a_concurrently_retained_server(
kwargs = {"domain": domain}
if retire_unlisted:
kwargs["retire_unlisted"] = True
if force_rebind:
kwargs["force_rebind"] = force_rebind
return real_reconcile(active, removed, **kwargs)
monkeypatch.setattr(pool, "reconcile_bindings", _reconcile_with_racing_retainer)
@ -2189,7 +2191,15 @@ async def test_rejected_revision_installs_no_epoch_and_no_tombstone(reconciler,
published = await cache_module.initialize_mcp_tools()
assert [tool.name for tool in published] == ["A"]
assert cache_module._cache_initialized is True
assert _binding(pool, "A") is durable
# The first successful claim with no trusted applied baseline
# conservatively re-epochs every active deployment binding, because a
# same-fingerprint ABA is indistinguishable from a compatible rebind.
rebound = _binding(pool, "A")
assert rebound is not durable
assert rebound.epoch > durable.epoch
assert rebound.fingerprint == startup_fingerprint
with pytest.raises(StaleMCPBindingError):
await pool.get_session("A", "task:1", startup_connection, binding=durable)
assert session_pool_module.get_session_pool() is pool
@ -2355,3 +2365,104 @@ async def test_reader_waiting_for_discovery_still_cleans_a_retired_pool(reconcil
assert pool._entries == {}
assert pool._inflight == {}
assert outcome.get("tools") == []
def test_committed_cold_path_fences_binding_inserted_after_the_retained_check(reconciler, tmp_path):
"""A durable bind that lands after the empty retained check must not be recorded as applied.
``_plan_committed_mcp_reconciliation_locked`` observes the pool without
holding its lock, so a durable-task caller can install a binding between the
"no local state" check and the baseline write. The cold path must still run
the domain-scoped reconciliation before recording ``_applied_mcp_revision``,
or the inserted binding survives while the process claims to have applied a
revision whose epochs were never installed.
"""
cfg = tmp_path / "extensions_config.json"
servers = {"A": _stdio("cmd-A")}
_write_config(cfg, servers)
candidate = ExtensionsConfig.from_file(str(cfg))
pool = MCPSessionPool()
session_pool_module._pool = pool
personal = pool.ensure_binding("A", "personal-fp", domain="personal")
real_retained = pool.retained_server_names
checked = threading.Event()
release = threading.Event()
def _retained(*, domain="deployment"):
# The check observes no deployment state, then the writer is parked
# before it acts on that observation.
observed = real_retained(domain=domain)
checked.set()
assert release.wait(timeout=5)
return observed
pool.retained_server_names = _retained # type: ignore[method-assign]
injected: dict[str, object] = {}
def _durable_bind():
assert checked.wait(timeout=5)
injected["binding"] = pool.ensure_binding("A", _fingerprint(servers, "A"), domain="deployment")
release.set()
thread = threading.Thread(target=_durable_bind)
thread.start()
pending = None
try:
pending = cache_module.prepare_mcp_reconciliation(candidate, config_path=cfg)
finally:
thread.join(timeout=5)
if pending is not None:
cache_module.finish_mcp_reconciliation(pending)
assert thread.is_alive() is False
inserted = injected["binding"]
current = pool._bindings[("deployment", "A")]
# The concurrently inserted binding must be superseded before the baseline
# is recorded: it can never be reported as an installed epoch.
assert current is not inserted
assert current.epoch > inserted.epoch
assert cache_module._applied_mcp_revision is not None
# The deployment-only reconciliation must leave the personal domain alone.
assert pool._bindings[("personal", "A")] is personal
@pytest.mark.asyncio
async def test_tombstone_only_change_advances_applied_tokens(reconciler, monkeypatch, tmp_path):
"""A history change that only moves a tombstone must advance the applied token map."""
from deerflow.mcp.lifecycle import LIFECYCLE_KEY, plan_mcp_lifecycle
cfg = tmp_path / "extensions_config.json"
pool = MCPSessionPool()
session_pool_module._pool = pool
log = _session_log()
servers = {"A": _stdio("cmd-A"), "B": _stdio("cmd-B")}
await _initialize(monkeypatch, cfg, servers, log)
# A controlled writer deletes A, recording a tombstone token.
raw_ab = {"mcpServers": servers, "skills": {}}
raw_b = {"mcpServers": {"B": _stdio("cmd-B")}, "skills": {}}
raw_b[LIFECYCLE_KEY] = plan_mcp_lifecycle(raw_ab, raw_b)
cfg.write_text(json.dumps(raw_b), encoding="utf-8")
assert cache_module.refresh_mcp_cache_if_active() is True
tombstone = cache_module._applied_mcp_revision.lifecycle_tokens["A"]
binding_b = _binding(pool, "B")
entry_b = _entry(pool, "B")
# W2 re-adds A then deletes it again: B is unchanged, but A's tombstone moves.
raw_ab2 = {"mcpServers": {"A": _stdio("cmd-A"), "B": _stdio("cmd-B")}, "skills": {}}
raw_ab2[LIFECYCLE_KEY] = plan_mcp_lifecycle(raw_b, raw_ab2)
raw_b2 = {"mcpServers": {"B": _stdio("cmd-B")}, "skills": {}}
raw_b2[LIFECYCLE_KEY] = plan_mcp_lifecycle(raw_ab2, raw_b2)
cfg.write_text(json.dumps(raw_b2), encoding="utf-8")
assert raw_b2[LIFECYCLE_KEY]["servers"]["A"] != tombstone
# No active server changed, so this is a no-op for sessions...
assert cache_module.refresh_mcp_cache_if_active() is False
assert _binding(pool, "B") is binding_b
assert _entry(pool, "B") is entry_b
# ...but the applied token map must not lag the consumed shared version.
assert cache_module._applied_mcp_revision.lifecycle_tokens["A"] == raw_b2[LIFECYCLE_KEY]["servers"]["A"]

View File

@ -0,0 +1,361 @@
"""Cross-worker (two-process) coverage for missed-history MCP ABA.
Problem this pins
-----------------
MCP reconciliation is *process-local*. A Gateway worker installs a binding
epoch only for the transitions it personally observes. A second worker sharing
the same ``extensions_config.json`` never runs that write, so the only thing it
can observe is the file it finds on its next staleness check.
A single process that commits A(v1) -> delete(A) -> A(v1) still advances A's
epoch: the committed planner compares each committed revision against the applied
baseline and therefore sees the intermediate delete. That guarantee does not
survive across workers: W2's final file is equivalent to the revision W1 already
applied, so the read-side planner's "effective configuration unchanged" fast path
keeps A's old session.
This file is the end-to-end acceptance test for the shared ``mcpLifecycle``
generation ledger. The writer is a real ``subprocess`` with its own
module globals, cache and session pool, committing through the same primitive the
Gateway routes use (``plan_mcp_lifecycle`` -> atomic write -> committed fence).
Asserting against one shared in-memory cache would not prove cross-process
behavior, so it is not done here. No test-side token injection or explicit reset
is used to make these pass.
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
import textwrap
from pathlib import Path
import pytest
# Reuse the process-local harness (config writer, recording sessions, fake
# discovery) so reader state is built exactly like the sibling suite.
from test_mcp_cache_reconciliation import ( # noqa: F401 (``reconciler`` fixture)
_binding,
_entry,
_fingerprint,
_initialize,
_session_log,
_stdio,
_wait_until,
_write_config,
reconciler,
)
import deerflow.mcp.cache as cache_module
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.mcp import session_pool as session_pool_module
from deerflow.mcp.session_pool import MCPSessionPool, StaleMCPBindingError
# One Gateway worker committing A's delete + identical re-add against the shared
# config, in a separate interpreter. It prints its own epochs so the reader can
# prove the writer genuinely retired A before asserting that the reader does not.
_PEER_WORKER_PROGRAM = textwrap.dedent(
"""
import copy
import json
import os
from pathlib import Path
from deerflow.config.extensions_config import (
ExtensionsConfig,
atomic_write_extensions_config,
extensions_config_file_lock,
extensions_config_write_lock,
reload_extensions_config,
validate_raw_extensions_config,
)
from deerflow.mcp import cache as cache_module
from deerflow.mcp import session_pool as session_pool_module
from deerflow.mcp.client import build_servers_config
from deerflow.mcp.lifecycle import LIFECYCLE_KEY, plan_mcp_lifecycle
from deerflow.mcp.session_pool import normalized_connection_fingerprint
cfg = Path(os.environ["DEER_FLOW_EXTENSIONS_CONFIG_PATH"])
raw = json.loads(cfg.read_text(encoding="utf-8"))
original_a = raw["mcpServers"]["A"]
# W2 is a live worker: it already holds deployment bindings for A and B.
pool = session_pool_module.get_session_pool()
baseline = ExtensionsConfig.from_file(str(cfg))
for name, connection in build_servers_config(baseline).items():
pool.ensure_binding(
name,
normalized_connection_fingerprint(connection),
domain="deployment",
)
cache_module.finish_mcp_reconciliation(
cache_module.prepare_mcp_reconciliation(baseline, config_path=cfg)
)
a_epoch_baseline = pool._bindings[("deployment", "A")].epoch
b_epoch_baseline = pool._bindings[("deployment", "B")].epoch
def commit(previous, candidate):
# Mirrors the controlled writer: validate the exact candidate, persist
# config + ledger atomically, then install the fence under the config
# write lock and finish teardown after releasing it.
candidate[LIFECYCLE_KEY] = plan_mcp_lifecycle(previous, candidate)
candidate_model = validate_raw_extensions_config(candidate)
with extensions_config_write_lock, extensions_config_file_lock(cfg):
atomic_write_extensions_config(cfg, candidate)
reload_extensions_config(str(cfg))
pending = cache_module.prepare_mcp_reconciliation(
candidate_model, config_path=cfg
)
cache_module.finish_mcp_reconciliation(pending)
# delete A (candidate is independent of the previous raw dict)
previous = copy.deepcopy(raw)
candidate = copy.deepcopy(raw)
del candidate["mcpServers"]["A"]
commit(previous, candidate)
a_epoch_after_delete = pool._bindings[("deployment", "A")].epoch
# identical re-add, same declaration order as the original file
previous = copy.deepcopy(candidate)
candidate = copy.deepcopy(candidate)
candidate["mcpServers"] = {"A": original_a, **candidate["mcpServers"]}
commit(previous, candidate)
a_epoch_after_readd = pool._bindings[("deployment", "A")].epoch
b_epoch_after_readd = pool._bindings[("deployment", "B")].epoch
print(json.dumps({
"a_epoch_baseline": a_epoch_baseline,
"a_epoch_after_delete": a_epoch_after_delete,
"a_epoch_after_readd": a_epoch_after_readd,
"b_epoch_baseline": b_epoch_baseline,
"b_epoch_after_readd": b_epoch_after_readd,
}))
"""
)
# A second worker committing the same ABA through the real Gateway mutation
# routes (not the plan/write primitive), so the cross-process test covers the
# full controlled-writer protocol: locks -> read -> candidate -> plan ->
# validate -> write -> committed fence.
_GATEWAY_WRITER_PROGRAM = textwrap.dedent(
"""
import json
import os
from pathlib import Path
from app.gateway.routers import mcp as mcp_router
from app.gateway.routers.mcp import McpConfigUpdateRequest, McpServerConfigResponse
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.mcp import cache as cache_module
from deerflow.mcp import session_pool as session_pool_module
from deerflow.mcp.client import build_servers_config
from deerflow.mcp.session_pool import normalized_connection_fingerprint
cfg = Path(os.environ["DEER_FLOW_EXTENSIONS_CONFIG_PATH"])
pool = session_pool_module.get_session_pool()
baseline = ExtensionsConfig.from_file(str(cfg))
for name, connection in build_servers_config(baseline).items():
pool.ensure_binding(
name,
normalized_connection_fingerprint(connection),
domain="deployment",
)
cache_module.finish_mcp_reconciliation(
cache_module.prepare_mcp_reconciliation(baseline, config_path=cfg)
)
a_epoch_baseline = pool._bindings[("deployment", "A")].epoch
b_epoch_baseline = pool._bindings[("deployment", "B")].epoch
mcp_router._apply_mcp_server_delete("A")
a_epoch_after_delete = pool._bindings[("deployment", "A")].epoch
mcp_router._apply_mcp_servers_create(
McpConfigUpdateRequest(
mcp_servers={
"A": McpServerConfigResponse(enabled=True, type="stdio", command="npx")
}
)
)
a_epoch_after_readd = pool._bindings[("deployment", "A")].epoch
b_epoch_after_readd = pool._bindings[("deployment", "B")].epoch
print(json.dumps({
"a_epoch_baseline": a_epoch_baseline,
"a_epoch_after_delete": a_epoch_after_delete,
"a_epoch_after_readd": a_epoch_after_readd,
"b_epoch_baseline": b_epoch_baseline,
"b_epoch_after_readd": b_epoch_after_readd,
}))
"""
)
def _run_worker(program: str, config_path: Path) -> dict:
"""Run one writer program in a second, independent interpreter."""
env = os.environ.copy()
env["DEER_FLOW_EXTENSIONS_CONFIG_PATH"] = str(config_path)
completed = subprocess.run(
[sys.executable, "-c", program],
cwd=str(Path(__file__).resolve().parent.parent),
env=env,
capture_output=True,
text=True,
timeout=120,
)
assert completed.returncode == 0, f"peer worker failed ({completed.returncode})\n--- stdout ---\n{completed.stdout}\n--- stderr ---\n{completed.stderr}"
return json.loads(completed.stdout.strip().splitlines()[-1])
def _run_peer_worker(config_path: Path) -> dict:
"""Commit delete(A) + identical re-add(A) through the plan/commit primitive."""
return _run_worker(_PEER_WORKER_PROGRAM, config_path)
def _run_gateway_writer(config_path: Path) -> dict:
"""Commit delete(A) + identical re-add(A) through the real Gateway routes."""
return _run_worker(_GATEWAY_WRITER_PROGRAM, config_path)
@pytest.mark.asyncio
async def test_peer_worker_delete_then_identical_readd_retires_only_that_server(
reconciler, # noqa: F811 - fixture imported from the sibling process-local suite
monkeypatch,
tmp_path,
):
"""A worker that never saw the delete must still retire A, and only A."""
cfg = tmp_path / "extensions_config.json"
pool = MCPSessionPool()
session_pool_module._pool = pool
log = _session_log()
servers = {"A": _stdio("cmd-A"), "B": _stdio("cmd-B")}
await _initialize(monkeypatch, cfg, servers, log)
snapshot_before = cache_module._mcp_config_snapshot
binding_a_before = _binding(pool, "A")
binding_b_before = _binding(pool, "B")
entry_b_before = _entry(pool, "B")
peer = _run_peer_worker(cfg)
# Premise: W2 really committed the history and retired A locally. Its A
# advances through a tombstone and a fresh binding while B is untouched.
assert peer["a_epoch_after_delete"] > peer["a_epoch_baseline"]
assert peer["a_epoch_after_readd"] > peer["a_epoch_after_delete"]
assert peer["b_epoch_after_readd"] == peer["b_epoch_baseline"]
# Premise: the final effective MCP slice W1 can read is the one it already
# applied, yet the file (and its lifecycle ledger) changed underneath it.
final_config = ExtensionsConfig.from_file(str(cfg))
assert cache_module._effective_mcp_config_snapshot(final_config) == snapshot_before
assert cache_module._get_config_signature(cfg) != cache_module._config_signature
# Contract: observing the shared lifecycle history retires A (new
# epoch, old session closed) while leaving B's binding/session intact.
changed = cache_module.refresh_mcp_cache_if_active()
assert changed is True
binding_a_after = _binding(pool, "A")
assert binding_a_after is not binding_a_before
assert binding_a_after.epoch > binding_a_before.epoch
assert _binding(pool, "B") is binding_b_before
assert _entry(pool, "B") is entry_b_before
await _wait_until(lambda: log["exited"].get("cmd-A") == 1)
assert log["exited"].get("cmd-B") is None
# Idempotency: the token is recorded in the applied revision, so observing
# the same generation again is a no-op (force_rebind is one-shot).
assert cache_module.refresh_mcp_cache_if_active() is False
assert _binding(pool, "A") is binding_a_after
@pytest.mark.asyncio
async def test_real_gateway_writer_delete_then_identical_readd_is_consumed_cross_process(
reconciler, # noqa: F811 - fixture imported from the sibling process-local suite
monkeypatch,
tmp_path,
):
"""The real Gateway delete/create writers must produce a cross-process-consumable ABA."""
cfg = tmp_path / "extensions_config.json"
pool = MCPSessionPool()
session_pool_module._pool = pool
log = _session_log()
# Initial order is B, A because the real create route appends the re-added A
# at the end, which then reproduces this exact declaration order.
servers = {"B": _stdio("cmd-B"), "A": _stdio("npx")}
await _initialize(monkeypatch, cfg, servers, log)
snapshot_before = cache_module._mcp_config_snapshot
binding_a_before = _binding(pool, "A")
binding_b_before = _binding(pool, "B")
entry_b_before = _entry(pool, "B")
peer = _run_gateway_writer(cfg)
assert peer["a_epoch_after_delete"] > peer["a_epoch_baseline"]
assert peer["a_epoch_after_readd"] > peer["a_epoch_after_delete"]
assert peer["b_epoch_after_readd"] == peer["b_epoch_baseline"]
final_config = ExtensionsConfig.from_file(str(cfg))
assert cache_module._effective_mcp_config_snapshot(final_config) == snapshot_before
assert cache_module.refresh_mcp_cache_if_active() is True
binding_a_after = _binding(pool, "A")
assert binding_a_after is not binding_a_before
assert _binding(pool, "B") is binding_b_before
assert _entry(pool, "B") is entry_b_before
await _wait_until(lambda: log["exited"].get("npx") == 1)
assert log["exited"].get("cmd-B") is None
assert cache_module.refresh_mcp_cache_if_active() is False
assert _binding(pool, "A") is binding_a_after
@pytest.mark.asyncio
async def test_durable_only_without_baseline_is_retired_conservatively(
reconciler, # noqa: F811 - fixture imported from the sibling process-local suite
monkeypatch,
tmp_path,
):
"""No published cache and no applied baseline -> conservative deployment rebind."""
cfg = tmp_path / "extensions_config.json"
servers = {"A": _stdio("cmd-A"), "B": _stdio("cmd-B")}
_write_config(cfg, servers)
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(cfg))
pool = MCPSessionPool()
session_pool_module._pool = pool
durable_a = pool.ensure_binding("A", _fingerprint(servers, "A"), domain="deployment")
durable_b = pool.ensure_binding("B", _fingerprint(servers, "B"), domain="deployment")
# Durable-only worker: state exists, but nothing was published and no
# applied revision was ever recorded.
assert cache_module._cache_initialized is False
assert cache_module._applied_mcp_revision is None
peer = _run_peer_worker(cfg)
assert peer["a_epoch_after_readd"] > peer["a_epoch_after_delete"] > peer["a_epoch_baseline"]
# No trustworthy baseline, so the whole deployment domain is conservatively
# re-epoched (personal state is untouched); selective preservation is not
# claimed for this first observation.
assert cache_module.refresh_mcp_cache_if_active() is True
rebound_a = _binding(pool, "A")
rebound_b = _binding(pool, "B")
assert rebound_a is not durable_a and rebound_a.epoch > durable_a.epoch
assert rebound_b is not durable_b and rebound_b.epoch > durable_b.epoch
assert cache_module._applied_mcp_revision is not None
assert cache_module._cache_initialized is False
with pytest.raises(StaleMCPBindingError):
await pool.get_session("A", "u:t", servers["A"], binding=durable_a)
# The conservative rebind installed a stable baseline: the same revision is
# now a no-op instead of resetting on every refresh.
assert cache_module.refresh_mcp_cache_if_active() is False

View File

@ -0,0 +1,161 @@
"""Unit coverage for the shared ``mcpLifecycle`` generation ledger."""
from __future__ import annotations
import pytest
from deerflow.config.extensions_config import validate_raw_extensions_config
from deerflow.mcp.lifecycle import (
LIFECYCLE_KEY,
InvalidMcpLifecycle,
_stdio_fingerprints,
effective_tokens,
legacy_token,
lifecycle_delta,
new_token,
parse_mcp_lifecycle,
plan_mcp_lifecycle,
)
def _stdio(command: str, **extra) -> dict:
return {"enabled": True, "type": "stdio", "command": command, **extra}
def _config(servers: dict) -> dict:
return {"mcpServers": servers, "skills": {}}
def _ledger(**servers: str) -> dict:
return {LIFECYCLE_KEY: {"version": 1, "servers": dict(servers)}}
def _fingerprint(config: dict, name: str) -> str:
return _stdio_fingerprints(validate_raw_extensions_config(config))[name]
# ---------------------------------------------------------------------------
# Strict parsing (tri-state)
# ---------------------------------------------------------------------------
def test_absent_ledger_is_legacy_none():
assert parse_mcp_lifecycle({"mcpServers": {}}) is None
assert parse_mcp_lifecycle(None) is None
def test_valid_ledger_parses_with_servers():
ledger = parse_mcp_lifecycle(_ledger(A="a" * 32, B="b" * 32))
assert ledger is not None
assert ledger.version == 1
assert ledger.servers == {"A": "a" * 32, "B": "b" * 32}
@pytest.mark.parametrize(
"block",
[
{"version": True, "servers": {}},
{"version": 2, "servers": {}},
{"version": "1", "servers": {}},
{"version": 1.0, "servers": {}},
{"version": 1, "servers": []},
{"version": 1, "servers": {"A": "A" * 32}}, # uppercase
{"version": 1, "servers": {"A": "abc"}}, # too short
{"version": 1, "servers": {"A": "a" * 31}}, # 31 chars
{"version": 1, "servers": {"A": "a" * 33}}, # 33 chars
{"version": 1, "servers": {"A": "a" * 32 + "\n"}}, # trailing newline
{"version": 1, "servers": {"A": "\n" + "a" * 32}}, # leading newline
{"version": 1, "servers": {"A": "a" * 32 + " "}}, # trailing space
{"version": 1, "servers": {"A": 1}}, # wrong type
{"version": 1, "servers": {"": "a" * 32}}, # empty name
{"version": 1, "servers": {}, "extra": True},
],
)
def test_invalid_ledger_fails_closed(block):
with pytest.raises(InvalidMcpLifecycle):
parse_mcp_lifecycle({LIFECYCLE_KEY: block})
# ---------------------------------------------------------------------------
# Legacy derivation and planner
# ---------------------------------------------------------------------------
def test_legacy_token_is_deterministic_and_name_fingerprint_sensitive():
fp = "f" * 64
assert legacy_token("A", fp) == legacy_token("A", fp)
assert legacy_token("A", fp) != legacy_token("B", fp)
assert legacy_token("A", fp) != legacy_token("A", "e" * 64)
assert len(legacy_token("A", fp)) == 64
def test_unchanged_legacy_servers_are_not_retired():
previous = _config({"A": _stdio("cmd-A1"), "B": _stdio("cmd-B1")})
candidate = _config({"A": _stdio("cmd-A2"), "B": _stdio("cmd-B1")})
plan = plan_mcp_lifecycle(previous, candidate)
assert plan["version"] == 1
# Only A changed; B stays a derived legacy token (no explicit entry).
assert set(plan["servers"]) == {"A"}
tokens = effective_tokens(validate_raw_extensions_config(candidate), parse_mcp_lifecycle(plan))
assert tokens["B"] == legacy_token("B", _fingerprint(candidate, "B"))
def test_delete_and_identical_readd_each_advance_the_generation():
original = _config({"A": _stdio("cmd-A1")})
first = plan_mcp_lifecycle(None, original)["servers"]["A"]
deleted = _config({})
after_delete = plan_mcp_lifecycle(original, deleted)["servers"]["A"]
assert after_delete != first
re_added = plan_mcp_lifecycle(deleted, original)["servers"]["A"]
assert re_added not in {first, after_delete}
def test_metadata_only_change_does_not_advance():
previous = _config({"A": _stdio("cmd-A1")})
candidate = _config({"A": _stdio("cmd-A1", description="renamed", tool_name_prefix=False)})
assert plan_mcp_lifecycle(previous, candidate)["servers"] == {}
def test_schema_invalid_previous_config_is_a_conservative_migration():
# Raw JSON is valid, but the server map has the wrong shape.
previous = {"mcpServers": "not-a-dict", "skills": {}}
candidate = _config({"A": _stdio("cmd-A1"), "B": _stdio("cmd-B1")})
plan = plan_mcp_lifecycle(previous, candidate)
assert set(plan["servers"]) == {"A", "B"}
assert all(token != "" for token in plan["servers"].values())
def test_effective_tokens_keeps_tombstones():
candidate = _config({"A": _stdio("cmd-A1")})
ledger = parse_mcp_lifecycle(_ledger(A="a" * 32, B="b" * 32))
tokens = effective_tokens(validate_raw_extensions_config(candidate), ledger)
assert tokens["A"] == "a" * 32 # explicit ledger token wins
assert tokens["B"] == "b" * 32 # tombstone retained
def test_lifecycle_delta_reports_only_changed_active_servers():
delta = lifecycle_delta(
previous_tokens={"A": "1" * 32, "B": "2" * 32},
candidate_tokens={"A": "3" * 32, "B": "2" * 32, "C": "4" * 32},
active_names={"A", "B"},
)
assert delta == frozenset({"A"})
def test_new_token_is_strict_hex():
token = new_token()
assert len(token) == 32
assert token == token.lower()
assert all(ch in "0123456789abcdef" for ch in token)
def test_token_rejects_trailing_newline():
"""``$`` would accept a trailing newline; the validator must use fullmatch."""
with pytest.raises(InvalidMcpLifecycle):
parse_mcp_lifecycle({LIFECYCLE_KEY: {"version": 1, "servers": {"A": "a" * 32 + "\n"}}})
assert parse_mcp_lifecycle({LIFECYCLE_KEY: {"version": 1, "servers": {"A": "a" * 32}}}) is not None

View File

@ -0,0 +1,138 @@
"""Invalid ``mcpLifecycle`` must abort lazy MCP initialization deterministically.
An unverifiable ledger can never be published, so a retry loop that keeps
re-running discovery is both a hang and a resource leak. These tests drive the
real ``get_cached_mcp_tools()`` entry in a subprocess (with a timeout) so a
regression cannot hang the pytest worker.
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
import textwrap
from pathlib import Path
import pytest
_CONFIG = {"mcpServers": {"A": {"enabled": True, "type": "stdio", "command": "npx"}}, "skills": {}}
# Aborts on an unverifiable ledger before discovery: the fake discovery counter
# must stay at 0, and a pre-existing durable binding must be retired.
_INVALID_PROGRAM = textwrap.dedent(
"""
import json
import os
from pathlib import Path
import deerflow.mcp.tools as tools_module
from deerflow.mcp import cache as cache_module
from deerflow.mcp import session_pool as session_pool_module
cfg = Path(os.environ["DEER_FLOW_EXTENSIONS_CONFIG_PATH"])
pool = session_pool_module.get_session_pool()
pool.ensure_binding("A", "durable-fp", domain="deployment")
calls = {"n": 0}
async def _fake_get_mcp_tools(**_kwargs):
calls["n"] += 1
return ["should-not-publish"]
tools_module.get_mcp_tools = _fake_get_mcp_tools
result = cache_module.get_cached_mcp_tools()
print(json.dumps({
"result": len(result),
"discovery_calls": calls["n"],
"old_pool_retired": pool._retired,
"pool_singleton_is_none": session_pool_module._pool is None,
}))
"""
)
# The same call must recover once the ledger is fixed, without a restart.
_RECOVERY_PROGRAM = textwrap.dedent(
"""
import json
import os
from pathlib import Path
import deerflow.mcp.tools as tools_module
from deerflow.mcp import cache as cache_module
cfg = Path(os.environ["DEER_FLOW_EXTENSIONS_CONFIG_PATH"])
calls = {"n": 0}
async def _fake_get_mcp_tools(**_kwargs):
calls["n"] += 1
return ["tool"]
tools_module.get_mcp_tools = _fake_get_mcp_tools
first = cache_module.get_cached_mcp_tools()
first_calls = calls["n"]
cfg.write_text(os.environ["DEER_FLOW_VALID_CONFIG"], encoding="utf-8")
second = cache_module.get_cached_mcp_tools()
print(json.dumps({
"first": len(first),
"first_calls": first_calls,
"second": len(second),
"total_calls": calls["n"],
}))
"""
)
def _run_program(program: str, cfg: Path) -> dict:
env = os.environ.copy()
env["DEER_FLOW_EXTENSIONS_CONFIG_PATH"] = str(cfg)
env["DEER_FLOW_VALID_CONFIG"] = json.dumps(_CONFIG)
try:
completed = subprocess.run(
[sys.executable, "-c", program],
cwd=str(Path(__file__).resolve().parent.parent),
env=env,
capture_output=True,
text=True,
timeout=30,
)
except subprocess.TimeoutExpired:
pytest.fail("get_cached_mcp_tools() did not terminate on an invalid mcpLifecycle")
assert completed.returncode == 0, f"worker failed ({completed.returncode})\n--- stdout ---\n{completed.stdout}\n--- stderr ---\n{completed.stderr}"
return json.loads(completed.stdout.strip().splitlines()[-1])
@pytest.mark.parametrize(
"ledger",
[
{"version": 2, "servers": {}},
{"version": 1, "servers": {"A": "a" * 32 + "\n"}},
],
)
def test_invalid_lifecycle_aborts_lazy_initialization_without_discovery(ledger, tmp_path):
cfg = tmp_path / "extensions_config.json"
cfg.write_text(json.dumps({**_CONFIG, "mcpLifecycle": ledger}), encoding="utf-8")
payload = _run_program(_INVALID_PROGRAM, cfg)
assert payload["result"] == 0
assert payload["discovery_calls"] == 0 # discovery is never entered
assert payload["old_pool_retired"] is True # stale bindings are not reused
assert payload["pool_singleton_is_none"] is True
def test_invalid_lifecycle_recovers_after_the_config_is_fixed(tmp_path):
cfg = tmp_path / "extensions_config.json"
cfg.write_text(json.dumps({**_CONFIG, "mcpLifecycle": {"version": 2, "servers": {}}}), encoding="utf-8")
payload = _run_program(_RECOVERY_PROGRAM, cfg)
assert payload["first"] == 0
assert payload["first_calls"] == 0
assert payload["second"] == 1 # valid config initializes normally
assert payload["total_calls"] == 1

View File

@ -0,0 +1,196 @@
"""Gateway writer coverage for the shared ``mcpLifecycle`` generation."""
from __future__ import annotations
import json
import pytest
# Reuse the process-local isolation fixture so these writer tests do not leak
# cache globals or the session-pool singleton.
from test_mcp_cache_reconciliation import reconciler # noqa: F401
from app.gateway.routers import mcp as mcp_router
from app.gateway.routers.mcp import (
McpConfigUpdateRequest,
McpServerConfigResponse,
McpServerConfigUpdateRequest,
McpServerStateUpdateRequest,
)
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.mcp.lifecycle import _stdio_fingerprints, legacy_token, parse_mcp_lifecycle
def _stdio(command: str) -> dict:
return {"enabled": True, "type": "stdio", "command": command}
def _write(cfg, servers: dict) -> None:
cfg.write_text(json.dumps({"mcpServers": servers, "skills": {}}), encoding="utf-8")
def _ledger(cfg):
return parse_mcp_lifecycle(json.loads(cfg.read_text(encoding="utf-8")))
def _legacy(cfg, name: str) -> str:
fingerprint = _stdio_fingerprints(ExtensionsConfig.from_file(str(cfg)))[name]
return legacy_token(name, fingerprint)
def test_state_disable_and_reenable_advance_the_generation(
reconciler, # noqa: F811 - fixture imported from the sibling suite
monkeypatch,
tmp_path,
):
"""Raw in-place mutation must not hide the enabled -> disabled transition."""
cfg = tmp_path / "extensions_config.json"
_write(cfg, {"A": _stdio("npx")})
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(cfg))
baseline = _legacy(cfg, "A")
mcp_router._apply_mcp_server_state_update(McpServerStateUpdateRequest(server_name="A", enabled=False))
disabled = _ledger(cfg).servers["A"]
assert disabled != baseline
mcp_router._apply_mcp_server_state_update(McpServerStateUpdateRequest(server_name="A", enabled=True))
reenabled = _ledger(cfg).servers["A"]
assert reenabled not in {baseline, disabled}
def test_delete_then_identical_readd_advance_the_generation(
reconciler, # noqa: F811 - fixture imported from the sibling suite
monkeypatch,
tmp_path,
):
"""Delete tombstones the token and an identical re-add must rotate it again."""
cfg = tmp_path / "extensions_config.json"
_write(cfg, {"A": _stdio("npx")})
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(cfg))
mcp_router._apply_mcp_server_delete("A")
tombstone = _ledger(cfg).servers["A"]
mcp_router._apply_mcp_servers_create(
McpConfigUpdateRequest(
mcp_servers={"A": McpServerConfigResponse(enabled=True, type="stdio", command="npx")},
)
)
rebound = _ledger(cfg).servers["A"]
assert rebound != tombstone
def test_commit_helper_requires_previous_raw():
"""A controlled write must not be able to skip the lifecycle commit."""
import inspect
parameters = inspect.signature(mcp_router._commit_mcp_config_write).parameters
assert parameters["previous_raw"].default is inspect.Parameter.empty
def test_bulk_update_rotates_only_the_changed_server_token(
reconciler, # noqa: F811 - fixture imported from the sibling suite
monkeypatch,
tmp_path,
):
"""A connection change rotates only that server's token; $VAR stays raw."""
cfg = tmp_path / "extensions_config.json"
cfg.write_text(
json.dumps(
{
"mcpServers": {
"A": {"enabled": True, "type": "stdio", "command": "npx", "args": ["pkg-v1"]},
"B": {"enabled": True, "type": "stdio", "command": "npx", "args": ["pkg-b"], "env": {"TOKEN": "$PE_VAR"}},
},
"skills": {},
}
),
encoding="utf-8",
)
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(cfg))
monkeypatch.setenv("PE_VAR", "secret-value")
baseline_a = _legacy(cfg, "A")
mcp_router._apply_mcp_config_update(
McpConfigUpdateRequest(
mcp_servers={
"A": McpServerConfigResponse(enabled=True, type="stdio", command="npx", args=["pkg-v2"]),
"B": McpServerConfigResponse(enabled=True, type="stdio", command="npx", args=["pkg-b"], env={"TOKEN": "$PE_VAR"}),
}
)
)
ledger = _ledger(cfg)
assert ledger is not None
assert ledger.servers["A"] != baseline_a # the connection changed
assert "B" not in ledger.servers # unchanged legacy server stays derived
written = cfg.read_text(encoding="utf-8")
assert "$PE_VAR" in written
assert "secret-value" not in written
@pytest.mark.asyncio
async def test_targeted_metadata_update_keeps_token_and_session(
reconciler, # noqa: F811 - fixture imported from the sibling suite
monkeypatch,
tmp_path,
):
"""A metadata-only targeted update must not rotate the token or the session."""
import test_mcp_cache_reconciliation as harness
from deerflow.mcp import session_pool as session_pool_module
from deerflow.mcp.session_pool import MCPSessionPool
cfg = tmp_path / "extensions_config.json"
pool = MCPSessionPool()
session_pool_module._pool = pool
log = harness._session_log()
await harness._initialize(monkeypatch, cfg, {"A": _stdio("npx")}, log)
binding_before = harness._binding(pool, "A")
entry_before = harness._entry(pool, "A")
mcp_router._apply_mcp_server_config_update(
McpServerConfigUpdateRequest(
server_name="A",
server=McpServerConfigResponse(enabled=True, type="stdio", command="npx", description="renamed"),
)
)
ledger = _ledger(cfg)
assert ledger is not None and "A" not in ledger.servers # metadata-only: no token
assert harness._binding(pool, "A") is binding_before
assert harness._entry(pool, "A") is entry_before
assert json.loads(cfg.read_text(encoding="utf-8"))["mcpServers"]["A"]["description"] == "renamed"
def test_skills_writer_preserves_the_lifecycle_ledger(
reconciler, # noqa: F811 - fixture imported from the sibling suite
monkeypatch,
tmp_path,
):
"""A skills-only write must round-trip mcpLifecycle exactly (no recompute)."""
from unittest.mock import MagicMock
from app.gateway.routers import skills as skills_router
cfg = tmp_path / "extensions_config.json"
ledger = {"version": 1, "servers": {"A": "a" * 32, "B": "b" * 32}}
cfg.write_text(
json.dumps(
{
"mcpServers": {"A": {"enabled": True, "type": "stdio", "command": "npx"}},
"skills": {"demo": {"enabled": True}},
"mcpLifecycle": ledger,
}
),
encoding="utf-8",
)
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(cfg))
monkeypatch.setattr("app.gateway.routers.skills.reload_extensions_config", lambda *a, **k: None)
skills_router._write_extensions_skill_state(MagicMock(), "demo", False, rebuild_public_projection=False)
written = json.loads(cfg.read_text(encoding="utf-8"))
assert written["mcpLifecycle"] == ledger # preserved byte-for-value, never recomputed
assert written["skills"]["demo"]["enabled"] is False

View File

@ -15,7 +15,7 @@ import pytest
from mcp.shared.exceptions import McpError
from mcp.types import CONNECTION_CLOSED, CallToolResult, ErrorData, TextContent
from deerflow.mcp.session_pool import MCPSessionPool, call_pooled_session_tool, get_session_pool, reset_session_pool
from deerflow.mcp.session_pool import MCPSessionPool, call_pooled_session_tool, current_session_pool, get_session_pool, reset_session_pool
from deerflow.mcp_scope import (
THREAD_INCARNATION_METADATA_GUARD_KEY,
mcp_scope_belongs_to_thread,
@ -663,6 +663,14 @@ async def test_close_all():
# ---------------------------------------------------------------------------
def test_current_session_pool_does_not_create_one():
"""Observing the singleton must not create one and must take the pool lock."""
reset_session_pool()
assert current_session_pool() is None
pool = get_session_pool()
assert current_session_pool() is pool
def test_get_session_pool_singleton():
"""get_session_pool returns the same instance."""
p1 = get_session_pool()