deer-flow/backend/app/gateway/routers/mcp.py

import asyncio
import json
import logging
import os
import re
from pathlib import Path
from typing import Any, Literal, NamedTuple

from fastapi import APIRouter, HTTPException, Request, status
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator

from app.gateway.deps import require_admin_user
from deerflow.config.extensions_config import (
    ExtensionsConfig,
    McpRoutingConfig,
    McpTaskToolsetConfig,
    McpToolOverride,
    atomic_write_extensions_config,
    extensions_config_file_lock,
    extensions_config_write_lock,
    get_extensions_config,
    normalize_mcp_transport_alias,
    reload_extensions_config,
)
from deerflow.constants import DEFAULT_MCP_SESSION_INIT_TIMEOUT
from deerflow.mcp.cache import reset_mcp_tools_cache

logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api", tags=["mcp"])

_ADMIN_REQUIRED_DETAIL = "Admin privileges required to manage MCP configuration."


_MCP_STDIO_COMMAND_ALLOWLIST_ENV = "DEER_FLOW_MCP_STDIO_COMMAND_ALLOWLIST"
_DEFAULT_MCP_STDIO_COMMAND_ALLOWLIST = frozenset({"npx", "uvx"})
_SHELL_METACHARS = frozenset(";|&`$<>\n\r")

# Flags that turn an allowlisted launcher into an arbitrary code evaluator.
# Validating only the command name leaves the allowlist naming a binary
# without constraining what that binary runs, so these are screened too.
# The spellings below mean "evaluate this string" across every launcher an
# operator would plausibly allowlist (npx/uvx `--call`, python/sh `-c`,
# node/perl/ruby `-e`/`--eval`, node `--print`), plus npx's pass-through into
# node's own argv.
#
# This is defense in depth, not a trust boundary. `npx`/`uvx` exist to fetch
# and run remote code, so an admin can still point one at a package they
# published; the boundary remains admin authentication plus not exposing the
# Gateway to untrusted networks.
_ARBITRARY_EXEC_ARGS = frozenset(
    {
        "-c",
        "--call",
        "-e",
        "--eval",
        "--print",
        "--shell",
        "--node-arg",
        "--node-options",
    }
)


# Package launchers parse their own options only until the package name; every
# later token is handed to the spawned server's own CLI, where `-c` is commonly
# "config" and `-e` "env". Screening those rejected ordinary third-party servers
# without covering anything, so the screen is scoped to the option region.
#
# Finding that region needs each launcher's option *arity*, because a value is
# not a positional: `npx -p <pkg> -c '<command>'` runs the command -- `-p` is
# exec's `--package`, so `<pkg>` is its value and npm keeps parsing its own
# flags. Ending the region at the first non-flag token would walk past it.
# (Verified against npm 10.9.4 / uv 0.11.1.)
#
# The two launchers get opposite defaults for an option neither table lists,
# and the reason is the exec set above, not symmetry:
#
#   `npx` really does own exec flags here (`-c`/`--call`), so an unlisted option
#   must not be able to hide one. Unknown therefore consumes a value, keeping
#   the region open. npm *errors* on an option it does not define, so this
#   cannot reject an invocation that would otherwise work; enumerating npm's
#   booleans (rather than its much larger value-taking set) is what makes the
#   common `npx -y <pkg> ...` shape land on the package name.
#
#   `uvx` owns no exec flag at all -- uv has no "evaluate this string" option --
#   so its screen is a tripwire, not a control, and an imprecise region cannot
#   walk past anything real. Unknown therefore consumes nothing, which keeps
#   uv's large and growing boolean surface from over-blocking.
#
# A launcher outside this table is not a package runner and keeps the
# conservative whole-args screen below.
class _LauncherGrammar(NamedTuple):
    """How one package launcher separates its own options from the server's."""

    exec_args: frozenset[str]
    known_args: frozenset[str]
    unknown_consumes_value: bool

    def consumes_value(self, flag: str) -> bool:
        if self.unknown_consumes_value:
            return flag not in self.known_args
        return flag in self.known_args


# npm's boolean configs, i.e. the options that do *not* consume the next token.
# Generated from `@npmcli/config`'s definitions (npm 10.9.4): every config whose
# type is Boolean, plus every nopt shorthand expanding to one of them or to a
# complete assignment such as `-d` -> `--loglevel info`. Regenerate against a
# newer npm rather than editing by hand. A boolean missing here over-blocks one
# invocation and names the flag in the rejection, which is the failure direction
# this file prefers.
_NPM_BOOLEAN_ARGS = frozenset(
    {
        "--all",
        "--allow-same-version",
        "--audit",
        "--bin-links",
        "--commit-hooks",
        "--description",
        "--dev",
        "--diff-ignore-all-space",
        "--diff-name-only",
        "--diff-no-prefix",
        "--diff-text",
        "--dry-run",
        "--engine-strict",
        "--expect-results",
        "--force",
        "--foreground-scripts",
        "--format-package-lock",
        "--fund",
        "--git-tag-version",
        "--global",
        "--global-style",
        "--if-present",
        "--ignore-scripts",
        "--include-staged",
        "--include-workspace-root",
        "--install-links",
        "--json",
        "--legacy-bundling",
        "--legacy-peer-deps",
        "--link",
        "--long",
        "--offline",
        "--omit-lockfile-registry-resolved",
        "--optional",
        "--package-lock",
        "--package-lock-only",
        "--parseable",
        "--prefer-dedupe",
        "--prefer-offline",
        "--prefer-online",
        "--production",
        "--progress",
        "--provenance",
        "--read-only",
        "--rebuild-bundle",
        "--save",
        "--save-bundle",
        "--save-dev",
        "--save-exact",
        "--save-optional",
        "--save-peer",
        "--save-prod",
        "--shrinkwrap",
        "--sign-git-commit",
        "--sign-git-tag",
        "--strict-peer-deps",
        "--strict-ssl",
        "--timing",
        "--unicode",
        "--update-notifier",
        "--usage",
        "--version",
        "--versions",
        "--workspaces",
        "--workspaces-update",
        "--yes",
        "-?",
        "-B",
        "-D",
        "-E",
        "-H",
        "-O",
        "-P",
        "-S",
        "-a",
        "-d",
        "-dd",
        "-ddd",
        "-desc",
        "-f",
        "-g",
        "-h",
        "-help",
        "-iwr",
        "-l",
        "-local",
        "-n",
        "-no",
        "-porcelain",
        "-q",
        "-quiet",
        "-readonly",
        "-s",
        "-silent",
        "-v",
        "-verbose",
        "-ws",
        "-y",
    }
)

# `npm exec` overrides the global `-p` shorthand: it is `--package <spec>` there,
# not the boolean `--parseable`. Confirmed by running it -- `npx -p . -c '<cmd>'`
# executes the command, i.e. `.` was consumed as a value and never ended the
# option region. Treating it as boolean is exactly the bypass this table exists
# to prevent, so the override is applied explicitly rather than left implicit.
_NPX_BOOLEAN_ARGS = _NPM_BOOLEAN_ARGS - {"-p"}

# uv's value-taking options (`uvx --help`, uv 0.11.1). Everything absent is
# treated as boolean; see the unknown-option note above for why that default is
# safe here and inverted for npx.
_UVX_VALUE_ARGS = frozenset(
    {
        "--allow-insecure-host",
        "--build-constraints",
        "--cache-dir",
        "--color",
        "--config-file",
        "--config-setting",
        "--config-settings-package",
        "--constraints",
        "--default-index",
        "--directory",
        "--env-file",
        "--exclude-newer",
        "--exclude-newer-package",
        "--extra-index-url",
        "--find-links",
        "--fork-strategy",
        "--from",
        "--index",
        "--index-strategy",
        "--index-url",
        "--keyring-provider",
        "--link-mode",
        "--no-binary-package",
        "--no-build-isolation-package",
        "--no-build-package",
        "--no-sources-package",
        "--overrides",
        "--prerelease",
        "--project",
        "--python",
        "--python-platform",
        "--refresh-package",
        "--reinstall-package",
        "--resolution",
        "--torch-backend",
        "--upgrade-package",
        "--with",
        "--with-editable",
        "--with-requirements",
        "-C",
        "-P",
        "-b",
        "-c",
        "-f",
        "-i",
        "-p",
        "-w",
    }
)

_PACKAGE_LAUNCHERS: dict[str, _LauncherGrammar] = {
    "npx": _LauncherGrammar(
        exec_args=_ARBITRARY_EXEC_ARGS,
        known_args=_NPX_BOOLEAN_ARGS,
        unknown_consumes_value=True,
    ),
    # uv spells `-c` `--constraints` and `-p` `--python`, so the short forms are
    # dropped from its exec set; the long spellings stay as a tripwire in case a
    # future uv grows one. Derived so a new entry above cannot forget this.
    "uvx": _LauncherGrammar(
        exec_args=frozenset(flag for flag in _ARBITRARY_EXEC_ARGS if flag.startswith("--")),
        known_args=_UVX_VALUE_ARGS,
        unknown_consumes_value=False,
    ),
}

# `-p` is `--print` (evaluate and print) on node, so exempting it everywhere
# left the short and long spellings of one flag disagreeing as soon as an
# operator extended the allowlist. It stays scoped to commands outside
# `_PACKAGE_LAUNCHERS`, where it is an ordinary selector (`--package` for npx,
# `--python` for uv), so the default allowlist is unaffected.
_EXEC_ARGS_OUTSIDE_PACKAGE_LAUNCHERS = frozenset({"-p"})

# Short options combine into one token (`node -pe`, `perl -we`, `python -Ic`),
# which whole-token matching does not see. Derived rather than restated so a
# new single-letter entry above cannot forget its clustered spelling.
_CLUSTERED_EXEC_LETTERS = frozenset(flag[1] for flag in _ARBITRARY_EXEC_ARGS | _EXEC_ARGS_OUTSIDE_PACKAGE_LAUNCHERS if len(flag) == 2 and flag.startswith("-"))

# Environment variables that inject code into a process at startup, which is
# the same bypass as an exec flag by another name.
#
# `PYTHONPATH` matters most: `site` imports `sitecustomize.py` from any
# `sys.path` entry before the tool's entry point runs, so a caller-controlled
# directory is code execution under `uvx` -- on the *default* allowlist.
# `PYTHONSTARTUP` is inert for the non-interactive launchers in scope and is
# kept only as belt-and-braces for an operator who allowlists a REPL.
#
# Known residual, accepted. Every entry below executes code *unconditionally*
# at process startup. Caller-controlled *search paths* are a different, weaker
# shape -- they reach code only if the process happens to load a name the
# caller can shadow -- and they stay out:
#
#   `LD_LIBRARY_PATH`/`DYLD_LIBRARY_PATH` run a shadowed library's constructor,
#   and native-dependency servers legitimately set them.
#
#   `NODE_PATH` is narrower still, and not for the reason it first looks like.
#   Node searches it *after* the local `node_modules` chain -- the resolver
#   unshifts the requiring module's own paths ahead of it -- so it cannot
#   shadow an installed dependency, and ESM `import` ignores it entirely. It
#   can only supply a CJS module that would otherwise fail to resolve, i.e. an
#   optional `try { require(...) } catch {}` dependency absent from the install.
#
# Adding them would make the "unconditional" rule above untrue, and a
# defense-in-depth list that grows because each entry was cheap is how it ends
# up mistaken for a boundary. A denylist is not what makes MCP registration
# safe for an untrusted admin anyway.
_CODE_INJECTING_ENV_VARS = frozenset(
    {
        "BASH_ENV",
        "DYLD_INSERT_LIBRARIES",
        "ENV",
        "LD_AUDIT",
        "LD_PRELOAD",
        "NODE_OPTIONS",
        "PERL5OPT",
        "PYTHONHOME",
        "PYTHONPATH",
        "PYTHONSTARTUP",
        "RUBYOPT",
    }
)


class McpUserScopedAuthConfigResponse(BaseModel):
    """Per-user credential injection configuration for an MCP server."""

    enabled: bool = Field(default=True, description="Whether user-scoped credential injection is enabled")
    header: str = Field(default="Authorization", description="HTTP header to set with the resolved user credential")
    users: dict[str, str] = Field(default_factory=dict, description="Map of DeerFlow user id to credential header value")
    on_missing: Literal["deny", "passthrough"] = Field(default="deny", description="Behavior when the calling user has no mapped credential")
    # Mirror the harness-side McpUserScopedAuthConfig (extra="allow"): without
    # this, an operator's unknown key inside user_auth would be silently
    # stripped by the next admin PUT, while server-level extras are preserved.
    model_config = ConfigDict(extra="allow")

    # Mirror the harness-side non-blank check: a blank header accepted here
    # would be persisted, then fail ExtensionsConfig validation on reload —
    # wedging every subsequent config load until the file is hand-edited.
    @field_validator("header")
    @classmethod
    def _validate_header_not_blank(cls, value: str) -> str:
        if not value.strip():
            raise ValueError("user_auth.header must not be empty")
        return value


class McpContextHeadersConfigResponse(BaseModel):
    """Per-request credential injection configuration for an MCP server.

    Holds header names and run-context key names only — never a credential —
    so unlike ``user_auth`` its declared fields are returned unmasked by GET.
    """

    enabled: bool = Field(default=True, description="Whether request-scoped header injection is enabled")
    headers: dict[str, str] = Field(
        default_factory=dict,
        description="Map of HTTP header name to the key read from the run request's config.context.secrets",
    )
    on_missing: Literal["deny", "passthrough"] = Field(default="deny", description="Behavior when a mapped key is absent from the request secrets")
    # Mirror the harness-side McpContextHeadersConfig (extra="allow"): without
    # this, an operator's unknown key inside headers_from_context would be
    # silently stripped by the next admin PUT.
    model_config = ConfigDict(extra="allow")

    # Mirror the harness-side entry check: a blank name accepted here would be
    # persisted, then fail ExtensionsConfig validation on reload — wedging every
    # subsequent config load until the file is hand-edited.
    @field_validator("headers")
    @classmethod
    def _validate_mapping_entries(cls, value: dict[str, str]) -> dict[str, str]:
        seen: dict[str, str] = {}
        for header_name, secret_key in value.items():
            if not header_name.strip():
                raise ValueError("headers_from_context.headers must not contain a blank header name")
            if not isinstance(secret_key, str) or not secret_key.strip():
                raise ValueError(f"headers_from_context.headers[{header_name!r}] must name a non-blank secret key from config.context.secrets")
            lowered = header_name.lower()
            if lowered in seen:
                raise ValueError(f"headers_from_context.headers maps the same HTTP header under two spellings ({seen[lowered]!r} and {header_name!r}); header names are case-insensitive, so keep only one")
            seen[lowered] = header_name
        return value


class McpOAuthConfigResponse(BaseModel):
    """OAuth configuration for an MCP server."""

    enabled: bool = Field(default=True, description="Whether OAuth token injection is enabled")
    token_url: str = Field(default="", description="OAuth token endpoint URL")
    grant_type: Literal["client_credentials", "refresh_token"] = Field(default="client_credentials", description="OAuth grant type")
    client_id: str | None = Field(default=None, description="OAuth client ID")
    client_secret: str | None = Field(default=None, description="OAuth client secret")
    refresh_token: str | None = Field(default=None, description="OAuth refresh token")
    scope: str | None = Field(default=None, description="OAuth scope")
    audience: str | None = Field(default=None, description="OAuth audience")
    token_field: str = Field(default="access_token", description="Token response field containing access token")
    token_type_field: str = Field(default="token_type", description="Token response field containing token type")
    expires_in_field: str = Field(default="expires_in", description="Token response field containing expires-in seconds")
    default_token_type: str = Field(default="Bearer", description="Default token type when response omits token_type")
    refresh_skew_seconds: int = Field(default=60, description="Refresh this many seconds before expiry")
    extra_token_params: dict[str, str] = Field(default_factory=dict, description="Additional form params sent to token endpoint")


class McpServerConfigResponse(BaseModel):
    """Response model for MCP server configuration."""

    enabled: bool = Field(default=True, description="Whether this MCP server is enabled")
    type: str = Field(default="stdio", description="Transport type: 'stdio', 'sse', or 'http'")
    command: str | None = Field(default=None, description="Command to execute to start the MCP server (for stdio type)")
    args: list[str] = Field(default_factory=list, description="Arguments to pass to the command (for stdio type)")
    env: dict[str, str] = Field(default_factory=dict, description="Environment variables for the MCP server")
    url: str | None = Field(default=None, description="URL of the MCP server (for sse or http type)")
    headers: dict[str, str] = Field(default_factory=dict, description="HTTP headers to send (for sse or http type)")
    oauth: McpOAuthConfigResponse | None = Field(default=None, description="OAuth configuration for MCP HTTP/SSE servers")
    user_auth: McpUserScopedAuthConfigResponse | None = Field(default=None, description="Per-user credential injection for MCP HTTP/SSE servers")
    headers_from_context: McpContextHeadersConfigResponse | None = Field(default=None, description="Per-request credential injection for MCP HTTP/SSE servers: map header names to config.context.secrets keys")
    description: str = Field(default="", description="Human-readable description of what this MCP server provides")
    routing: McpRoutingConfig = Field(default_factory=McpRoutingConfig, description="Soft routing hints for tools from this MCP server")
    tools: dict[str, McpToolOverride] = Field(default_factory=dict, description="Per-original-tool MCP configuration overrides")
    tool_name_prefix: bool = Field(default=True, description="Whether to prefix discovered tool names with the MCP server name")
    tool_call_timeout: float | None = Field(
        default=None,
        description="Timeout in seconds for individual stdio MCP calls and durable-task calls on every transport",
    )
    # Default matches McpServerConfig: this model's defaults feed model_dump()
    # into the persisted extensions config on PUT, so an API-created server that
    # omits the field must get the same bring-up timeout as a file-created one.
    session_init_timeout: float | None = Field(
        default=DEFAULT_MCP_SESSION_INIT_TIMEOUT,
        description="Timeout in seconds for MCP server bring-up and durable HTTP/SSE task-session initialization; null means no timeout",
    )
    task_toolsets: list[McpTaskToolsetConfig] = Field(
        default_factory=list,
        description="Raw submit/status/cancel tool groups managed as durable background tasks",
    )
    model_config = ConfigDict(extra="allow")

    @field_validator("headers")
    @classmethod
    def _validate_header_names(cls, value: dict[str, str]) -> dict[str, str]:
        # Mirror the harness-side McpServerConfig check: HTTP field names are
        # case-insensitive, so a config carrying one header under two spellings
        # would persist, reload into a connection with both fields, and let a
        # later override replace only one of them. Reject at the API boundary
        # instead of wedging the next ExtensionsConfig reload.
        seen: dict[str, str] = {}
        for header_name in value:
            lowered = header_name.lower()
            if lowered in seen:
                raise ValueError(f"headers maps the same HTTP header under two spellings ({seen[lowered]!r} and {header_name!r}); header names are case-insensitive, so keep only one")
            seen[lowered] = header_name
        return value

    @model_validator(mode="before")
    @classmethod
    def _accept_transport_alias(cls, data: Any) -> Any:
        """Keep API parsing aligned with the runtime MCP config model."""
        return normalize_mcp_transport_alias(data)


class McpConfigResponse(BaseModel):
    """Response model for MCP configuration."""

    mcp_servers: dict[str, McpServerConfigResponse] = Field(
        default_factory=dict,
        description="Map of MCP server name to configuration",
    )


class McpConfigUpdateRequest(BaseModel):
    """Request model for updating MCP configuration."""

    mcp_servers: dict[str, McpServerConfigResponse] = Field(
        ...,
        description="Map of MCP server name to configuration",
    )


class McpServerStateUpdateRequest(BaseModel):
    """Request model for enabling or disabling one MCP server."""

    server_name: str = Field(
        ...,
        min_length=1,
        description="Name of the MCP server to update",
    )
    enabled: bool = Field(..., description="Whether the MCP server is enabled")


class McpCacheResetResponse(BaseModel):
    """Response model for resetting the MCP tools cache."""

    success: bool = Field(description="Whether the MCP tools cache was reset")
    message: str = Field(description="Human-readable reset status")


_MASKED_VALUE = "***"
_SENSITIVE_EXTRA_KEY_RE = re.compile(
    r"(^|_)(api_key|apikey|access_key|private_key|client_secret|secret|token|password|passwd|credential|credentials|authorization|bearer)(_|$)",
    re.IGNORECASE,
)


def _normalize_config_key(key: str) -> str:
    with_boundaries = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", key)
    with_boundaries = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", with_boundaries)
    return re.sub(r"[^a-z0-9]+", "_", with_boundaries.lower()).strip("_")


def _is_sensitive_extra_key(key: str) -> bool:
    return bool(_SENSITIVE_EXTRA_KEY_RE.search(_normalize_config_key(key)))


def _mask_sensitive_extra_value(value: Any) -> Any:
    if isinstance(value, dict):
        return {key: _MASKED_VALUE if _is_sensitive_extra_key(str(key)) else _mask_sensitive_extra_value(nested) for key, nested in value.items()}
    if isinstance(value, list):
        return [_mask_sensitive_extra_value(item) for item in value]
    return value


def _merge_extra_value_preserving_masked(key: str, incoming_value: Any, existing_value: Any, *, existing_present: bool) -> Any:
    if incoming_value == _MASKED_VALUE and _is_sensitive_extra_key(key):
        if existing_present:
            return existing_value
        raise HTTPException(
            status_code=400,
            detail=f"Cannot set extra config key '{key}' to masked value '***'; provide a real value.",
        )

    if isinstance(incoming_value, dict) and isinstance(existing_value, dict):
        merged: dict[str, Any] = {}
        for nested_key, nested_value in incoming_value.items():
            nested_present = nested_key in existing_value
            merged[nested_key] = _merge_extra_value_preserving_masked(
                str(nested_key),
                nested_value,
                existing_value.get(nested_key),
                existing_present=nested_present,
            )
        return merged

    if isinstance(incoming_value, list) and isinstance(existing_value, list) and len(incoming_value) == len(existing_value):
        return [_merge_extra_value_preserving_masked(key, nested_value, existing_value[index], existing_present=True) for index, nested_value in enumerate(incoming_value)]

    return incoming_value


def _allowed_stdio_commands() -> set[str]:
    """Return executable names allowed for API-managed stdio MCP servers."""
    raw = os.environ.get(_MCP_STDIO_COMMAND_ALLOWLIST_ENV)
    base = set(_DEFAULT_MCP_STDIO_COMMAND_ALLOWLIST)
    if raw is None:
        return base
    extra = {item.strip() for item in raw.split(",") if item.strip()}
    return base | extra


def _stdio_command_name(command: str | None, *, server_name: str) -> str:
    """Normalize and validate a stdio command field from the API boundary."""
    if command is None or not command.strip():
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail=f"MCP server '{server_name}' with stdio transport requires a command.",
        )

    stripped = command.strip()
    has_path_separator = "/" in stripped or "\\" in stripped
    if stripped != command or has_path_separator or any(ch.isspace() for ch in stripped) or any(ch in stripped for ch in _SHELL_METACHARS):
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail=(f"MCP server '{server_name}' command must be a single executable name; put parameters in args instead."),
        )

    return stripped


def _launcher_option_region(args: list[str], *, grammar: _LauncherGrammar) -> list[str]:
    """Return the leading args a package launcher parses as its own options.

    The region ends at a bare ``--`` or at the package name -- the first token
    that is neither a flag nor the value of one. A ``--flag=value`` token
    carries its own value and never consumes the next one.

    Arity is looked up case-sensitively, because a launcher's short options are:
    npm reads ``-c`` as ``--call`` but ``-C`` as ``--prefix``, which takes a
    value.
    """
    region: list[str] = []
    index = 0
    while index < len(args):
        arg = args[index]
        if not isinstance(arg, str):
            break
        token = arg.strip()
        if token == "--" or token == "-" or not token.startswith("-"):
            break
        region.append(token)
        index += 1
        if "=" not in token and grammar.consumes_value(token):
            index += 1
    return region


def _arbitrary_exec_arg(args: list[str], *, command: str) -> str | None:
    """Return the offending flag when an argument makes the launcher eval a string.

    Handles both ``--call value`` and ``--call=value`` spellings.

    For a package launcher (:data:`_PACKAGE_LAUNCHERS`) only the launcher's own
    option region is screened, because everything from the package name onward
    is the spawned server's argv -- ``npx -y <pkg> -c config.json`` hands
    ``-c config.json`` to the server, where it is "config", not eval. A bare
    ``--`` ends the region too: only the *first* token after it is the package
    name, and the rest are that package's arguments.

    Every other command is screened whole, and two extra rules apply because
    such a command is an interpreter rather than a package runner: ``-p`` is an
    exec flag (node's ``--print``) instead of a package/python selector, and
    combined short-option clusters are decomposed so ``-pe`` cannot smuggle
    past a check that only splits on ``=``.

    Only the normalized flag is returned, never the caller's value, so the
    rejection message does not echo a payload string back into the response.
    """
    grammar = _PACKAGE_LAUNCHERS.get(command.lower())
    if grammar is not None:
        for token in _launcher_option_region(args, grammar=grammar):
            flag = token.split("=", 1)[0]
            # Long options are matched case-insensitively as before; a short one
            # is not, because its case selects a different option -- npm's `-C`
            # is `--prefix`, and folding it onto `-c` rejected an ordinary flag.
            flag = flag.lower() if flag.startswith("--") else flag
            if flag in grammar.exec_args:
                return flag
        return None

    denied = _ARBITRARY_EXEC_ARGS | _EXEC_ARGS_OUTSIDE_PACKAGE_LAUNCHERS
    for arg in args:
        if not isinstance(arg, str):
            continue
        flag = arg.split("=", 1)[0].strip().lower()
        if flag in denied:
            return flag
        if not flag.startswith("-") or flag.startswith("--"):
            continue
        for letter in flag[1:]:
            if letter in _CLUSTERED_EXEC_LETTERS:
                return f"-{letter}"
    return None


def _validate_mcp_update_request(request: McpConfigUpdateRequest) -> None:
    """Validate API-submitted MCP config before it is persisted.

    Local config files can still express arbitrary advanced setups, but the
    HTTP API is an untrusted boundary. Restricting stdio commands here reduces
    the blast radius of a compromised authenticated browser session.

    The command name alone is not a meaningful restriction, so the launcher's
    ``args`` and ``env`` are screened for the flags and variables that turn an
    allowlisted binary into an arbitrary code evaluator.
    """
    allowed_commands = _allowed_stdio_commands()
    for name, server in request.mcp_servers.items():
        transport_type = (server.type or "stdio").lower()
        if transport_type != "stdio":
            continue

        command_name = _stdio_command_name(server.command, server_name=name)
        if command_name not in allowed_commands:
            allowed = ", ".join(sorted(allowed_commands)) or "<none>"
            raise HTTPException(
                status_code=status.HTTP_400_BAD_REQUEST,
                detail=(f"MCP server '{name}' uses disallowed stdio command '{command_name}'. Allowed commands: {allowed}. Configure {_MCP_STDIO_COMMAND_ALLOWLIST_ENV} to extend this list."),
            )

        exec_flag = _arbitrary_exec_arg(server.args, command=command_name)
        if exec_flag is not None:
            raise HTTPException(
                status_code=status.HTTP_400_BAD_REQUEST,
                detail=(f"MCP server '{name}' passes '{exec_flag}' to '{command_name}', which would run arbitrary code. Point the server at a package or module instead."),
            )

        for env_name in server.env:
            if env_name.strip().upper() in _CODE_INJECTING_ENV_VARS:
                raise HTTPException(
                    status_code=status.HTTP_400_BAD_REQUEST,
                    detail=(f"MCP server '{name}' sets environment variable '{env_name}', which would run arbitrary code at process startup."),
                )


def _mask_server_config(server: McpServerConfigResponse) -> McpServerConfigResponse:
    """Return a copy of server config with sensitive fields masked.

    Masks env values, header values, and removes OAuth secrets so they
    are not exposed through the GET API endpoint.
    """
    masked_env = {k: _MASKED_VALUE for k in server.env}
    masked_headers = {k: _MASKED_VALUE for k in server.headers}
    masked_oauth = None
    if server.oauth is not None:
        masked_oauth = server.oauth.model_copy(
            update={
                "client_secret": None,
                "refresh_token": None,
            }
        )
    masked_user_auth = None
    if server.user_auth is not None:
        # Extras inside user_auth get the same sensitive-key masking as
        # server-level extras: they round-trip through PUT (extra="allow"), so
        # an operator-stored secret-bearing key must not come back in
        # cleartext from GET while the identical key at server level is masked.
        masked_ua_extra = {key: _MASKED_VALUE if _is_sensitive_extra_key(key) else _mask_sensitive_extra_value(value) for key, value in (server.user_auth.model_extra or {}).items()}
        masked_user_auth = server.user_auth.model_copy(update={"users": {k: _MASKED_VALUE for k in server.user_auth.users}, **masked_ua_extra})
    masked_headers_from_context = None
    if server.headers_from_context is not None:
        # The declared fields hold names only and stay in cleartext — masking
        # them would show operators `***` where a header name belongs. Extras
        # get the same treatment as everywhere else, since `extra="allow"` lets
        # an operator store a secret-bearing key that round-trips through PUT.
        masked_ch_extra = {key: _MASKED_VALUE if _is_sensitive_extra_key(key) else _mask_sensitive_extra_value(value) for key, value in (server.headers_from_context.model_extra or {}).items()}
        masked_headers_from_context = server.headers_from_context.model_copy(update=masked_ch_extra)
    masked_extra = {key: _MASKED_VALUE if _is_sensitive_extra_key(key) else _mask_sensitive_extra_value(value) for key, value in (server.model_extra or {}).items()}
    return server.model_copy(
        update={
            "env": masked_env,
            "headers": masked_headers,
            "oauth": masked_oauth,
            "user_auth": masked_user_auth,
            "headers_from_context": masked_headers_from_context,
            **masked_extra,
        }
    )


def _merge_preserving_secrets(
    incoming: McpServerConfigResponse,
    existing: McpServerConfigResponse,
) -> McpServerConfigResponse:
    """Merge incoming config with existing, preserving secrets masked by GET.

    When the frontend toggles ``enabled`` it round-trips the full config:
    GET (masked) → modify enabled → PUT (masked values sent back).
    This function ensures masked values (``***``) are replaced with the
    real secrets from the current on-disk config.

    ``***`` is only accepted for keys that already exist in *existing*.
    New keys must provide a real value.

    For OAuth secrets, ``None`` means "preserve the existing stored value"
    so masked GET responses can be safely round-tripped. To explicitly clear
    a stored secret, clients may send an empty string, which is converted
    to ``None`` before persisting.
    """
    merged_env = {}
    for k, v in incoming.env.items():
        if v == _MASKED_VALUE:
            if k in existing.env:
                merged_env[k] = existing.env[k]
            else:
                raise HTTPException(
                    status_code=400,
                    detail=f"Cannot set env key '{k}' to masked value '***'; provide a real value.",
                )
        else:
            merged_env[k] = v

    merged_headers = {}
    for k, v in incoming.headers.items():
        if v == _MASKED_VALUE:
            if k in existing.headers:
                merged_headers[k] = existing.headers[k]
            else:
                raise HTTPException(
                    status_code=400,
                    detail=f"Cannot set header '{k}' to masked value '***'; provide a real value.",
                )
        else:
            merged_headers[k] = v

    merged_oauth = incoming.oauth
    if incoming.oauth is not None and existing.oauth is not None:
        # None = preserve (masked round-trip), "" = explicitly clear, else = new value
        merged_client_secret = existing.oauth.client_secret if incoming.oauth.client_secret is None else (None if incoming.oauth.client_secret == "" else incoming.oauth.client_secret)
        merged_refresh_token = existing.oauth.refresh_token if incoming.oauth.refresh_token is None else (None if incoming.oauth.refresh_token == "" else incoming.oauth.refresh_token)
        merged_oauth = incoming.oauth.model_copy(
            update={
                "client_secret": merged_client_secret,
                "refresh_token": merged_refresh_token,
            }
        )
    merged_user_auth = incoming.user_auth
    if incoming.user_auth is not None:
        # Sub-field-aware merge: a partial user_auth payload (e.g. just
        # {"enabled": false}) must not wipe the stored credential map or reset
        # other stored sub-fields. Only sub-fields the request explicitly set
        # replace stored values; the rest carry over — the same contract the
        # block-level `model_fields_set` check below applies one level up.
        incoming_ua = incoming.user_auth
        base = existing.user_auth
        set_fields = incoming_ua.model_fields_set
        effective: dict[str, Any] = {}
        if base is not None:
            effective.update({name: getattr(base, name) for name in ("enabled", "header", "users", "on_missing")})
            effective.update(base.model_extra or {})
        for name in ("enabled", "header", "on_missing"):
            if name in set_fields:
                effective[name] = getattr(incoming_ua, name)
        # Extras are masked by GET (see _mask_server_config), so a round-trip
        # PUT must swap masked sentinel values back for the stored ones —
        # the same contract server-level extras get below.
        base_extra = (base.model_extra or {}) if base is not None else {}
        for key, value in (incoming_ua.model_extra or {}).items():
            effective[key] = _merge_extra_value_preserving_masked(
                key,
                value,
                base_extra.get(key),
                existing_present=key in base_extra,
            )
        if "users" in set_fields:
            # An explicitly sent map replaces the stored one (so a full
            # round-trip can remove a user), with masked values swapped back
            # for the stored credentials.
            existing_users = base.users if base is not None else {}
            merged_users = {}
            for k, v in incoming_ua.users.items():
                if v == _MASKED_VALUE:
                    if k in existing_users:
                        merged_users[k] = existing_users[k]
                    else:
                        raise HTTPException(
                            status_code=400,
                            detail=f"Cannot set user_auth credential for '{k}' to masked value '***'; provide a real value.",
                        )
                else:
                    merged_users[k] = v
            effective["users"] = merged_users
        merged_user_auth = McpUserScopedAuthConfigResponse(**effective)

    merged_context_headers = incoming.headers_from_context
    if incoming.headers_from_context is not None:
        # Sub-field-aware merge, mirroring user_auth above: an explicit partial
        # block (e.g. {"enabled": false}) must not wipe the stored mapping or
        # reset on_missing back to its default. Only fields the request set are
        # replaced, so an explicitly supplied ``headers`` (even {}) replaces the
        # map while omitted fields carry over from the stored block.
        incoming_ch = incoming.headers_from_context
        base_ch = existing.headers_from_context
        set_fields = incoming_ch.model_fields_set
        effective: dict[str, Any] = {}
        if base_ch is not None:
            effective.update({name: getattr(base_ch, name) for name in ("enabled", "headers", "on_missing")})
            effective.update(base_ch.model_extra or {})
        for name in ("enabled", "headers", "on_missing"):
            if name in set_fields:
                effective[name] = getattr(incoming_ch, name)
        # Extras are masked by GET (see _mask_server_config), so a round-trip
        # PUT must swap masked sentinel values back for the stored ones — the
        # same contract user_auth extras and server-level extras get.
        base_ch_extra = (base_ch.model_extra or {}) if base_ch is not None else {}
        for key, value in (incoming_ch.model_extra or {}).items():
            effective[key] = _merge_extra_value_preserving_masked(
                key,
                value,
                base_ch_extra.get(key),
                existing_present=key in base_ch_extra,
            )
        merged_context_headers = McpContextHeadersConfigResponse(**effective)

    update = {
        "env": merged_env,
        "headers": merged_headers,
        "oauth": merged_oauth,
        "user_auth": merged_user_auth,
        "headers_from_context": merged_context_headers,
    }
    if "user_auth" not in incoming.model_fields_set:
        update["user_auth"] = existing.user_auth
    if "headers_from_context" not in incoming.model_fields_set:
        update["headers_from_context"] = existing.headers_from_context
    if "routing" not in incoming.model_fields_set:
        update["routing"] = existing.routing
    if "tools" not in incoming.model_fields_set:
        update["tools"] = existing.tools
    incoming_extra = incoming.model_extra or {}
    existing_extra = existing.model_extra or {}
    for key, value in incoming_extra.items():
        update[key] = _merge_extra_value_preserving_masked(
            key,
            value,
            existing_extra.get(key),
            existing_present=key in existing_extra,
        )
    for key, value in (existing.model_extra or {}).items():
        if key not in (incoming.model_extra or {}):
            update[key] = value
    return incoming.model_copy(update=update)


@router.get(
    "/mcp/config",
    response_model=McpConfigResponse,
    summary="Get MCP Configuration",
    description="Retrieve the current Model Context Protocol (MCP) server configurations.",
)
async def get_mcp_configuration(request: Request) -> McpConfigResponse:
    """Get the current MCP configuration.

    Returns:
        The current MCP configuration with all servers.

    Example:
        ```json
        {
            "mcp_servers": {
                "github": {
                    "enabled": true,
                    "command": "npx",
                    "args": ["-y", "@modelcontextprotocol/server-github"],
                    "env": {"GITHUB_TOKEN": "***"},
                    "description": "GitHub MCP server for repository operations"
                }
            }
        }
        ```
    """
    await require_admin_user(request, detail=_ADMIN_REQUIRED_DETAIL)

    config = get_extensions_config()

    servers = {name: _mask_server_config(McpServerConfigResponse(**server.model_dump())) for name, server in config.mcp_servers.items()}
    return McpConfigResponse(mcp_servers=servers)


def _apply_mcp_config_update(body: McpConfigUpdateRequest) -> dict:
    """Worker-thread body for :func:`update_mcp_configuration`.

    Resolving the config path, the existence probe, reading the raw JSON,
    writing the merged config, and reloading it are all blocking filesystem IO
    that must stay off the event loop. The merge is pure in-memory work but
    lives here too so the whole read-modify-write is a single worker hop.
    Returns the reloaded MCP server configs for the response.
    """
    # Resolve before entering the critical section so every writer locks the
    # same sidecar path for the complete read-modify-write cycle.
    config_path = ExtensionsConfig.resolve_config_path()
    if config_path is None:
        config_path = Path.cwd().parent / "extensions_config.json"
        logger.info(f"No existing extensions config found. Creating new config at: {config_path}")

    with extensions_config_write_lock, extensions_config_file_lock(config_path):
        # Load raw (un-resolved) JSON from disk to use as the merge source.
        # This preserves $VAR placeholders in env values and top-level keys
        # like mcpInterceptors that would otherwise be lost.
        raw_servers: dict[str, dict] = {}
        raw_other_keys: dict = {}
        raw_skills: dict[str, dict] | None = None
        if config_path is not None and config_path.exists():
            with open(config_path, encoding="utf-8") as f:
                raw_data = json.load(f)
            raw_servers = raw_data.get("mcpServers", {})
            if isinstance(raw_data.get("skills"), dict):
                raw_skills = raw_data["skills"]
            # Preserve any top-level keys beyond mcpServers/skills
            for key, value in raw_data.items():
                if key not in ("mcpServers", "skills"):
                    raw_other_keys[key] = value

        # Merge incoming server configs with raw on-disk secrets
        merged_servers: dict[str, McpServerConfigResponse] = {}
        for name, incoming in body.mcp_servers.items():
            raw_server = raw_servers.get(name)
            if raw_server is not None:
                merged_servers[name] = _merge_preserving_secrets(
                    incoming,
                    McpServerConfigResponse(**raw_server),
                )
            else:
                merged_servers[name] = incoming

        # Build config data preserving all top-level keys from the original file
        config_data = dict(raw_other_keys)
        config_data["mcpServers"] = {name: server.model_dump() for name, server in merged_servers.items()}
        if raw_skills is None:
            current_config = get_extensions_config()
            raw_skills = {name: {"enabled": skill.enabled} for name, skill in current_config.skills.items()}
        config_data["skills"] = raw_skills

        atomic_write_extensions_config(config_path, config_data)

        logger.info(f"MCP configuration updated and saved to: {config_path}")

        # Reload the Gateway configuration and update the global cache. The
        # agent runtime lives in Gateway, so this keeps API reads and tool
        # execution aligned after extensions_config.json changes.
        reloaded_config = reload_extensions_config()
        return reloaded_config.mcp_servers


def _apply_mcp_server_state_update(body: McpServerStateUpdateRequest) -> dict:
    """Update one server state while preserving the raw extensions config."""
    config_path = ExtensionsConfig.resolve_config_path()
    if config_path is None:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail=f"MCP server '{body.server_name}' not found",
        )

    with extensions_config_write_lock, extensions_config_file_lock(config_path):
        if not config_path.exists():
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail=f"MCP server '{body.server_name}' not found",
            )

        with open(config_path, encoding="utf-8") as f:
            raw_data = json.load(f)

        raw_servers = raw_data.get("mcpServers", {})
        raw_server = raw_servers.get(body.server_name) if isinstance(raw_servers, dict) else None
        if not isinstance(raw_server, dict):
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail=f"MCP server '{body.server_name}' not found",
            )

        if body.enabled:
            target_server = McpServerConfigResponse(**raw_server)
            _validate_mcp_update_request(
                McpConfigUpdateRequest(
                    mcp_servers={body.server_name: target_server},
                )
            )

        raw_server["enabled"] = body.enabled
        atomic_write_extensions_config(config_path, raw_data)

        logger.info("MCP server %s enabled state updated to %s", body.server_name, body.enabled)
        reloaded_config = reload_extensions_config()
        return reloaded_config.mcp_servers


@router.post(
    "/mcp/cache/reset",
    response_model=McpCacheResetResponse,
    summary="Reset MCP Tools Cache",
    description=("Reset cached MCP tools and pooled sessions process-wide so tools are reloaded on next use. This affects all threads and users in the current Gateway process."),
)
async def reset_mcp_tools_cache_endpoint(request: Request) -> McpCacheResetResponse:
    """Reset cached MCP tools and persistent sessions process-wide.

    The next agent run or tool lookup will reload tools from the configured MCP
    servers. This affects all threads and users in the current Gateway process,
    and avoids relying on extensions_config.json mtime changes.
    """
    await require_admin_user(request, detail=_ADMIN_REQUIRED_DETAIL)
    reset_mcp_tools_cache()
    return McpCacheResetResponse(
        success=True,
        message="MCP tools cache reset. Tools will reload on next use.",
    )


@router.put(
    "/mcp/config",
    response_model=McpConfigResponse,
    summary="Update MCP Configuration",
    description="Update Model Context Protocol (MCP) server configurations and save to file.",
)
async def update_mcp_configuration(request: Request, body: McpConfigUpdateRequest) -> McpConfigResponse:
    """Update the MCP configuration.

    This will:
    1. Save the new configuration to the mcp_config.json file
    2. Reload the configuration cache
    3. Reset MCP tools cache to trigger reinitialization

    Args:
        request: The new MCP configuration to save.

    Returns:
        The updated MCP configuration.

    Raises:
        HTTPException: 500 if the configuration file cannot be written.

    Example Request:
        ```json
        {
            "mcp_servers": {
                "github": {
                    "enabled": true,
                    "command": "npx",
                    "args": ["-y", "@modelcontextprotocol/server-github"],
                    "env": {"GITHUB_TOKEN": "$GITHUB_TOKEN"},
                    "description": "GitHub MCP server for repository operations"
                }
            }
        }
        ```
    """
    try:
        await require_admin_user(request, detail=_ADMIN_REQUIRED_DETAIL)
        _validate_mcp_update_request(body)

        # Offload the blocking read-modify-write of extensions_config.json
        # (path resolve, existence probe, raw read, merged write, reload). The
        # worker takes extensions_config_write_lock for the whole RMW, so it stays
        # atomic and serialized against the skills router (the other writer of
        # this file) even if this request is cancelled mid-write.
        reloaded_servers = await asyncio.to_thread(_apply_mcp_config_update, body)

        servers = {name: _mask_server_config(McpServerConfigResponse(**server.model_dump())) for name, server in reloaded_servers.items()}
        reset_mcp_tools_cache()
        return McpConfigResponse(mcp_servers=servers)

    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Failed to update MCP configuration: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Failed to update MCP configuration: {str(e)}")


@router.patch(
    "/mcp/config",
    response_model=McpConfigResponse,
    summary="Update MCP Server State",
    description="Enable or disable one MCP server without replacing the full extensions configuration.",
)
async def update_mcp_server_state(request: Request, body: McpServerStateUpdateRequest) -> McpConfigResponse:
    """Enable or disable one MCP server and reload the MCP tool cache."""
    try:
        await require_admin_user(request, detail=_ADMIN_REQUIRED_DETAIL)
        reloaded_servers = await asyncio.to_thread(_apply_mcp_server_state_update, body)

        servers = {name: _mask_server_config(McpServerConfigResponse(**server.model_dump())) for name, server in reloaded_servers.items()}
        reset_mcp_tools_cache()
        return McpConfigResponse(mcp_servers=servers)
    except HTTPException:
        raise
    except Exception as e:
        logger.error("Failed to update MCP server %s state: %s", body.server_name, e, exc_info=True)
        raise HTTPException(status_code=500, detail=f"Failed to update MCP server state: {str(e)}")