diff --git a/.dockerignore b/.dockerignore index a571fb086..e21af833b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -69,3 +69,10 @@ backend/.coverage !README.md !frontend/README.md !backend/README.md + +# Extension manager snapshots must enter the backend builder intact. A Python +# package may require README metadata, native modules, or package assets that +# the general image-context exclusions above intentionally omit elsewhere. +!backend/extensions/ +!backend/extensions/sources/ +!backend/extensions/sources/** diff --git a/.github/workflows/backend-blocking-io-tests.yml b/.github/workflows/backend-blocking-io-tests.yml index 88eb37324..affc84b5b 100644 --- a/.github/workflows/backend-blocking-io-tests.yml +++ b/.github/workflows/backend-blocking-io-tests.yml @@ -35,7 +35,10 @@ jobs: python-version: "3.12" - name: Install uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@v7 + with: + # Must match backend/Dockerfile's UV_IMAGE tag; pinned by backend/tests/test_ci_uv_version_pin.py + version: "0.11.1" - name: Install backend dependencies working-directory: backend diff --git a/.github/workflows/backend-unit-tests.yml b/.github/workflows/backend-unit-tests.yml index 29779d00a..11178aa5b 100644 --- a/.github/workflows/backend-unit-tests.yml +++ b/.github/workflows/backend-unit-tests.yml @@ -33,6 +33,9 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 + with: + # Must match backend/Dockerfile's UV_IMAGE tag; pinned by backend/tests/test_ci_uv_version_pin.py + version: "0.11.1" - name: Install backend dependencies (documented default) working-directory: backend @@ -85,6 +88,9 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 + with: + # Must match backend/Dockerfile's UV_IMAGE tag; pinned by backend/tests/test_ci_uv_version_pin.py + version: "0.11.1" - name: Install backend dependencies working-directory: backend diff --git a/.github/workflows/label-sync.yml b/.github/workflows/label-sync.yml index c270e06f6..d281eff13 100644 --- a/.github/workflows/label-sync.yml +++ b/.github/workflows/label-sync.yml @@ -30,6 +30,9 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 + with: + # Must match backend/Dockerfile's UV_IMAGE tag; pinned by backend/tests/test_ci_uv_version_pin.py + version: "0.11.1" - name: Sync labels run: uv run --with pyyaml python scripts/sync_labels.py diff --git a/.github/workflows/lint-check.yml b/.github/workflows/lint-check.yml index caf6c911b..79094c0c9 100644 --- a/.github/workflows/lint-check.yml +++ b/.github/workflows/lint-check.yml @@ -50,6 +50,9 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 + with: + # Must match backend/Dockerfile's UV_IMAGE tag; pinned by backend/tests/test_ci_uv_version_pin.py + version: "0.11.1" - name: Check uv.lock is in sync working-directory: backend diff --git a/.github/workflows/replay-e2e.yml b/.github/workflows/replay-e2e.yml index a60de4d9d..839f10e46 100644 --- a/.github/workflows/replay-e2e.yml +++ b/.github/workflows/replay-e2e.yml @@ -57,6 +57,9 @@ jobs: python-version: "3.12" - name: Install uv uses: astral-sh/setup-uv@v7 + with: + # Must match backend/Dockerfile's UV_IMAGE tag; pinned by backend/tests/test_ci_uv_version_pin.py + version: "0.11.1" - name: Install backend dependencies working-directory: backend run: uv sync --group dev @@ -77,6 +80,9 @@ jobs: python-version: "3.12" - name: Install uv uses: astral-sh/setup-uv@v7 + with: + # Must match backend/Dockerfile's UV_IMAGE tag; pinned by backend/tests/test_ci_uv_version_pin.py + version: "0.11.1" - name: Install backend dependencies (replay gateway) working-directory: backend run: uv sync --group dev diff --git a/.github/workflows/skill-review-ci.yml b/.github/workflows/skill-review-ci.yml index 8bd8438d4..e46be4824 100644 --- a/.github/workflows/skill-review-ci.yml +++ b/.github/workflows/skill-review-ci.yml @@ -48,6 +48,9 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 + with: + # Must match backend/Dockerfile's UV_IMAGE tag; pinned by backend/tests/test_ci_uv_version_pin.py + version: "0.11.1" - name: Install backend dependencies working-directory: backend diff --git a/AGENTS.md b/AGENTS.md index 753b76a54..a3e652c4e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,6 +57,7 @@ deer-flow/ ├── extensions_config.example.json # Template → copy to extensions_config.json (gitignored): MCP servers + skills ├── backend/ # Python backend — see backend/AGENTS.md │ ├── Makefile # Per-module backend commands (dev, gateway, test, lint, migrate-rev) +│ ├── extensions/sources/ # Deployable snapshots of locally installed Python extensions │ ├── packages/extension-api/ # deerflow-extension-api package (import: deerflow_extension_api.*) — public extension contract │ ├── packages/harness/ # deerflow-harness package (import: deerflow.*) — agent framework │ └── app/ # FastAPI Gateway + IM channels (import: app.*) @@ -66,6 +67,7 @@ deer-flow/ │ # Managed integration skill packs are global at .deer-flow/integrations/skills/{provider}/ │ # Integration credentials and enabled state remain per-user ├── contracts/ # Cross-component JSON contracts (e.g. subagent status, skill review) +├── examples/deerflow-extension-example/ # Standalone package demonstrating all extension contribution kinds ├── scripts/ # Root orchestration scripts invoked by the Makefile (check, configure, doctor, support_bundle, serve, nginx, docker, deploy, setup_wizard) ├── tests/ # Root-level tests (currently tests/skills/ — public skill tests) └── docs/ # Cross-cutting docs, plans, and design notes @@ -73,8 +75,15 @@ deer-flow/ Third-party extensions are loaded from a top-level `plugins:` list in `config.yaml` (operator-controlled on purpose — that list causes code to be imported, so it is deliberately -kept out of the API-writable `extensions_config.json`). See the Extension System section in -[backend/AGENTS.md](backend/AGENTS.md). +kept out of the API-writable `extensions_config.json`). Packaged extensions can contribute +middleware, task lifecycle, system-model observers, Gateway services, and FastAPI HTTP +routers; the [reference extension](examples/deerflow-extension-example/) demonstrates all +five. Manage them with `deerflow extensions install/list/enable/disable/remove` or the root +`make extension-*` wrappers. Every mutation requires a Gateway restart, and both build +hooks and extension code execute with Gateway privileges, so only trusted operator sources +belong in this path. The manager transaction, accepted source forms, lock discipline, and +contribution contract live in +[the extensions guide](backend/packages/harness/deerflow/extensions/AGENTS.md). Runtime config lives at the **repo root**: copy `config.example.yaml` → `config.yaml` (main app config) and `extensions_config.example.json` → `extensions_config.json` (MCP @@ -105,6 +114,11 @@ make support-bundle # Generate redacted troubleshooting summary, AI issue draft make config # Generate local config files from the examples make check # Check that required tools are installed make install # Install all dependencies (frontend + backend + pre-commit hooks) +make extension-install SOURCE=... # Install and enable a trusted Python extension +make extension-list # List configured Python extensions +make extension-enable NAME=... # Enable an installed extension (restart required) +make extension-disable NAME=... # Disable without uninstalling (restart required) +make extension-remove NAME=... # Remove package and config entry (restart required) make dev # Start all services with hot-reload (Gateway + Frontend + Nginx) make start # Start all services in production mode (local, optimized) make stop # Stop all running services diff --git a/Makefile b/Makefile index b40a77b08..ce37510a5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # DeerFlow - Unified Development Environment -.PHONY: help config config-upgrade check check-agent-guidance install setup doctor support-bundle detect-thread-boundaries detect-blocking-io dev dev-daemon start start-daemon nginx stop up down clean docker-init docker-start docker-stop docker-logs docker-logs-frontend docker-logs-gateway docker-logs-redis +.PHONY: help config config-upgrade check check-agent-guidance install extension-install extension-list extension-enable extension-disable extension-remove setup doctor support-bundle detect-thread-boundaries detect-blocking-io dev dev-daemon start start-daemon nginx stop up down clean docker-init docker-start docker-stop docker-logs docker-logs-frontend docker-logs-gateway docker-logs-redis BASH ?= bash BACKEND_UV_RUN = cd backend && uv run @@ -30,6 +30,11 @@ help: @echo " make detect-thread-boundaries - Inventory backend executor/thread/event-loop boundaries" @echo " make detect-blocking-io - Inventory blocking IO that may block the backend event loop" @echo " make install - Install all dependencies (frontend + backend + pre-commit hooks)" + @echo " make extension-install SOURCE=... - Install and enable a trusted Python extension" + @echo " make extension-list - List configured Python extensions" + @echo " make extension-enable NAME=... - Enable an installed extension" + @echo " make extension-disable NAME=... - Disable an extension without uninstalling it" + @echo " make extension-remove NAME=... - Uninstall a managed extension" @echo " make setup-sandbox - Pre-pull sandbox container image (recommended)" @echo " make dev - Start all services in development mode (with hot-reloading)" @echo " make dev-daemon - Start dev services in background (daemon mode)" @@ -84,7 +89,7 @@ check-agent-guidance: # Install all dependencies install: @echo "Installing backend dependencies..." - @cd backend && uv sync + @cd backend && uv sync --locked @echo "Installing frontend dependencies..." @cd frontend && $(FRONTEND_PNPM) install @echo "Installing pre-commit hooks..." @@ -100,6 +105,29 @@ install: @echo " make setup-sandbox" @echo "" +extension-install: export DEER_FLOW_EXTENSION_SOURCE := $(value SOURCE) +extension-install: + $(if $(and $(filter command line,$(origin SOURCE)),$(strip $(value SOURCE))),,$(error usage: make extension-install SOURCE=)) + @cd backend && uv run --frozen --no-group extensions deerflow extensions install --source-env __deerflow_extension_source__ + +extension-list: + @cd backend && uv run --frozen --no-group extensions deerflow extensions list + +extension-enable: export DEER_FLOW_EXTENSION_NAME := $(value NAME) +extension-enable: + $(if $(and $(filter command line,$(origin NAME)),$(strip $(value NAME))),,$(error usage: make extension-enable NAME=)) + @cd backend && uv run --frozen --no-group extensions deerflow extensions enable --name-env __deerflow_extension_name__ + +extension-disable: export DEER_FLOW_EXTENSION_NAME := $(value NAME) +extension-disable: + $(if $(and $(filter command line,$(origin NAME)),$(strip $(value NAME))),,$(error usage: make extension-disable NAME=)) + @cd backend && uv run --frozen --no-group extensions deerflow extensions disable --name-env __deerflow_extension_name__ + +extension-remove: export DEER_FLOW_EXTENSION_NAME := $(value NAME) +extension-remove: + $(if $(and $(filter command line,$(origin NAME)),$(strip $(value NAME))),,$(error usage: make extension-remove NAME=)) + @cd backend && uv run --frozen --no-group extensions deerflow extensions remove --name-env __deerflow_extension_name__ + # Pre-pull sandbox Docker image (optional but recommended) setup-sandbox: @$(RUN_WITH_GIT_BASH) ./scripts/setup-sandbox.sh diff --git a/README.md b/README.md index a771a417b..69d4e2624 100644 --- a/README.md +++ b/README.md @@ -835,21 +835,117 @@ Advanced deployments can enable pluggable authorization with `authorization.enab Advanced deployments can also extend the agent runtime itself by declaring zero-argument `AgentMiddleware` classes under `extensions.middlewares` in `config.yaml` or `extensions_config.json`. DeerFlow loads the same configured class list into the lead-agent and subagent pipelines after their built-in runtime middlewares and loop/token guards, but before the terminal-response/safety/clarification tail, so enterprise forks can add domain guardrails, tool-call governance, or observability hooks without patching the built-in middleware builders. Missing packages, invalid classes, and broken modules fail loudly at agent creation. Treat `config.yaml` and `extensions_config.json` as trusted operator-controlled files: middleware paths are code execution, just like custom tool, model, sandbox, guardrail, MCP server, and MCP interceptor declarations. Gateway skill/MCP toggle endpoints preserve this field but do not expose an API write path for `extensions.middlewares`. Per-context parameterization and separate lead-only/subagent-only middleware lists are not supported yet. -For packaged and configurable runtime integrations, use the top-level `plugins:` list in -`config.yaml`. A plugin exposes `module.path:install`, depends only on the standalone -`deerflow-extension-api` contract package, and can register exactly three contribution -kinds: isolated middleware at semantic lead/subagent model or tool positions, lead and -subagent task-lifecycle hooks, and observers for DeerFlow-owned system model calls such as -goal evaluation, memory extraction, title generation, and summarization. DeerFlow allocates -a task-scoped extension store only when one of those contribution kinds is registered and -uses the Gateway's canonical notification loop for lifecycle and system-model callbacks, -including subagents that execute on isolated loops. Plugin order is deterministic, -per-plugin configuration is passed to `install()`, and `required: true` makes load failure -abort startup; otherwise failures are reported and skipped. Plugins load once when the -Gateway app is constructed, so changes require a restart. Because this imports Python code, -`plugins:` is intentionally unavailable through the API-writable -`extensions_config.json`. In Docker deployments, install the plugin in the Gateway image -rather than only in the host environment. See `config.example.yaml` for configuration. +For packaged and configurable runtime integrations, use DeerFlow's extension manager. +It accepts a Python package requirement, a public HTTPS Git URL, or a local directory, installs the +package into the backend's dedicated `extensions` dependency group, updates +`backend/uv.lock`, and adds an enabled entry to the startup-only top-level `plugins:` list +in `config.yaml`: + +```bash +# PyPI — pin a version for a reproducible deployment +make extension-install SOURCE="deerflow-extension-acme==1.2.3" + +# Public HTTPS Git — pin an immutable commit +make extension-install \ + SOURCE="git+https://github.com/acme/deerflow-extension-acme.git@0123456789abcdef0123456789abcdef01234567" + +# Local package — an absolute path avoids Make's backend-relative working directory +make extension-install SOURCE="$PWD/examples/deerflow-extension-example" + +make extension-list +make extension-disable NAME=acme +make extension-enable NAME=acme +make extension-remove NAME=acme +``` + +Installation is interactive because package installation can execute Python build hooks, +and the loaded extension later runs with Gateway privileges. For an already-reviewed +source, automation can acknowledge that boundary explicitly with +`cd backend && uv run --frozen --no-group extensions deerflow extensions install --yes`. +The manager requires uv 0.8.0 or newer; the provided Docker images pin uv 0.11.1. +The other direct +commands are `deerflow extensions list`, `enable NAME`, `disable NAME`, and `remove NAME`; +`NAME` may be the extension name, Python distribution, or `module:install` value. Do not +put credentials in a source URL — a URL carrying embedded userinfo or a credential-looking +query parameter is rejected before uv runs. Remote Git sources must use public HTTPS; SSH +Git URLs are rejected because the stock Docker builder does not forward host SSH +credentials. Installing from a loopback URL is allowed for local tooling but warns, because +`127.0.0.1` recorded in the lock is a different machine inside the Docker builder. + +A managed package declares exactly one standard PEP 621 entry point: + +```toml +[project.entry-points."deerflow.extensions"] +acme = "acme_deerflow_extension:install" +``` + +That callable uses the standalone `deerflow-extension-api` contract and can register five +contribution kinds: isolated middleware at semantic lead/subagent model or tool positions, +lead and subagent task-lifecycle hooks, observers for DeerFlow-owned model calls that are +not wrapped by middleware model-call hooks (goal, memory, title, and summarization), +Gateway-lifetime services, and eager FastAPI HTTP routers. The contract package has no +framework dependencies; extensions must declare FastAPI, LangChain, LangGraph, or other +libraries they import. + +DeerFlow allocates a task-scoped extension store only for middleware, lifecycle, or +system-model observation. Services receive app-scoped runtime dependencies after Gateway +persistence is ready and stop in reverse order after active runs drain. Extension HTTP +routers are mounted after every host route; definite shadows and routes entering the +host's authentication- or CSRF-exempt paths are rejected with attributed diagnostics, +while unrelated routers continue to load. Because the host's public paths are a reserved +prefix list that extensions cannot enter, **every contributed endpoint requires an +authenticated session** — there is currently no way for an extension to expose an +unauthenticated route, so inbound provider webhooks and public status endpoints are out of +scope for this release. Router startup/shutdown hooks, custom lifespans, +Mounts, and WebSocket routes are not accepted; lifetime resources belong in +`ExtensionService`, and WebSocket contributions require a future host-owned +authentication/Origin wrapper. Lifecycle and system-model callbacks use the Gateway's +canonical notification loop, including subagents on isolated loops. +Plugin order is deterministic, per-plugin configuration is passed to `install()`, and +`required: true` makes load failure abort startup; otherwise failures are reported and +skipped. `enabled: false` skips resolution and import. The manager preserves the extension's +private `config` when toggling it and writes `name`, `package`, `use`, `enabled`, and +`required` metadata for managed installs. Installs are recorded `required: false` so a +later broken extension is reported rather than blocking Gateway startup; pass +`extensions install --required` when the package's absence should abort startup +instead. Plugins load once when the Gateway app is +constructed, so install, enable, disable, remove, and manual `plugins:` edits all require a +Gateway restart. Because this imports Python code, `plugins:` is intentionally unavailable +through the API-writable `extensions_config.json`. + +Management commands bootstrap the checkout environment without the extension group via +`uv run --frozen --no-group extensions`. Frozen mode lets `disable` and `remove` start even +when an installed extension's remote source or managed snapshot has become unavailable, +while a fresh checkout can still create the non-extension environment from the existing lock. The +manager itself owns the subsequent locked dependency transaction. +Mutations for one checkout are serialized through a process lock. The initial manager +surface is create/remove rather than in-place upgrade: to change an installed source, save +its private `plugins[].config`, remove it, reinstall the new pin, and restore that config. + +Local-directory installs are copied into +`backend/extensions/sources//`; this deployable snapshot, rather +than the original directory, is recorded in the lock. Git metadata, virtual environments, +bytecode caches, symbolic links, and likely credential files are not accepted as snapshot +content. Review what you install anyway: filtering accidental files does not sandbox an +extension, its build backend, or its runtime code. + +Local `make dev`/`make start`, Docker development, and the production Gateway image all +consume the same `backend/pyproject.toml` and `backend/uv.lock`. Local and Docker-dev +launchers perform a locked sync before starting; the production image performs that sync +during its build and includes managed local snapshots in the build context. Gateway runtime +commands then use the already-created environment without resolving or installing packages. +Local and Docker-development pre-start syncs may download missing locked artifacts. A +production deployment instead downloads them only during the explicit install or image +build; starting the resulting production Gateway container never resolves or installs +extensions from the network. A local wheel or `file://` Git URL is rejected because it +would not exist in the Docker build context; pass a source directory to create a managed +snapshot instead. Because environment configuration (such as a `UV_FIND_LINKS` wheelhouse) +can still resolve a plain package name to a local wheel, the manager audits every new lock +before enabling the extension: any local reference the stock image build cannot reproduce +rolls back the entire install or removal. +Rebuild with `make up` after changing the managed extension set. See +`config.example.yaml` and the +[reference extension](examples/deerflow-extension-example/) for a complete example. Gateway-generated follow-up suggestions now normalize both plain-string model output and block/list-style rich content before parsing the JSON array response, so provider-specific content wrappers do not silently drop suggestions. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index d33e1970f..f9dd77655 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -90,6 +90,11 @@ When making code changes, you MUST update the relevant documentation: ```bash make check # Check system requirements make install # Install all dependencies (frontend + backend) +make extension-install SOURCE=... # Install and enable a trusted Python extension +make extension-list # List configured Python extensions +make extension-enable NAME=... # Enable an installed extension +make extension-disable NAME=... # Disable an extension without uninstalling it +make extension-remove NAME=... # Remove a managed extension make detect-thread-boundaries # Inventory backend executor/thread/event-loop boundaries make dev # Start all services (Gateway + Frontend + Nginx), with config.yaml preflight make start # Start production services locally diff --git a/backend/Dockerfile b/backend/Dockerfile index a284bb19b..b1b17595b 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -4,7 +4,7 @@ # Stage 3 (runtime): clean image without compiler toolchain for production # UV source image (override for restricted networks that cannot reach ghcr.io) -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.7.20 +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.1 FROM ${UV_IMAGE} AS uv-source # ── Stage 1: Builder ────────────────────────────────────────────────────────── @@ -28,6 +28,7 @@ RUN if [ -n "${APT_MIRROR}" ]; then \ # Install build tools + Node.js (build-essential needed for native Python extensions) RUN apt-get update && apt-get install -y \ curl \ + git \ build-essential \ gnupg \ ca-certificates \ @@ -70,7 +71,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ esac; \ extras_flags="$extras_flags --extra $extra"; \ done; \ - UV_INDEX_URL=${UV_INDEX_URL:-https://pypi.org/simple} uv sync --extra redis $extras_flags' + UV_INDEX_URL=${UV_INDEX_URL:-https://pypi.org/simple} uv sync --locked --extra redis $extras_flags' # UTF-8 locale prevents UnicodeEncodeError on Chinese/emoji content in minimal # containers where locale configuration may be missing and the default encoding is not UTF-8. @@ -88,7 +89,7 @@ COPY --from=docker:cli /usr/local/bin/docker /usr/local/bin/docker EXPOSE 8001 -CMD ["sh", "-c", "cd backend && PYTHONPATH=. uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001"] +CMD ["sh", "-c", "cd backend && PYTHONPATH=. uv run --no-sync uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001"] # ── Stage 3: Runtime ────────────────────────────────────────────────────────── # Clean image without build-essential — reduces size (~200 MB) and attack surface. diff --git a/backend/Makefile b/backend/Makefile index 31b271954..19aa66d00 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -3,11 +3,11 @@ DEER_FLOW_HOME := $(abspath $(DEER_FLOW_HOME)) BACKEND_SANDBOX_HOME := $(abspath $(CURDIR)/sandbox) install: - uv sync + uv sync --locked dev: mkdir -p "$(DEER_FLOW_HOME)" "$(BACKEND_SANDBOX_HOME)" - PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 DEER_FLOW_HOME="$(DEER_FLOW_HOME)" uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 \ + PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 DEER_FLOW_HOME="$(DEER_FLOW_HOME)" uv run --locked uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 \ --reload \ --reload-include='*.yaml' \ --reload-include='.env' \ @@ -17,7 +17,7 @@ dev: --reload-exclude="$(DEER_FLOW_HOME)" gateway: - PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 + PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run --locked uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 test: PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run pytest -m "not live" tests/ -v diff --git a/backend/app/gateway/app.py b/backend/app/gateway/app.py index 20472649d..5cba7f55d 100644 --- a/backend/app/gateway/app.py +++ b/backend/app/gateway/app.py @@ -614,6 +614,7 @@ This gateway provides runtime endpoints for agent runs plus custom endpoints for ExtensionLoadError, initialize_runtime_diagnostics, load_extensions, + record_runtime_diagnostics, set_loaded_extensions, ) @@ -738,6 +739,14 @@ This gateway provides runtime endpoints for agent runs plus custom endpoints for """ return {"status": "healthy", "service": "deer-flow-gateway"} + # Extension routes are deliberately last: FastAPI/Starlette dispatches in + # registration order, so every host route (including conditional routes + # and /health) keeps precedence. Definite shadows are rejected with an + # attributed diagnostic while unrelated extension routers still mount. + from deerflow.extensions.gateway import include_contributed_routers + + record_runtime_diagnostics(include_contributed_routers(app, loaded_extensions)) + return app diff --git a/backend/app/gateway/auth_middleware.py b/backend/app/gateway/auth_middleware.py index 0f8bc36a9..e1bc0f468 100644 --- a/backend/app/gateway/auth_middleware.py +++ b/backend/app/gateway/auth_middleware.py @@ -26,6 +26,7 @@ from app.gateway.auth_disabled import ( ) from app.gateway.authz import AuthContext, resolve_route_permissions from app.gateway.internal_auth import INTERNAL_AUTH_HEADER_NAME, get_internal_user, is_valid_internal_auth_token +from app.gateway.request_path import get_request_route_path from deerflow.runtime.user_context import reset_current_user, set_current_user # Paths that never require authentication. @@ -86,7 +87,7 @@ class AuthMiddleware(BaseHTTPMiddleware): super().__init__(app) async def dispatch(self, request: Request, call_next: Callable) -> Response: - if _is_public(request.url.path): + if _is_public(get_request_route_path(request)): return await call_next(request) internal_user = None diff --git a/backend/app/gateway/csrf_middleware.py b/backend/app/gateway/csrf_middleware.py index fd9e3b2a1..d777417bf 100644 --- a/backend/app/gateway/csrf_middleware.py +++ b/backend/app/gateway/csrf_middleware.py @@ -17,10 +17,13 @@ from starlette.types import ASGIApp from app.gateway.auth.config import get_auth_config from app.gateway.auth.session_cookie_state import SESSION_COOKIE_ISSUED_STATE_ATTR, SESSION_COOKIE_MAX_AGE_STATE_ATTR, SESSION_COOKIE_SECURE_STATE_ATTR, SKIP_AUTH_CSRF_COOKIE_STATE_ATTR from app.gateway.auth_disabled import is_auth_disabled +from app.gateway.request_path import get_request_route_path CSRF_COOKIE_NAME = "csrf_token" CSRF_HEADER_NAME = "X-CSRF-Token" CSRF_TOKEN_LENGTH = 64 # bytes +_CSRF_STATE_CHANGING_METHODS: frozenset[str] = frozenset({"POST", "PUT", "DELETE", "PATCH"}) +_CSRF_EXEMPT_EXACT_PATHS: frozenset[str] = frozenset({"/api/v1/auth/me"}) def is_secure_request(request: Request) -> bool: @@ -39,19 +42,20 @@ def should_check_csrf(request: Request) -> bool: CSRF is checked for state-changing methods (POST, PUT, DELETE, PATCH). GET, HEAD, OPTIONS, and TRACE are exempt per RFC 7231. """ - if request.method not in ("POST", "PUT", "DELETE", "PATCH"): + if request.method not in _CSRF_STATE_CHANGING_METHODS: return False if is_auth_disabled(): return False - path = request.url.path.rstrip("/") - # Exempt /api/v1/auth/me endpoint - if path == "/api/v1/auth/me": + route_path = get_request_route_path(request) + path = route_path.rstrip("/") + # Exempt host-owned endpoints that implement their own request posture. + if path in _CSRF_EXEMPT_EXACT_PATHS: return False # Inbound webhooks authenticate themselves via provider-specific signatures # (e.g. GitHub's X-Hub-Signature-256), not the CSRF double-submit cookie. - if request.url.path.startswith("/api/webhooks/"): + if route_path.startswith("/api/webhooks/"): return False return True @@ -71,7 +75,7 @@ def is_auth_endpoint(request: Request) -> bool: Auth endpoints don't need CSRF validation on first call (no token). """ - return request.url.path.rstrip("/") in _AUTH_EXEMPT_PATHS + return get_request_route_path(request).rstrip("/") in _AUTH_EXEMPT_PATHS def _host_with_optional_port(hostname: str, port: int | None, scheme: str) -> str: diff --git a/backend/app/gateway/deps.py b/backend/app/gateway/deps.py index 844e6db3a..36b468fc9 100644 --- a/backend/app/gateway/deps.py +++ b/backend/app/gateway/deps.py @@ -22,7 +22,7 @@ import logging import os from collections.abc import AsyncGenerator, Callable from contextlib import AsyncExitStack, asynccontextmanager -from typing import TYPE_CHECKING, TypeVar, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from fastapi import FastAPI, HTTPException, Request from langgraph.types import Checkpointer @@ -420,6 +420,9 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen # Initialize persistence engine BEFORE checkpointer so that # auto-create-database logic runs first (postgres backend). + # Own cleanup before initialization so partial startup and host + # cancellation cannot strand an engine created along the way. + stack.push_async_callback(close_engine) await init_engine_from_config(config.database) app.state.checkpointer = await stack.enter_async_context(make_checkpointer(config)) @@ -439,6 +442,35 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen app.state.run_store = MemoryRunStore() app.state.feedback_repo = None + # Services are app-scoped. Capture this app's immutable extension set + # once and close over the same object for teardown; the process-wide + # singleton may be replaced by another app/test before shutdown. + from deerflow.extensions import EMPTY_EXTENSIONS, record_runtime_diagnostics + from deerflow.extensions.gateway import start_services, stop_services + + extensions = getattr(app.state, "extensions", EMPTY_EXTENSIONS) + attempted_services: list[tuple[str, Any]] = [] + + async def stop_extension_services() -> None: + record_runtime_diagnostics( + await stop_services( + extensions, + service_entries=attempted_services, + ) + ) + + # Register cleanup before starting: start() can partially acquire + # resources and then fail or be cancelled. + stack.push_async_callback(stop_extension_services) + record_runtime_diagnostics( + await start_services( + extensions, + config, + sf, + attempted_services=attempted_services, + ) + ) + from deerflow.persistence.thread_meta import make_thread_store app.state.thread_store = make_thread_store(sf, app.state.store) @@ -544,7 +576,6 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen ), ), ) - await close_engine() # --------------------------------------------------------------------------- diff --git a/backend/app/gateway/request_path.py b/backend/app/gateway/request_path.py new file mode 100644 index 000000000..3ba022871 --- /dev/null +++ b/backend/app/gateway/request_path.py @@ -0,0 +1,9 @@ +"""Canonical request-path projection shared by routing security middleware.""" + +from starlette._utils import get_route_path +from starlette.requests import Request + + +def get_request_route_path(request: Request) -> str: + """Return the same root-path-adjusted value Starlette routes match.""" + return get_route_path(request.scope) diff --git a/backend/extension_test_fixtures/demo_extensions.py b/backend/extension_test_fixtures/demo_extensions.py index 67b81bea4..5f978bf59 100644 --- a/backend/extension_test_fixtures/demo_extensions.py +++ b/backend/extension_test_fixtures/demo_extensions.py @@ -41,9 +41,13 @@ def install_newer_minor_api(registry: ExtensionRegistry, config: Mapping[str, An def install_partial_then_raise(registry: ExtensionRegistry, config: Mapping[str, Any]) -> None: - """Registers two contributors, then fails — exercises rollback.""" - registry.middlewares(_Contributor("partial")) - registry.task_lifecycle(_Contributor("partial")) + """Register every contribution kind, then fail to exercise five-bucket rollback.""" + partial = _Contributor("partial") + registry.middlewares(partial) + registry.task_lifecycle(partial) + registry.system_model_observer(partial) + registry.service(partial) + registry.routers((partial,)) raise ValueError("boom") diff --git a/backend/packages/extension-api/deerflow_extension_api/__init__.py b/backend/packages/extension-api/deerflow_extension_api/__init__.py index 5b9a65f1b..17a1dc728 100644 --- a/backend/packages/extension-api/deerflow_extension_api/__init__.py +++ b/backend/packages/extension-api/deerflow_extension_api/__init__.py @@ -1,8 +1,8 @@ """Public contracts for DeerFlow extensions. -This package MUST NOT import `deerflow`. Everything an extension needs to -integrate lives here, so an extension depends on this package alone and can -be released independently of the host. +This package MUST NOT import `deerflow`. Every host contract an extension +needs lives here, while framework imports remain direct extension dependencies; +extensions can therefore be released independently of the host. """ from __future__ import annotations @@ -10,6 +10,8 @@ from __future__ import annotations from deerflow_extension_api.contracts import ( ExtensionInstall, ExtensionRegistry, + ExtensionRuntimeDeps, + ExtensionService, HostPolicySnapshot, MiddlewareContributor, SystemModelCallObserver, @@ -33,11 +35,9 @@ from deerflow_extension_api.runtime_bridge import ( ) from deerflow_extension_api.state import ExtensionData -#: Contract version. Pre-1.0: the contract surface is observational only -#: (contributors and observers), so minors may break and only patches promise -#: to be additive. From 1.0 on, bump the major on any breaking change; see the -#: spec's evolution rules for what counts as additive. -API_VERSION = "0.1.1" +#: Contract version. Before 1.0, minors may break and patches are additive. +#: From 1.0 on, bump the major for breaking changes. +API_VERSION = "0.1.2" __all__ = [ "API_VERSION", @@ -47,6 +47,8 @@ __all__ = [ "ExtensionData", "ExtensionInstall", "ExtensionRegistry", + "ExtensionRuntimeDeps", + "ExtensionService", "HostPolicySnapshot", "MiddlewareContributor", "MiddlewarePlacement", diff --git a/backend/packages/extension-api/deerflow_extension_api/contracts.py b/backend/packages/extension-api/deerflow_extension_api/contracts.py index da66894f9..441cb7667 100644 --- a/backend/packages/extension-api/deerflow_extension_api/contracts.py +++ b/backend/packages/extension-api/deerflow_extension_api/contracts.py @@ -10,7 +10,7 @@ Compatibility rules enforced throughout this module: from __future__ import annotations from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import StrEnum from typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar, runtime_checkable @@ -152,6 +152,26 @@ class MiddlewareContributor(Protocol): return () +# --- Extension services ---------------------------------------------------- + + +@dataclass(frozen=True) +class ExtensionRuntimeDeps: + """Host capabilities bound after Gateway infrastructure is ready.""" + + app_store: ExtensionData | None = None + policy: HostPolicySnapshot = field(default_factory=HostPolicySnapshot) + session_factory: Any | None = None + + +class ExtensionService(Protocol): + async def start(self, deps: ExtensionRuntimeDeps) -> None: + return None + + async def stop(self) -> None: + return None + + # --- Registration surface --------------------------------------------------- @@ -159,11 +179,10 @@ class MiddlewareContributor(Protocol): class ExtensionRegistry(Protocol): """The write-only registration surface handed to ``install()``. - Structural and minimal on purpose. This first capability slice exposes - middleware contribution only; later slices can add defaulted registration - methods without breaking existing implementations. The host's concrete - registry additionally carries host-only machinery (attribution, positional - rollback, build) that is deliberately absent here. + Structural and minimal on purpose. Every method has a default so additive + contract releases remain compatible with older registry implementations. + The host's concrete registry additionally carries host-only machinery + (attribution, positional rollback, build) that is deliberately absent here. """ def middlewares(self, contributor: MiddlewareContributor) -> None: @@ -175,6 +194,18 @@ class ExtensionRegistry(Protocol): def system_model_observer(self, observer: SystemModelCallObserver) -> None: return None + def service(self, service: ExtensionService) -> None: + return None + + def routers(self, routers: Sequence[Any]) -> None: + """Register HTTP routers constructed eagerly during extension install. + + Router types stay ``Any`` so this contract package has no FastAPI + dependency. The host validates supported route shapes before mounting; + runtime resources belong in a separately registered service. + """ + return None + #: The install() entry point signature every extension exposes. ExtensionInstall = Callable[[ExtensionRegistry, Mapping[str, Any]], None] diff --git a/backend/packages/extension-api/pyproject.toml b/backend/packages/extension-api/pyproject.toml index f245f3fe7..4a1b0652d 100644 --- a/backend/packages/extension-api/pyproject.toml +++ b/backend/packages/extension-api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "deerflow-extension-api" -version = "0.1.1" +version = "0.1.2" description = "Public contracts for DeerFlow extensions" requires-python = ">=3.12" # Keep the contract package import-light and independent from the host. Public diff --git a/backend/packages/harness/deerflow/extensions/AGENTS.md b/backend/packages/harness/deerflow/extensions/AGENTS.md index 40eb55cbd..055fcf61f 100644 --- a/backend/packages/harness/deerflow/extensions/AGENTS.md +++ b/backend/packages/harness/deerflow/extensions/AGENTS.md @@ -1,4 +1,4 @@ -### Python Extension System (Runtime Slice) +### Python Extension System (Runtime and Distribution) Third-party Python packages can expose an `install(registry, config)` function and be loaded, in deterministic order, from the startup-only top-level `plugins:` list in @@ -7,12 +7,142 @@ through Gateway APIs, while importing Python entry points is an operator-control execution boundary. A plugin marked `required: true` fails Gateway construction when it cannot load; optional plugins fail open with attributed diagnostics. -The public package is `packages/extension-api/` and must never import `deerflow`. Its -registry contract exposes exactly three contribution kinds: middleware contributors, -task-lifecycle contributors, and system-model-call observers. Middleware contributions -declare lead/subagent scope, stable order, and a semantic placement (`MODEL_LOGICAL`, -`MODEL_PHYSICAL`, `TOOL_VISIBLE`, `TOOL_RAW`, or `STANDARD`) rather than a fragile list -index. `extensions/stack.py` is the single final composition point; do not inject inside +Packaged extensions use one PEP 621 entry point in the +`deerflow.extensions` group, for example +`example = "deerflow_extension_example:install"`. The operator CLI is dispatched from +the existing `deerflow` console script to `extensions/cli.py` and exposes only these +surfaces: `install SOURCE [--yes]`, `list`, `enable NAME`, `disable NAME`, and +`remove NAME`. `NAME` resolves against the entry-point name, distribution name, or +`module:install` value. The root `make extension-*` targets are convenience wrappers; +because they execute from `backend/`, documentation should use absolute local source +paths with `SOURCE=` unless backend-relative behavior is intentional. + +`ExtensionManager` owns the package/config transaction. Install runs a controlled +`uv add --project --group extensions --no-workspace --no-sync -- `, updates the dedicated +`[dependency-groups].extensions` list and `uv.lock`, discovers exactly one packaging entry +point, and inserts or adopts one +managed `plugins:` record with `name`, `package`, `use`, `enabled`, `required`, and +private `config`. New records are written `required: false`, matching the loader default: +`required: true` turns any later load failure — a broken wheel, a missing native library, a +deleted snapshot — into a Gateway startup abort recoverable only through shell access, so +it is an explicit `install --required` opt-in rather than the managed default. Adoption of +an existing hand-written record preserves whatever `required` the operator already chose. +Enable/disable changes only the host-level `enabled` flag and preserves +private configuration. Remove runs `uv remove --group extensions`, removes the plugin +record, and deletes its managed source snapshot. Install validates the selected config +file before running any uv command, because `uv add`/`uv sync` execute the package's build +backend: a config this manager could never write to must fail before that code runs, not +afterwards through rollback. +Failed install/remove operations restore `pyproject.toml` and `uv.lock` and resynchronize +the restored environment; that second restore runs even when the recovery sync itself fails +(a recovery sync without `--locked` writes a lock while resolving), and a failing recovery +sync reports the original failure alongside it. The restore is deliberately not blanket: +when recovery detects a concurrent external edit to the dependency files or the config it +preserves that edit and raises instead, and `remove` leaves the plugin deactivated in that +case rather than reviving a record whose package declaration may already be gone. A +cancellation skips the recovery sync entirely — the declarations are already restored and +the next locked startup sync reconciles the environment, whereas blocking an interrupt on a +full dependency resolve invites a second interrupt that escapes the handler mid-transaction. +Package mutation is deferred from environment mutation: after `uv add/remove` +updates the declaration and lock, one `uv sync --locked --all-packages` preserves the same +config-/environment-detected optional extras as normal startup. All three uv calls pin the +backend project explicitly and discard UV environment overrides that could redirect the +project, working directory, sync mode, lock policy, or target environment — including +`UV_PYTHON`, which would swap the interpreter that then loads the extension entry point, +and `UV_INSECURE_HOST`, which would remove the TLS validation the HTTPS-only source rule +depends on; index, proxy, cache, and credential-provider settings remain available. +The `--no-workspace` boundary requires uv 0.8.0 or newer. The stock Docker paths pin uv +0.11.1, and the manager fails before mutation when the host uv is older. +All install/remove/enable/disable mutations for a checkout hold the cross-process +`.deer-flow/extension-manager.lock`; remove deactivates config before changing the package +declaration, and rollback preserves a concurrent external config edit instead of replacing +it. The MVP has no in-place upgrade: operators retain private config, remove the old +package, install the new source pin, and restore that config. + +Local-directory installs are snapshots, not editable links. The manager validates the +source, derives the destination from the normalized distribution name, and copies it to +`backend/extensions/sources//`. It ignores Git metadata, virtual +environments, Python caches, and bytecode; rejects symbolic links, path-escaping +distribution names, and likely credential files; and the root `.dockerignore` explicitly +re-includes the entire managed tree so package READMEs, native modules, and assets reach +the backend builder. These checks prevent common packaging accidents, not malicious +code. Both Python build hooks and imported extension code execute with Gateway +privileges, so the CLI requires confirmation (or explicit `--yes`) and accepts only +trusted operator sources; source URLs containing embedded credentials are rejected. +Remote direct references are limited to HTTPS, and remote Git sources must use public +Git-over-HTTPS (with loopback HTTP accepted for local tooling). SSH Git URLs are rejected +because the stock Docker builder does not forward host SSH credentials; relative paths and +local wheels must use the managed directory snapshot path instead. Git's SCP-like shorthand +(`git@host:org/repo.git`) carries no URL scheme, so it is detected before the scheme rules +and reported with the same public-HTTPS correction rather than the local-path message. +Local wheel and `file://` sources are rejected because they cannot be reproduced inside the +Docker build context; local code must enter through the directory-snapshot path. Stock +production builds support public package indexes and public HTTPS Git sources reachable by +the builder; authenticated source configuration must not be embedded in the recorded URL. +Source validation alone cannot catch environment-driven resolution (for example a +`UV_FIND_LINKS` wheelhouse turning a plain package requirement into a local wheel +reference), so after every `uv add/remove` the manager audits the new lock before syncing +or enabling anything. Any local reference that the stock backend image build cannot +reproduce — absolute paths, `file:` URLs, or relative paths outside the project root, its +exact workspace members, and the managed `extensions/sources/` snapshots — fails the whole +transaction and rolls back the dependency files, config, snapshot, and environment. A +loopback URL recorded in the lock is warned about rather than rolled back: `127.0.0.1` +inside the image builder is a different machine, so the reference is just as +non-reproducible, but unlike an environment-driven wheelhouse resolution it is a source the +operator typed deliberately. A private-network index is left alone entirely — a builder on +that network can reach it. A +config with duplicate top-level `plugins:` keys is rejected outright rather than managed +against one block while the Gateway reads another. + +The managed `plugins:` block is rewritten in place, and both of its boundaries come from +the YAML parser rather than a key-shaped pattern. `AppConfig` allows extra top-level keys, +so a neighbouring section may be named anything YAML accepts (`my.key`, `2fa`, `$schema`, a +non-ASCII word); a pattern that fails to recognize the next key does not fail loudly, it +reports "no next section" and the rewrite replaces that neighbour and its whole subtree. +Trailing comments below a file-final block are preserved for the same reason — the manager +appends `plugins:` at end of file, so that is the steady-state shape. + +Dependency synchronization has one lock authority: the manager's `uv add/remove` calls +are the only extension workflow allowed to update `backend/uv.lock`, and each mutation is +followed by the local-source audit described above. The `extensions` +group is included in `[tool.uv].default-groups` alongside `dev`. Root/backend install +targets use `uv sync --locked`; direct backend `make dev`/`make gateway` use +`uv run --locked`; the local full-stack launcher and Docker-dev entrypoint perform one +locked sync and then launch with `uv run --no-sync`; the production Docker builder syncs +the same copied backend project and lock, and both image runtime commands use +`--no-sync`. Thus production may download locked remote artifacts while building an +image, but production container startup never resolves or installs an extension from the +network. Local and Docker-dev pre-start syncs may fetch missing locked artifacts. +`docker/dev-entrypoint.sh` retries a failed sync once after recreating `.venv`, but keeps +`--locked` on the retry: that repairs a broken virtualenv, not a stale lock. A second +failure aborts with recovery instructions instead of starting uvicorn against an +environment that does not match the lock, because startup must never silently resolve +dependencies. +That discipline assumes the uv writing the lock and the uv reading it stay compatible, so +uv is pinned rather than floating: `backend/Dockerfile`'s `UV_IMAGE` is the single source of +truth, both compose defaults repeat it, and every `astral-sh/setup-uv` step pins the same +version so CI exercises the manager against the binary production actually runs. Otherwise a +newer uv can bump `uv.lock`'s `revision` (or make `uv lock --check` disagree with a lock +generated elsewhere) while CI stays green, and the pinned uv in the production image then +fails on the committed lock. `backend/tests/test_ci_uv_version_pin.py` keeps the four +locations in step, which makes a uv upgrade one deliberate, reviewable change. +Rebuild the Gateway image after changing the managed set. Every install, enable, disable, +remove, or config mutation also requires a Gateway restart because plugin loading is +startup-only. +The root management wrappers bootstrap the checkout environment without the extension group +via `uv run --frozen --no-group extensions`, so a broken or disappeared extension source cannot +trigger project validation before the operator can list, disable, or remove it, while a +fresh checkout can still install the non-extension environment from the existing lock. After CLI +entry, the manager owns the controlled locked sync. + +The public package is `packages/extension-api/` and must never import `deerflow` or carry +framework dependencies. Extensions declare any FastAPI, LangChain, or LangGraph imports +themselves. Its registry contract exposes five contribution kinds: middleware +contributors, task-lifecycle contributors, system-model-call observers, Gateway-lifetime +services, and eager routers. Middleware contributions declare lead/subagent scope, stable +order, and a semantic placement (`MODEL_LOGICAL`, `MODEL_PHYSICAL`, `TOOL_VISIBLE`, +`TOOL_RAW`, or `STANDARD`) rather than a fragile list index. `extensions/stack.py` is the +single final composition point; do not inject inside the shared base builder because the lead builder appends more middleware afterward. `extensions/ordering.py` owns host ordering invariants and validates the final composed stack. Nothing under `extensions/` may import `agents.middlewares` at module scope: the @@ -32,8 +162,9 @@ capability, so a single-sided wrapper receives a pass-through counterpart; imple both sides when the extension must observe both synchronous and asynchronous execution paths. -Lead runs and subagents allocate an `ExtensionData` task store only when at least one of -the three contribution kinds is registered. Middleware and system-call sites recover the +Lead runs and subagents allocate an `ExtensionData` task store only when middleware, +task-lifecycle, or system-model observation is registered; services and routers are +app-scoped and do not allocate one. Middleware and system-call sites recover the live store through `EXTENSION_TASK_STORE_KEY` / `task_store_from_runtime()`; lifecycle contributors receive that same store directly. Each task resolves the immutable loaded-extension snapshot once and binds that same object through task-store allocation, @@ -84,6 +215,47 @@ subagent's isolated loop, while synchronous system callbacks submit fire-and-for there. Shutdown stops accepting detached observations before the memory shutdown flush and resets the loop only after in-flight run/subagent drain ordering is complete. +Gateway services start in registration order after the persistence engine and session +factory are ready. Each receives the same `ExtensionRuntimeDeps` snapshot containing the +app store, projected host policy, and session factory. Start failures are attributed and +fail open. The runtime captures `app.state.extensions` once, registers cleanup before the +start batch, and stops the attempted service prefix in reverse order after run/subagent +drain but before store, checkpointer, and engine teardown. Each stop has an independent +bounded timeout; failures do not starve later cleanup. A service-originated +`CancelledError` fails open, while a new cancellation of the host task still propagates +through the exit stack. Runtime diagnostics must be appended through +`record_runtime_diagnostics()` so `app.state.extension_diagnostics` remains the canonical +live list. + +Routers are constructed eagerly during `install()` and mounted only after all host routes, +so host handlers always win. The Gateway rejects a contributed router atomically when an +earlier host or extension route provably covers one of its paths for the same HTTP method. +The conservative matcher proves common shadows through normalized parameter names, +static-vs-dynamic matching, known built-in-converter containment, supported compound +segments, full-segment `path` catch-alls, and `Mount` descendants reducible to those same +rules. Relationships requiring general regex-language inclusion are allowed rather than +guessed. Host WebSocket routes do not collide with contributed HTTP routes, but contributed +WebSocket routes are rejected until the host can supply authentication and Origin checks. +Because `include_router()` recompiles contributed routes, preflight projects the converter +registry at include time. Nonstandard converters fail closed against reserved security +paths but otherwise prove a shadow only when their normalized matchers are identical. +Host authentication- and CSRF-exempt paths are reserved, and contributed Mounts, unsupported +route items, startup/shutdown hooks, and custom router lifespans are rejected; lifetime +resources must use `ExtensionService`. Auth and CSRF classify +`get_request_route_path(request)`, the same root-path-adjusted ASGI path Starlette routes +match; do not switch those security predicates back to reconstructed `request.url.path`. +That helper delegates to the private `starlette._utils.get_route_path` on purpose. Its +requirement is not "strip `root_path` correctly" but "return exactly what the router is +matching on", so importing the dispatcher's own implementation keeps the two in lockstep by +construction. Do not vendor a local copy: a private import that disappears fails loudly at +startup, while a stale copy diverges silently at a security boundary. `starlette` is +therefore a declared, bounded direct dependency so the bump is visible in review, and +`tests/test_gateway_request_path.py` pins the agreement independently of the mechanism. +Any preflight, conflict, or include failure rolls back the whole router without preventing +later routers from mounting. Do not introduce a framework-bound `RouterContributor` +contract: the public registry accepts `Sequence[Any]` +to keep extension-api dependency-free. + The memory kind reaches those observers through a different shape, and the difference is deliberate rather than an oversight to be "aligned" away. DeerMem must stay vendorable and cannot import the extension API, so it reports through the `MemoryCallbacks.on_memory_llm_result` @@ -96,7 +268,8 @@ The host hook wrapper around the callback stays at `Exception`: only the hook's are non-fatal, and an observability path must not swallow `SystemExit` / `KeyboardInterrupt`. Gateway `create_app()` loads plugins once, stores the immutable registry on `app.state` -and in the process-wide singleton, and installs one canonical live diagnostics list. +and in the process-wide singleton, mounts contributed routers last, and installs one +canonical live diagnostics list. Changing `plugins` requires a restart. Any future contribution kind must be added to the public contract and host runtime in the same slice; never accept a registration method that the current host silently ignores. diff --git a/backend/packages/harness/deerflow/extensions/cli.py b/backend/packages/harness/deerflow/extensions/cli.py new file mode 100644 index 000000000..3200ce56f --- /dev/null +++ b/backend/packages/harness/deerflow/extensions/cli.py @@ -0,0 +1,127 @@ +"""Command-line interface for installing and managing trusted extensions.""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from collections.abc import Sequence +from pathlib import Path + +from deerflow.extensions.manager import ExtensionManager + +_NAME_ENV = "DEER_FLOW_EXTENSION_NAME" +_SOURCE_ENV = "DEER_FLOW_EXTENSION_SOURCE" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="deerflow extensions", + description="Install and manage trusted Python extensions for this DeerFlow checkout.", + ) + commands = parser.add_subparsers(dest="command", required=True) + install = commands.add_parser("install", help="install an extension and enable it in config.yaml") + install.add_argument("--source-env", action="store_true", help=argparse.SUPPRESS) + install.add_argument("source", help="local directory, Python package requirement, or Git URL") + install.add_argument( + "--yes", + action="store_true", + help="acknowledge that installing an extension executes trusted third-party code", + ) + install.add_argument( + "--required", + action="store_true", + help="abort Gateway startup when this extension fails to load (default: report and skip)", + ) + disable = commands.add_parser("disable", help="disable an extension without uninstalling it") + disable.add_argument("--name-env", action="store_true", help=argparse.SUPPRESS) + disable.add_argument("name", help="extension name, distribution, or module:install entry point") + enable = commands.add_parser("enable", help="enable an installed extension") + enable.add_argument("--name-env", action="store_true", help=argparse.SUPPRESS) + enable.add_argument("name", help="extension name, distribution, or module:install entry point") + commands.add_parser("list", help="list configured extensions") + remove = commands.add_parser("remove", help="uninstall an extension and remove its config entry") + remove.add_argument("--name-env", action="store_true", help=argparse.SUPPRESS) + remove.add_argument("name", help="extension name, distribution, or module:install entry point") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(list(argv) if argv is not None else None) + try: + root = find_project_root() + configured_path = os.environ.get("DEER_FLOW_CONFIG_PATH") + manager = ExtensionManager(root, config_path=configured_path) + if args.command == "install": + source = _source_argument(args) + trusted = args.yes + if not trusted: + print("Warning: a Python extension executes code with Gateway privileges.") + try: + trusted = input("Install this trusted source? [y/N] ").strip().lower() in {"y", "yes"} + except EOFError: + trusted = False + if not trusted: + print("Extension installation cancelled.", file=sys.stderr) + return 2 + installed = manager.install(source, yes=trusted, required=args.required) + print(f"Installed and enabled {installed.name} ({installed.distribution}). Restart DeerFlow to load it.") + return 0 + if args.command == "disable": + name = manager.set_enabled(_name_argument(args), enabled=False) + print(f"Disabled {name}. Restart DeerFlow to apply the change.") + return 0 + if args.command == "enable": + name = manager.set_enabled(_name_argument(args), enabled=True) + print(f"Enabled {name}. Restart DeerFlow to apply the change.") + return 0 + if args.command == "list": + configured = manager.list_configured() + print("NAME\tSTATE\tPACKAGE\tENTRY POINT") + for extension in configured: + state = "enabled" if extension.enabled else "disabled" + print(f"{extension.name}\t{state}\t{extension.distribution}\t{extension.use}") + return 0 + if args.command == "remove": + name = manager.remove(_name_argument(args)) + print(f"Removed {name}. Restart DeerFlow to apply the change.") + return 0 + except (OSError, RuntimeError, ValueError, subprocess.CalledProcessError) as exc: + print(f"extension command failed: {exc}", file=sys.stderr) + return 1 + raise AssertionError(f"unhandled extension command: {args.command}") + + +def _name_argument(args: argparse.Namespace) -> str: + if not args.name_env: + return args.name + name = os.environ.get(_NAME_ENV) + if name is None or not name.strip(): + raise ValueError(f"{_NAME_ENV} must contain an extension name") + return name + + +def _source_argument(args: argparse.Namespace) -> str: + if not args.source_env: + return args.source + source = os.environ.get(_SOURCE_ENV) + if source is None or not source.strip(): + raise ValueError(f"{_SOURCE_ENV} must contain an extension source") + return source + + +def find_project_root() -> Path: + configured = os.environ.get("DEER_FLOW_PROJECT_ROOT") + if configured: + candidate = Path(configured).expanduser().resolve() + if (candidate / "backend" / "pyproject.toml").is_file(): + return candidate + raise FileNotFoundError(f"DEER_FLOW_PROJECT_ROOT is not a DeerFlow checkout: {candidate}") + + candidates = (Path.cwd(), *Path.cwd().parents) + for candidate in candidates: + candidate = candidate.resolve() + if (candidate / "backend" / "pyproject.toml").is_file(): + return candidate + raise FileNotFoundError("could not find a DeerFlow checkout; set DEER_FLOW_PROJECT_ROOT") diff --git a/backend/packages/harness/deerflow/extensions/gateway.py b/backend/packages/harness/deerflow/extensions/gateway.py new file mode 100644 index 000000000..8aaec6541 --- /dev/null +++ b/backend/packages/harness/deerflow/extensions/gateway.py @@ -0,0 +1,637 @@ +"""Gateway-side plumbing for app-scoped extension contributions.""" + +from __future__ import annotations + +import asyncio +import logging +import re +from dataclasses import dataclass +from typing import Any + +from deerflow_extension_api import ExtensionRuntimeDeps + +from deerflow.extensions.loader import Diagnostic +from deerflow.extensions.policy import project_host_policy +from deerflow.extensions.registry import LoadedExtensions + +logger = logging.getLogger(__name__) + +DEFAULT_STOP_TIMEOUT_SECONDS = 30.0 + +_PATH_PARAMETER_GROUP = re.compile(r"\(\?P<[^>]+>") +_PATH_PARAMETER = re.compile(r"{([a-zA-Z_][a-zA-Z0-9_]*)(?::([a-zA-Z_][a-zA-Z0-9_]*))?}") +_RouteMethods = frozenset[str] | None +_RouteScopes = frozenset[str] +_HOST_PUBLIC_PATH_PREFIXES = ( + "/health", + "/docs", + "/redoc", + "/openapi.json", + "/api/v1/auth/oauth/", + "/api/v1/auth/callback/", + "/api/webhooks/", +) +_HOST_PUBLIC_EXACT_PATHS = frozenset( + { + "/api/v1/auth/login/local", + "/api/v1/auth/register", + "/api/v1/auth/logout", + "/api/v1/auth/setup-status", + "/api/v1/auth/initialize", + "/api/v1/auth/providers", + } +) +_HOST_CSRF_EXEMPT_EXACT_PATHS = frozenset({"/api/v1/auth/me"}) +_CSRF_STATE_CHANGING_METHODS = frozenset({"POST", "PUT", "DELETE", "PATCH"}) +_STANDARD_CONVERTOR_REGEXES = { + "str": "[^/]+", + "path": ".*", + "int": "[0-9]+", + "float": r"[0-9]+(\.[0-9]+)?", + "uuid": "[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}", +} + + +@dataclass(frozen=True) +class _RouteClaim: + path: str + matcher: str + methods: _RouteMethods + scopes: _RouteScopes + mount_prefix: str | None = None + standard_convertors: bool = True + + +_RouteOwners = list[tuple[_RouteClaim, str]] + + +def _cancellation_count() -> int: + task = asyncio.current_task() + return task.cancelling() if task is not None else 0 + + +def _route_path_matcher(route: Any) -> str | None: + path = getattr(route, "path", None) + if path is None: + return None + pattern = getattr(getattr(route, "path_regex", None), "pattern", None) + if not pattern: + return path + return _PATH_PARAMETER_GROUP.sub("(?:", pattern) + + +def _route_methods(route: Any) -> _RouteMethods: + methods = getattr(route, "methods", None) + return frozenset(methods) if methods else None + + +def _route_scopes(route: Any) -> _RouteScopes: + from starlette.routing import Mount, Route, WebSocketRoute + + if isinstance(route, WebSocketRoute): + return frozenset({"websocket"}) + if isinstance(route, Route): + return frozenset({"http"}) + if isinstance(route, Mount): + return frozenset({"http", "websocket"}) + return frozenset({"http", "websocket"}) + + +def _route_claim(route: Any, *, recompile: bool = False) -> _RouteClaim | None: + from starlette.routing import Mount, compile_path + + path = getattr(route, "path", None) + if recompile and path is not None: + path_regex, _path_format, param_convertors = compile_path(path) + matcher = _PATH_PARAMETER_GROUP.sub("(?:", path_regex.pattern) + else: + matcher = _route_path_matcher(route) + param_convertors = getattr(route, "param_convertors", {}) + if path is None or matcher is None: + return None + is_mount = isinstance(route, Mount) + return _RouteClaim( + path=path, + matcher=matcher, + methods=_route_methods(route), + scopes=_route_scopes(route), + mount_prefix=path.rstrip("/") if is_mount else None, + standard_convertors=_uses_standard_convertors( + path, + param_convertors, + is_mount=is_mount, + ), + ) + + +def _uses_standard_convertors( + path: str, + param_convertors: Any, + *, + is_mount: bool, +) -> bool: + for match in _PATH_PARAMETER.finditer(path): + parameter_name = match.group(1) + convertor_name = match.group(2) or "str" + expected_regex = _STANDARD_CONVERTOR_REGEXES.get(convertor_name) + actual_regex = getattr(param_convertors.get(parameter_name), "regex", None) + if expected_regex is None or actual_regex != expected_regex: + return False + + if is_mount: + return getattr(param_convertors.get("path"), "regex", None) == _STANDARD_CONVERTOR_REGEXES["path"] + return True + + +def _convertor_can_extend_prefix(convertor: str, prefix: str) -> bool: + """Return a proven built-in-convertor witness beginning with ``prefix``. + + Public-route protection is a security boundary, so an unknown custom + convertor fails closed when its value could begin inside a public prefix. + Shadow detection remains separate and continues to allow relationships it + cannot prove. + """ + if convertor == "path": + return True + if convertor == "str": + return "/" not in prefix + if convertor == "int": + return all(character in "0123456789" for character in prefix) + if convertor == "float": + if not prefix: + return True + return bool(re.fullmatch(r"[0-9]+", prefix) or re.fullmatch(r"[0-9]+\.", prefix) or re.fullmatch(r"[0-9]+\.[0-9]+", prefix)) + if convertor == "uuid": + groups = (8, 4, 4, 4, 12) + for mask in range(1 << (len(groups) - 1)): + shape = "" + for index, width in enumerate(groups): + shape += "h" * width + if index < len(groups) - 1 and mask & (1 << index): + shape += "-" + if len(prefix) <= len(shape) and all((expected == "h" and character in "0123456789abcdefABCDEF") or character == expected for character, expected in zip(prefix, shape, strict=False)): + return True + return False + return True + + +def _path_template_can_start_with(path: str, prefix: str) -> bool: + """Whether a built-in route template has a concrete path under ``prefix``.""" + from functools import cache + + tokens: list[tuple[str, str]] = [] + cursor = 0 + for match in _PATH_PARAMETER.finditer(path): + literal = path[cursor : match.start()] + if literal: + tokens.append(("literal", literal)) + tokens.append(("parameter", match.group(2) or "str")) + cursor = match.end() + trailing_literal = path[cursor:] + if trailing_literal: + tokens.append(("literal", trailing_literal)) + + @cache + def can_match(token_index: int, prefix_index: int) -> bool: + if prefix_index == len(prefix): + return True + if token_index == len(tokens): + return False + + kind, value = tokens[token_index] + remaining = prefix[prefix_index:] + if kind == "literal": + if value.startswith(remaining): + return True + if remaining.startswith(value): + return can_match(token_index + 1, prefix_index + len(value)) + return False + + registered_regex = _STANDARD_CONVERTOR_REGEXES.get(value) + if registered_regex is None: + return False + for end_index in range(prefix_index, len(prefix) + 1): + concrete = prefix[prefix_index:end_index] + if re.fullmatch(registered_regex, concrete) is not None and can_match( + token_index + 1, + end_index, + ): + return True + return _convertor_can_extend_prefix(value, remaining) + + return can_match(0, 0) + + +def _claim_can_enter_prefix(claim: _RouteClaim, prefix: str) -> bool: + if claim.standard_convertors: + return _path_template_can_start_with(claim.path, prefix) + literal_prefix = claim.path.partition("{")[0] + return claim.path.startswith(prefix) or prefix.startswith(literal_prefix) + + +def _claim_can_enter_exact_path(claim: _RouteClaim, exact_path: str) -> bool: + """Whether a route can dispatch to ``exact_path`` plus trailing slashes.""" + if not claim.standard_convertors: + return _claim_can_enter_prefix(claim, exact_path) + + # Auth and CSRF normalize with rstrip("/"), so two or more trailing + # slashes are just as exempt as one. For built-in convertors, a shortest + # slash-only witness is bounded by the template's literal length because + # ``path`` may be empty and every other built-in rejects slash. + for slash_count in range(len(claim.path) + 2): + if ( + re.fullmatch( + claim.matcher, + exact_path + "/" * slash_count, + ) + is not None + ): + return True + + # Custom regex language inclusion is deliberately not guessed at a + # security boundary. If its template can reach the exact-path prefix, + # reject it fail-closed; shadow matching below remains fail-open. + return False + + +def _router_routes(router: Any) -> list[_RouteClaim]: + from fastapi.routing import _DefaultLifespan + from starlette.routing import Mount, Route, WebSocketRoute + + if getattr(router, "on_startup", ()) or getattr(router, "on_shutdown", ()): + raise TypeError("contributed router lifecycle hooks are not supported; register an ExtensionService instead") + lifespan_context = getattr(router, "lifespan_context", None) + if lifespan_context is not None and not isinstance(lifespan_context, _DefaultLifespan): + raise TypeError("contributed router lifespan is not supported; register an ExtensionService instead") + + claims: list[_RouteClaim] = [] + for route in getattr(router, "routes", []): + if isinstance(route, Mount): + raise TypeError("contributed router contains a Starlette Mount, which FastAPI.include_router() ignores") + if isinstance(route, WebSocketRoute): + raise TypeError("contributed WebSocket routes are not supported until the host can apply authentication and Origin checks") + if not isinstance(route, Route): + raise TypeError(f"contributed router contains an unsupported route item: {type(route).__name__}") + # FastAPI.include_router() reconstructs every route from ``route.path`` + # using the converter registry at include time. Preflight must project + # those same semantics, not the router object's older compiled regex. + claim = _route_claim(route, recompile=True) + if claim is not None: + enters_public_exact_path = any(_claim_can_enter_exact_path(claim, public_path) for public_path in _HOST_PUBLIC_EXACT_PATHS) + enters_csrf_exact_path = _methods_overlap( + claim.methods, + _CSRF_STATE_CHANGING_METHODS, + ) and any(_claim_can_enter_exact_path(claim, exempt_path) for exempt_path in _HOST_CSRF_EXEMPT_EXACT_PATHS) + enters_public_prefix = any(_claim_can_enter_prefix(claim, public_prefix) for public_prefix in _HOST_PUBLIC_PATH_PREFIXES) + if enters_public_prefix: + raise TypeError(f"contributed route {claim.path} can enter a host public namespace") + if enters_public_exact_path or enters_csrf_exact_path: + raise TypeError(f"contributed route {claim.path} can enter a host-reserved exact path") + claims.append(claim) + return claims + + +def _methods_overlap(left: _RouteMethods, right: _RouteMethods) -> bool: + return left is None or right is None or not left.isdisjoint(right) + + +def _dispatches_overlap(left: _RouteClaim, right: _RouteClaim) -> bool: + shared_scopes = left.scopes & right.scopes + if "websocket" in shared_scopes: + return True + return "http" in shared_scopes and _methods_overlap(left.methods, right.methods) + + +def _path_shape(path: str) -> tuple[tuple[str, ...], tuple[str, ...]]: + literals: list[str] = [] + convertors: list[str] = [] + cursor = 0 + for match in _PATH_PARAMETER.finditer(path): + literals.append(path[cursor : match.start()]) + convertors.append(match.group(2) or "str") + cursor = match.end() + literals.append(path[cursor:]) + return tuple(literals), tuple(convertors) + + +def _convertor_covers(owner: str, candidate: str) -> bool: + if owner == candidate: + return True + if owner == "path": + return candidate in {"int", "float", "uuid"} + if owner == "str": + return candidate in {"str", "int", "float", "uuid"} + if owner == "float": + return candidate in {"float", "int"} + return False + + +def _path_segments(path: str) -> tuple[tuple[str, str], ...] | None: + if not path.startswith("/"): + return None + if path == "/": + return () + + segments: list[tuple[str, str]] = [] + for segment in path[1:].split("/"): + match = _PATH_PARAMETER.fullmatch(segment) + if match is not None: + segments.append(("parameter", match.group(2) or "str")) + elif _PATH_PARAMETER.search(segment): + segments.append(("compound", segment)) + else: + segments.append(("literal", segment)) + return tuple(segments) + + +def _static_segment_matches(convertor: str, value: str) -> bool: + registered_regex = _STANDARD_CONVERTOR_REGEXES.get(convertor) + return registered_regex is not None and re.fullmatch(registered_regex, value) is not None + + +def _compound_segment_covers(owner: str, candidate: str) -> bool: + owner_literals, owner_convertors = _path_shape(owner) + candidate_literals, candidate_convertors = _path_shape(candidate) + if ( + owner_literals == candidate_literals + and len(owner_convertors) == len(candidate_convertors) + and all( + _convertor_covers(owner_convertor, candidate_convertor) + for owner_convertor, candidate_convertor in zip( + owner_convertors, + candidate_convertors, + strict=True, + ) + ) + ): + return True + + if len(owner_convertors) != 1 or owner_convertors[0] != "str": + return False + owner_prefix, owner_suffix = owner_literals + minimum_candidate_length = sum(map(len, candidate_literals)) + sum(0 if convertor == "path" else 1 for convertor in candidate_convertors) + return ( + candidate_literals[0].startswith(owner_prefix) + and candidate_literals[-1].endswith(owner_suffix) + and minimum_candidate_length > len(owner_prefix) + len(owner_suffix) + and all(convertor in {"str", "int", "float", "uuid"} for convertor in candidate_convertors) + ) + + +def _compound_is_ascii_digits(compound: str) -> bool: + literals, convertors = _path_shape(compound) + return all(character in "0123456789" for literal in literals for character in literal) and all(convertor == "int" for convertor in convertors) + + +def _segment_excludes_newline(segment: tuple[str, str]) -> bool: + kind, value = segment + if kind == "literal": + return "\n" not in value + if kind == "parameter": + return value in {"path", "int", "float", "uuid"} + literals, convertors = _path_shape(value) + return all("\n" not in literal for literal in literals) and all(convertor in {"path", "int", "float", "uuid"} for convertor in convertors) + + +def _compound_segment_matches_static(compound: str, value: str) -> bool: + pattern = "" + cursor = 0 + for match in _PATH_PARAMETER.finditer(compound): + pattern += re.escape(compound[cursor : match.start()]) + convertor = match.group(2) or "str" + registered_regex = _STANDARD_CONVERTOR_REGEXES.get(convertor) + if registered_regex is None: + return False + pattern += f"(?:{registered_regex})" + cursor = match.end() + pattern += re.escape(compound[cursor:]) + return re.fullmatch(pattern, value) is not None + + +def _segment_covers(owner: tuple[str, str], candidate: tuple[str, str]) -> bool: + owner_kind, owner_value = owner + candidate_kind, candidate_value = candidate + if owner_kind == "literal": + return candidate_kind == "literal" and owner_value == candidate_value + if owner_kind == "parameter" and candidate_kind == "compound": + if owner_value == "path": + return _segment_excludes_newline(candidate) + if owner_value == "str": + return all((match.group(2) or "str") in {"str", "int", "float", "uuid"} for match in _PATH_PARAMETER.finditer(candidate_value)) + if owner_value in {"int", "float"}: + return _compound_is_ascii_digits(candidate_value) + return False + if owner_kind == "compound": + if candidate_kind == "literal": + return _compound_segment_matches_static(owner_value, candidate_value) + if candidate_kind == "compound": + return _compound_segment_covers(owner_value, candidate_value) + return False + if candidate_kind == "compound": + return False + if candidate_kind == "literal": + return _static_segment_matches(owner_value, candidate_value) + return _convertor_covers(owner_value, candidate_value) + + +def _segmented_path_covers(owner_path: str, candidate_path: str) -> bool: + owner = _path_segments(owner_path) + candidate = _path_segments(candidate_path) + if owner is None or candidate is None: + return False + + if owner and owner[-1] == ("parameter", "path"): + prefix = owner[:-1] + return ( + len(candidate) > len(prefix) + and all( + _segment_covers(owner_segment, candidate_segment) + for owner_segment, candidate_segment in zip( + prefix, + candidate, + strict=False, + ) + ) + and all(_segment_excludes_newline(candidate_segment) for candidate_segment in candidate[len(prefix) :]) + ) + + return len(owner) == len(candidate) and all(_segment_covers(owner_segment, candidate_segment) for owner_segment, candidate_segment in zip(owner, candidate, strict=True)) + + +def _matcher_covers(owner: _RouteClaim, candidate: _RouteClaim) -> bool: + """Return whether every candidate path is consumed by an earlier owner.""" + if owner.matcher == candidate.matcher: + return True + + if not owner.standard_convertors or not candidate.standard_convertors: + return False + + owner_literals, owner_convertors = _path_shape(owner.path) + candidate_literals, candidate_convertors = _path_shape(candidate.path) + + if not candidate_convertors: + return re.fullmatch(owner.matcher, candidate.path) is not None + + if owner.mount_prefix is not None: + if owner.mount_prefix == "": + candidate_segments = _path_segments(candidate.path) + return candidate_segments is not None and all(_segment_excludes_newline(segment) for segment in candidate_segments) + return _segmented_path_covers( + f"{owner.mount_prefix}/{{mount_path:path}}", + candidate.path, + ) + + if _segmented_path_covers(owner.path, candidate.path): + return True + + return ( + owner_literals == candidate_literals + and len(owner_convertors) == len(candidate_convertors) + and all( + _convertor_covers(owner_convertor, candidate_convertor) + for owner_convertor, candidate_convertor in zip( + owner_convertors, + candidate_convertors, + strict=True, + ) + ) + ) + + +def _find_route_clash( + routes: list[_RouteClaim], + owners: _RouteOwners, + candidate_holder: str, +) -> tuple[str, str] | None: + tentative_owners = list(owners) + for route in routes: + for owner, holder in tentative_owners: + if _dispatches_overlap(route, owner) and _matcher_covers(owner, route): + return route.path, holder + tentative_owners.append((route, candidate_holder)) + return None + + +def include_contributed_routers(app: Any, extensions: LoadedExtensions) -> list[Diagnostic]: + """Mount reachable routers in order and reject definite shadows atomically.""" + diagnostics: list[Diagnostic] = [] + if not extensions.routers: + return diagnostics + + mounted: list[str] = [] + owners: _RouteOwners = [] + for route in getattr(app, "routes", []): + claim = _route_claim(route) + if claim is not None: + owners.append((claim, "host")) + + for source, router in extensions.routers: + try: + routes = _router_routes(router) + if not routes: + raise TypeError(f"contributed router exposes no routes: {router!r}") + clash = _find_route_clash(routes, owners, source) + if clash is not None: + path, holder = clash + message = f"router path {path} is already served by {holder}; this router was not mounted" + diagnostics.append(Diagnostic.error(source, message)) + logger.error("Extension %s: %s", source, message) + continue + app_routes = getattr(getattr(app, "router", None), "routes", None) + route_mark = len(app_routes) if isinstance(app_routes, list) else None + try: + app.include_router(router) + except BaseException: + # FastAPI copies one route at a time. If a later copy fails, + # remove every route added by this attempt before either + # continuing fail-open or propagating a host-level exception. + if route_mark is not None: + del app_routes[route_mark:] + raise + for route in routes: + owners.append((route, source)) + mounted.append(f"{source} -> {route.path}") + except Exception as exc: + message = f"router could not be mounted; continuing without it: {exc}" + diagnostics.append(Diagnostic.error(source, message)) + logger.exception("Extension %s: %s", source, message) + + if mounted: + logger.info("Extension routers mounted: %s", "; ".join(mounted)) + return diagnostics + + +async def start_services( + extensions: LoadedExtensions, + app_config: Any, + session_factory: Any | None, + *, + attempted_services: list[tuple[str, Any]] | None = None, +) -> list[Diagnostic]: + """Start extension services in registration order, failing open per item.""" + diagnostics: list[Diagnostic] = [] + if not extensions.services: + return diagnostics + + deps = ExtensionRuntimeDeps( + app_store=extensions.app_store, + policy=project_host_policy(app_config), + session_factory=session_factory, + ) + for entry in extensions.services: + source, service = entry + if attempted_services is not None: + # Record before awaiting start(): a service may acquire resources + # and then fail or be cancelled, so it still owns stop(). + attempted_services.append(entry) + cancellation_count = _cancellation_count() + try: + await service.start(deps) + except asyncio.CancelledError: + if _cancellation_count() > cancellation_count: + raise + message = "service start() raised CancelledError; continuing without it" + diagnostics.append(Diagnostic.error(source, message)) + logger.exception("Extension %s: %s", source, message) + except Exception as exc: + message = f"service start() failed; continuing without it: {exc}" + diagnostics.append(Diagnostic.error(source, message)) + logger.exception("Extension %s: %s", source, message) + return diagnostics + + +async def stop_services( + extensions: LoadedExtensions, + timeout_seconds: float = DEFAULT_STOP_TIMEOUT_SECONDS, + *, + service_entries: tuple[tuple[str, Any], ...] | list[tuple[str, Any]] | None = None, +) -> list[Diagnostic]: + """Stop services in reverse order with an independent budget per item.""" + diagnostics: list[Diagnostic] = [] + entries = extensions.services if service_entries is None else service_entries + for source, service in reversed(entries): + cancellation_count = _cancellation_count() + timeout = asyncio.timeout(timeout_seconds) + try: + async with timeout: + await service.stop() + except TimeoutError as exc: + if timeout.expired(): + message = f"service stop() timed out after {timeout_seconds}s; continuing shutdown" + diagnostics.append(Diagnostic.error(source, message)) + logger.error("Extension %s: %s", source, message) + else: + message = f"service stop() failed; continuing shutdown: {exc}" + diagnostics.append(Diagnostic.error(source, message)) + logger.exception("Extension %s: %s", source, message) + except asyncio.CancelledError: + if _cancellation_count() > cancellation_count: + raise + message = "service stop() raised CancelledError; continuing shutdown" + diagnostics.append(Diagnostic.error(source, message)) + logger.exception("Extension %s: %s", source, message) + except Exception as exc: + message = f"service stop() failed; continuing shutdown: {exc}" + diagnostics.append(Diagnostic.error(source, message)) + logger.exception("Extension %s: %s", source, message) + return diagnostics diff --git a/backend/packages/harness/deerflow/extensions/loader.py b/backend/packages/harness/deerflow/extensions/loader.py index 6ddf5745b..fef1b67f8 100644 --- a/backend/packages/harness/deerflow/extensions/loader.py +++ b/backend/packages/harness/deerflow/extensions/loader.py @@ -29,6 +29,18 @@ class ExtensionSpec(BaseModel): model_config = ConfigDict(extra="forbid") + enabled: bool = Field( + default=True, + description="When false, skip the extension without resolving or importing it", + ) + name: str | None = Field( + default=None, + description="Stable operator-facing name recorded by the extension manager", + ) + package: str | None = Field( + default=None, + description="Installed Python distribution recorded by the extension manager", + ) use: str = Field(description="Entry point path, e.g. 'my_extension:install'") config: dict[str, Any] = Field( default_factory=dict, @@ -85,8 +97,8 @@ def _parse_version(version: object) -> tuple[int, ...] | None: def _compatible(declared: str, current: str) -> bool: """One-directional, with the semver window for the contract's life stage. - Pre-1.0 the contract surface is observational only and minors may break, - so the window is same major.minor with patches additive: host >= declared. + Pre-1.0 minors may break, so the window is same major.minor with patches + additive: host >= declared. From 1.0 on contracts only grow within a major, so a newer host stays compatible with older extensions while an extension written against a newer minor is refused — it would reach for contract additions the host @@ -130,6 +142,9 @@ def load_extensions(specs: Sequence[ExtensionSpec]) -> tuple[LoadedExtensions, l loaded_sources: list[str] = [] for spec in specs: + if not spec.enabled: + continue + try: install = resolve_variable(spec.use) except Exception as exc: diff --git a/backend/packages/harness/deerflow/extensions/manager.py b/backend/packages/harness/deerflow/extensions/manager.py new file mode 100644 index 000000000..533d306c5 --- /dev/null +++ b/backend/packages/harness/deerflow/extensions/manager.py @@ -0,0 +1,1002 @@ +"""Operator-facing installation management for packaged DeerFlow extensions.""" + +from __future__ import annotations + +import ipaddress +import json +import logging +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +import time +import tomllib +import urllib.parse +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml +from packaging.requirements import InvalidRequirement, Requirement + +logger = logging.getLogger(__name__) + +_ENTRY_POINT_GROUP = "deerflow.extensions" +_LOCK_RETRY_INTERVAL_SECONDS = 0.2 +_SNAPSHOT_IGNORES = (".git", ".venv", "venv", "__pycache__", "*.pyc") +_DISTRIBUTION_NAME = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$") +_SENSITIVE_FILENAMES = {".npmrc", ".pypirc", "credentials.json"} +_SENSITIVE_SUFFIXES = {".key", ".pem", ".p12", ".pfx"} +_SECRET_QUERY_KEY = re.compile( + r"(?:^|[-_])(?:api[-_]?key|access[-_]?key|auth(?:orization)?|code|credential|key|pass(?:wd|word)?|pw|sas|secret|signature|sig|token)(?:$|[-_])", + re.IGNORECASE, +) +# The camel-case splitter only fires on case transitions, so `accessToken` is +# separated into words while `accesstoken` and `ACCESSTOKEN` are not. These +# spellings are distinctive enough to match without a word boundary; short +# generic words stay boundary-anchored above so `keyword` is still installable. +_SECRET_QUERY_SUBSTRING = re.compile( + r"access[-_]?token|api[-_]?key|auth[-_]?token|session[-_]?token|credential|password|passwd|signature|secret", + re.IGNORECASE, +) +# Git's SCP-like shorthand (`git@host:org/repo.git`) is a remote source that +# carries no URL scheme, so it reaches validation looking like a bare path. +_SCP_LIKE_REFERENCE = re.compile(r"^[^\s/:@]+@[^\s/:@]+:(?!/)") +_PEP508_NAME_PREFIX = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*\s*@\s*") +_UV_ENV_OVERRIDES = { + "UV_ACTIVE", + "UV_ALL_GROUPS", + "UV_CONFIG_FILE", + "UV_CONSTRAINT", + "UV_DEFAULT_GROUPS", + "UV_DEV", + "UV_FROZEN", + "UV_INSECURE_HOST", + "UV_LOCKED", + "UV_NO_BUILD_ISOLATION", + "UV_NO_CONFIG", + "UV_NO_DEFAULT_GROUPS", + "UV_NO_DEV", + "UV_NO_GROUP", + "UV_NO_SOURCES", + "UV_NO_SYNC", + "UV_ONLY_DEV", + "UV_ONLY_GROUP", + "UV_PACKAGE", + "UV_PROJECT", + "UV_PROJECT_ENVIRONMENT", + "UV_PYTHON", + "UV_SCRIPT", + "UV_WORKING_DIR", +} + + +@dataclass(frozen=True) +class InstalledExtension: + """One extension made importable and active by :class:`ExtensionManager`.""" + + name: str + distribution: str + use: str + + +@dataclass(frozen=True) +class ConfiguredExtension: + """Operator-visible activation state for one configured extension.""" + + name: str + distribution: str + use: str + enabled: bool + required: bool + + +@dataclass(frozen=True) +class _FileSnapshot: + path: Path + content: bytes | None + + @classmethod + def capture(cls, path: Path) -> _FileSnapshot: + return cls(path, path.read_bytes() if path.exists() else None) + + def restore(self) -> None: + if self.content is None: + self.path.unlink(missing_ok=True) + else: + self.path.write_bytes(self.content) + + +def _read_optional_bytes(path: Path) -> bytes | None: + return path.read_bytes() if path.is_file() else None + + +class ExtensionManager: + """Install trusted Python extensions into one DeerFlow checkout.""" + + def __init__(self, project_root: str | Path, *, config_path: str | Path | None = None) -> None: + self.project_root = Path(project_root).resolve() + self.backend_dir = self.project_root / "backend" + self.pyproject_path = self.backend_dir / "pyproject.toml" + if config_path is not None: + selected_config = Path(config_path).expanduser() + else: + root_config = self.project_root / "config.yaml" + legacy_config = self.backend_dir / "config.yaml" + selected_config = root_config if root_config.is_file() or not legacy_config.is_file() else legacy_config + self.config_path = selected_config.resolve() + + def install(self, source: str, *, yes: bool = False, required: bool = False) -> InstalledExtension: + """Install an extension source and enable its packaging entry point.""" + with _manager_lock(self.project_root): + return self._install(source, yes=yes, required=required) + + def _install(self, source: str, *, yes: bool, required: bool) -> InstalledExtension: + if not yes: + raise PermissionError("installing an extension executes trusted third-party code; pass yes=True to continue") + + source_argument = Path(source).expanduser() + if _is_link_like(source_argument): + raise ValueError("local extension snapshots cannot contain symbolic links or junctions") + source_path = source_argument.resolve() + managed_source: Path | None = None + metadata: tuple[str, str, str] | None = None + uv_source = source + if source_path.is_dir(): + _validate_local_snapshot(source_path) + metadata = _read_local_extension_metadata(source_path) + distribution = metadata[0] + normalized_distribution = _normalize_distribution(distribution) + managed_root = (self.backend_dir / "extensions" / "sources").resolve() + managed_source = (managed_root / normalized_distribution).resolve() + if not managed_source.is_relative_to(managed_root): + raise ValueError(f"invalid extension distribution name: {distribution!r}") + if managed_source.exists(): + raise FileExistsError(f"extension source is already installed: {managed_source}") + uv_source = str(managed_source.relative_to(self.backend_dir)) + else: + if source_argument.exists(): + raise ValueError("local extension sources must be directories so they can be snapshotted for deployment") + _validate_remote_source(source) + # uv add/sync execute the package's build backend. A config this manager + # could never write to must fail before that code runs, not afterwards + # through rollback. + self._read_plugins() + _require_supported_uv(self.backend_dir) + + dependencies_before = _extension_dependency_names(self.pyproject_path) + dependency_snapshots = ( + _FileSnapshot.capture(self.pyproject_path), + _FileSnapshot.capture(self.backend_dir / "uv.lock"), + ) + managed_dependency_contents: tuple[bytes | None, ...] | None = None + uv_attempted = False + try: + if managed_source is not None: + managed_source.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree( + source_path, + managed_source, + ignore=shutil.ignore_patterns(*_SNAPSHOT_IGNORES), + ) + uv_attempted = True + try: + _run_uv( + [ + "uv", + "add", + "--project", + str(self.backend_dir), + "--group", + "extensions", + "--no-workspace", + "--no-sync", + "--", + uv_source, + ], + self.backend_dir, + ) + except BaseException: + # The manager owns dependency-file mutation while uv is + # running. Treat even a failed command's partial writes as + # manager output so the outer transaction can restore them. + managed_dependency_contents = tuple(_read_optional_bytes(snapshot.path) for snapshot in dependency_snapshots) + raise + managed_dependency_contents = tuple(_read_optional_bytes(snapshot.path) for snapshot in dependency_snapshots) + _validate_locked_local_sources(self.backend_dir / "uv.lock", self.backend_dir) + _sync_environment(self.project_root, self.backend_dir, self.config_path) + if metadata is None: + added = _extension_dependency_names(self.pyproject_path) - dependencies_before + if len(added) != 1: + raise RuntimeError("could not identify the distribution added by uv") + distribution = next(iter(added)) + name, use = _discover_installed_entry_point(self.backend_dir, distribution) + metadata = (distribution, name, use) + else: + installed_entry_point = _discover_installed_entry_point(self.backend_dir, metadata[0]) + if installed_entry_point != metadata[1:]: + raise ValueError("installed extension entry point does not match its source metadata") + distribution, name, use = metadata + self._enable_plugin( + { + "name": name, + "package": distribution, + "use": use, + "enabled": True, + "required": required, + "config": {}, + } + ) + except BaseException as operation_error: + # _enable_plugin performs the only config mutation as the final, + # atomic step. A failure before it must not roll back an operator + # edit made while dependency resolution was running. + expected_contents = managed_dependency_contents or tuple(snapshot.content for snapshot in dependency_snapshots) + dependency_recovery_conflict = any( + _read_optional_bytes(snapshot.path) != expected + for snapshot, expected in zip( + dependency_snapshots, + expected_contents, + strict=True, + ) + ) + if dependency_recovery_conflict: + raise RuntimeError("extension installation recovery preserved a concurrent dependency-file edit") from operation_error + for snapshot in dependency_snapshots: + snapshot.restore() + if managed_source is not None: + shutil.rmtree(managed_source, ignore_errors=True) + # The recovery sync itself may rewrite the dependency files, so the + # second restore has to run even when that sync fails. + try: + # An interrupt is not answered by a full dependency resolve: the + # declarations are already restored, and the next locked startup + # sync reconciles the environment. + if uv_attempted and isinstance(operation_error, Exception): + _sync_restored_environment(self.project_root, self.backend_dir, self.config_path) + except RuntimeError as sync_error: + raise RuntimeError(f"{sync_error}; original failure: {operation_error}") from operation_error + finally: + for snapshot in dependency_snapshots: + snapshot.restore() + raise + + return InstalledExtension(name=name, distribution=distribution, use=use) + + def set_enabled(self, identifier: str, *, enabled: bool) -> str: + """Enable or disable one configured extension without losing its config.""" + with _manager_lock(self.project_root): + return self._set_enabled(identifier, enabled=enabled) + + def _set_enabled(self, identifier: str, *, enabled: bool) -> str: + original, plugins = self._read_plugins() + plugin = _find_plugin(plugins, identifier) + plugin["enabled"] = enabled + _write_plugins_block(self.config_path, original, plugins) + return str(plugin.get("name") or plugin.get("use") or identifier) + + def remove(self, identifier: str) -> str: + """Uninstall one managed distribution and remove its activation entry.""" + with _manager_lock(self.project_root): + return self._remove(identifier) + + def _remove(self, identifier: str) -> str: + original, plugins = self._read_plugins() + plugin = _find_plugin(plugins, identifier) + distribution = plugin.get("package") + if not isinstance(distribution, str) or not distribution: + raise ValueError(f"configured extension {identifier!r} has no managed package metadata") + plugins.remove(plugin) + if any(isinstance(candidate, dict) and _same_distribution(candidate.get("package"), distribution) for candidate in plugins): + _write_plugins_block(self.config_path, original, plugins) + return str(plugin.get("name") or plugin.get("use") or identifier) + managed_source = self.backend_dir / "extensions" / "sources" / _normalize_distribution(distribution) + dependency_snapshots = ( + _FileSnapshot.capture(self.pyproject_path), + _FileSnapshot.capture(self.backend_dir / "uv.lock"), + ) + config_snapshot = _FileSnapshot.capture(self.config_path) + staging_root: Path | None = None + staged_source: Path | None = None + managed_dependency_contents: tuple[bytes | None, ...] | None = None + managed_config_content: bytes | None = None + uv_attempted = False + try: + # Deactivate first so an abrupt process exit cannot leave a required + # plugin configured after its distribution declaration is gone. + _write_plugins_block(self.config_path, original, plugins) + managed_config_content = self.config_path.read_bytes() + uv_attempted = True + try: + _run_uv( + [ + "uv", + "remove", + "--project", + str(self.backend_dir), + "--group", + "extensions", + "--no-sync", + "--", + distribution, + ], + self.backend_dir, + ) + except BaseException: + # A failed uv mutation may still have rewritten either + # dependency file. Those writes belong to this locked + # transaction and must be rolled back with the config entry. + managed_dependency_contents = tuple(_read_optional_bytes(snapshot.path) for snapshot in dependency_snapshots) + raise + managed_dependency_contents = tuple(_read_optional_bytes(snapshot.path) for snapshot in dependency_snapshots) + _validate_locked_local_sources(self.backend_dir / "uv.lock", self.backend_dir) + if managed_source.is_dir(): + staging_root = Path( + tempfile.mkdtemp( + prefix=f".{managed_source.name}.remove-", + dir=managed_source.parent, + ) + ) + staged_source = staging_root / "source" + managed_source.rename(staged_source) + _sync_environment(self.project_root, self.backend_dir, self.config_path) + except BaseException as operation_error: + expected_contents = managed_dependency_contents or tuple(snapshot.content for snapshot in dependency_snapshots) + dependency_recovery_conflict = any( + _read_optional_bytes(snapshot.path) != expected + for snapshot, expected in zip( + dependency_snapshots, + expected_contents, + strict=True, + ) + ) + current_config_content = self.config_path.read_bytes() if self.config_path.is_file() else None + config_recovery_conflict = managed_config_content is not None and current_config_content != managed_config_content + if staged_source is not None and staged_source.exists() and not managed_source.exists(): + staged_source.rename(managed_source) + if staging_root is not None: + shutil.rmtree(staging_root, ignore_errors=True) + if dependency_recovery_conflict: + raise RuntimeError("extension removal recovery preserved a concurrent dependency-file edit") from operation_error + for snapshot in dependency_snapshots: + snapshot.restore() + if managed_config_content is not None and not config_recovery_conflict: + config_snapshot.restore() + # The recovery sync itself may rewrite the dependency files, so the + # second restore has to run even when that sync fails. + try: + # An interrupt is not answered by a full dependency resolve: the + # declarations are already restored, and the next locked startup + # sync reconciles the environment. + if uv_attempted and isinstance(operation_error, Exception): + _sync_restored_environment(self.project_root, self.backend_dir, self.config_path) + except RuntimeError as sync_error: + raise RuntimeError(f"{sync_error}; original failure: {operation_error}") from operation_error + finally: + for snapshot in dependency_snapshots: + snapshot.restore() + if config_recovery_conflict: + raise RuntimeError("extension removal recovery preserved a concurrent config edit") from operation_error + raise + if staging_root is not None: + shutil.rmtree(staging_root, ignore_errors=True) + return str(plugin.get("name") or plugin.get("use") or identifier) + + def list_configured(self) -> tuple[ConfiguredExtension, ...]: + """Return configured extensions in their deterministic load order.""" + from deerflow.extensions.loader import ExtensionSpec + + _, plugins = self._read_plugins() + configured: list[ConfiguredExtension] = [] + for plugin in plugins: + spec = ExtensionSpec.model_validate(plugin) + configured.append( + ConfiguredExtension( + name=spec.name or spec.use, + distribution=spec.package or "-", + use=spec.use, + enabled=spec.enabled, + required=spec.required, + ) + ) + return tuple(configured) + + def _enable_plugin(self, plugin: dict[str, Any]) -> None: + original, plugins = self._read_plugins() + exact_use_matches = [item for item in plugins if isinstance(item, dict) and item.get("use") == plugin["use"]] + identity_conflicts = [item for item in plugins if isinstance(item, dict) and item not in exact_use_matches and (item.get("name") == plugin["name"] or _same_distribution(item.get("package"), plugin["package"]))] + if len(exact_use_matches) > 1 or identity_conflicts: + raise ValueError(f"multiple configured plugins conflict with extension {plugin['name']!r}") + if exact_use_matches: + existing = exact_use_matches[0] + existing_package = existing.get("package") + if existing.get("name") not in (None, plugin["name"]) or (existing_package is not None and not _same_distribution(existing_package, plugin["package"])): + raise ValueError(f"configured plugin conflicts with extension {plugin['name']!r}") + existing["name"] = plugin["name"] + existing["package"] = plugin["package"] + existing["use"] = plugin["use"] + existing["enabled"] = True + existing.setdefault("required", plugin["required"]) + existing.setdefault("config", {}) + _write_plugins_block(self.config_path, original, plugins) + return + plugins.append(plugin) + _write_plugins_block(self.config_path, original, plugins) + + def _read_plugins(self) -> tuple[str, list[Any]]: + if not self.config_path.is_file(): + raise FileNotFoundError(f"DeerFlow config not found: {self.config_path}") + with self.config_path.open("r", encoding="utf-8", newline="") as stream: + original = stream.read() + try: + config_node = yaml.compose(original) + config = yaml.safe_load(original) or {} + except yaml.YAMLError as exc: + raise ValueError("invalid DeerFlow config YAML") from exc + if isinstance(config_node, yaml.MappingNode): + plugins_keys = [key for key, _ in config_node.value if isinstance(key, yaml.ScalarNode) and key.value == "plugins"] + if len(plugins_keys) > 1: + raise ValueError("config.yaml contains duplicate top-level plugins keys") + if not isinstance(config, dict): + raise ValueError("DeerFlow config root must be a mapping") + plugins = config.get("plugins") + if plugins is None: + plugins = [] + if not isinstance(plugins, list): + raise ValueError("config.yaml plugins must be a list") + return original, plugins + + +def _normalize_distribution(name: str) -> str: + if not _DISTRIBUTION_NAME.fullmatch(name): + raise ValueError(f"invalid extension distribution name: {name!r}") + return re.sub(r"[-_.]+", "-", name).lower() + + +def _retry_until_locked(acquire: Callable[[], None], *, sleep: Callable[[float], None] = time.sleep) -> None: + """Retry a non-blocking lock acquisition until the region is free. + + Windows' blocking mode (``msvcrt.LK_LOCK``) gives up after roughly ten + seconds. A real install holds this lock across ``uv add`` plus a full + ``uv sync``, so blocking mode reports contention as ``Permission denied`` + instead of serializing the two operations. + """ + while True: + try: + acquire() + return + except OSError: + sleep(_LOCK_RETRY_INTERVAL_SECONDS) + + +@contextmanager +def _manager_lock(project_root: Path) -> Iterator[None]: + """Serialize extension mutations across processes for one checkout.""" + lock_directory = project_root / ".deer-flow" + lock_directory.mkdir(parents=True, exist_ok=True) + lock_path = lock_directory / "extension-manager.lock" + with lock_path.open("a+b") as stream: + if os.name == "nt": + import msvcrt + + if stream.seek(0, os.SEEK_END) == 0: + stream.write(b"\0") + stream.flush() + + def _acquire_region() -> None: + # msvcrt locks a byte range from the current file position. + stream.seek(0) + msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) + + _retry_until_locked(_acquire_region) + try: + yield + finally: + stream.seek(0) + msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(stream.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + + +def _same_distribution(left: object, right: object) -> bool: + if not isinstance(left, str) or not isinstance(right, str): + return False + try: + return _normalize_distribution(left) == _normalize_distribution(right) + except ValueError: + return False + + +def _read_local_extension_metadata(source: Path) -> tuple[str, str, str]: + pyproject = source / "pyproject.toml" + if not pyproject.is_file(): + raise ValueError(f"extension has no pyproject.toml: {source}") + with pyproject.open("rb") as stream: + document = tomllib.load(stream) + project = document.get("project") + if not isinstance(project, dict) or not isinstance(project.get("name"), str): + raise ValueError("extension pyproject.toml must declare project.name") + entry_point_groups = project.get("entry-points", {}) + if not isinstance(entry_point_groups, dict): + raise ValueError(f"extension must declare exactly one {_ENTRY_POINT_GROUP!r} entry point") + entry_points = entry_point_groups.get(_ENTRY_POINT_GROUP, {}) + if not isinstance(entry_points, dict) or len(entry_points) != 1: + raise ValueError(f"extension must declare exactly one {_ENTRY_POINT_GROUP!r} entry point") + name, use = next(iter(entry_points.items())) + if not isinstance(name, str) or not isinstance(use, str): + raise ValueError(f"invalid {_ENTRY_POINT_GROUP!r} entry point metadata") + _validate_entry_point(name, use) + return project["name"], name, use + + +def _validate_entry_point(name: str, use: str) -> None: + if not name or name != name.strip() or any(character in name for character in "\r\n\t"): + raise ValueError(f"invalid {_ENTRY_POINT_GROUP!r} entry point name") + try: + module, attribute = use.rsplit(":", 1) + except ValueError as exc: + raise ValueError(f"invalid {_ENTRY_POINT_GROUP!r} entry point target") from exc + if not attribute.isidentifier() or not module or any(not part.isidentifier() for part in module.split(".")): + raise ValueError(f"invalid {_ENTRY_POINT_GROUP!r} entry point target") + + +def _validate_local_snapshot(source: Path) -> None: + ignored_names = {".git", ".venv", "venv", "__pycache__"} + for directory, dirnames, filenames in os.walk(source, followlinks=False): + directory_path = Path(directory) + retained_dirs: list[str] = [] + for name in dirnames: + if name in ignored_names: + continue + candidate = directory_path / name + if _is_link_like(candidate): + raise ValueError("local extension snapshots cannot contain symbolic links or junctions") + retained_dirs.append(name) + dirnames[:] = retained_dirs + for name in filenames: + candidate = directory_path / name + if name == ".env" or name.startswith(".env.") or name in _SENSITIVE_FILENAMES or Path(name).suffix.lower() in _SENSITIVE_SUFFIXES: + raise ValueError(f"local extension snapshot contains a likely sensitive file: {name}") + if name.endswith(".pyc"): + continue + if _is_link_like(candidate): + raise ValueError("local extension snapshots cannot contain symbolic links or junctions") + if not candidate.is_file(): + raise ValueError("local extension snapshots may contain only directories and regular files") + + +def _is_link_like(path: Path) -> bool: + return path.is_symlink() or path.is_junction() + + +def _validate_remote_source(source: str) -> None: + raw_source = source.strip() + try: + requirement = Requirement(raw_source) + except InvalidRequirement: + requirement = None + if requirement is not None and requirement.url is None: + return + + candidate = _strip_git_prefix(requirement.url if requirement is not None else raw_source) + parsed = urllib.parse.urlsplit(candidate) + scheme = parsed.scheme.lower() + for query in (parsed.query, parsed.fragment): + if any(_is_secret_query_key(key) for key, _ in urllib.parse.parse_qsl(query, keep_blank_values=True)): + raise ValueError("extension source URLs cannot contain credential-like query parameters") + if _is_scp_like_reference(raw_source): + raise ValueError("Git SSH shorthand is not deployable; remote Git sources must use public HTTPS, as in git+https://host/org/repo.git") + if not scheme: + raise ValueError("local path references are not deployable; pass a local directory so DeerFlow can snapshot it") + if scheme == "file": + raise ValueError("file URLs are not deployable; pass a local directory so DeerFlow can snapshot it") + if scheme == "ssh": + raise ValueError("remote Git sources must use public HTTPS; SSH sources are not deployable by the stock Docker builder") + if scheme == "http" and parsed.hostname not in {"localhost", "127.0.0.1", "::1"}: + raise ValueError("remote extension sources must use HTTPS") + if scheme not in {"http", "https"}: + raise ValueError("remote extension sources must use HTTPS") + if parsed.password is not None or (parsed.username is not None and scheme != "ssh"): + raise ValueError("extension source URLs cannot contain embedded credentials") + + +def _strip_git_prefix(reference: str) -> str: + return reference[4:] if reference.lower().startswith("git+") else reference + + +def _is_scp_like_reference(source: str) -> bool: + # The bare shorthand is checked directly; a PEP 508 direct reference keeps + # it behind the requirement name, which packaging strips off the URL. + candidates = [source] + named = _PEP508_NAME_PREFIX.sub("", source, count=1) + if named != source: + candidates.append(named) + return any(_SCP_LIKE_REFERENCE.match(_strip_git_prefix(candidate)) for candidate in candidates) + + +def _is_secret_query_key(key: str) -> bool: + normalized = _normalize_query_key(key) + return bool(_SECRET_QUERY_KEY.search(normalized) or _SECRET_QUERY_SUBSTRING.search(normalized)) + + +def _normalize_query_key(key: str) -> str: + camel_case_split = re.sub( + r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", + "-", + key, + ) + return re.sub(r"[^A-Za-z0-9]+", "-", camel_case_split).strip("-").lower() + + +def _extension_dependency_names(pyproject: Path) -> set[str]: + with pyproject.open("rb") as stream: + document = tomllib.load(stream) + dependencies = document.get("dependency-groups", {}).get("extensions", []) + names: set[str] = set() + for dependency in dependencies: + if not isinstance(dependency, str): + continue + match = re.match(r"\s*([A-Za-z0-9][A-Za-z0-9._-]*)", dependency) + if match: + names.add(_normalize_distribution(match.group(1))) + return names + + +_LOCK_LOCAL_PATH_KEYS = frozenset({"path", "directory", "editable", "virtual"}) +_LOCK_LOCAL_URL_KEYS = frozenset({"registry", "url", "git"}) +_LOCK_LOCAL_SOURCE_VIOLATION = "uv.lock contains a local dependency source outside the backend Docker build context" +_LOCK_LOOPBACK_SOURCE_WARNING = "uv.lock records a loopback dependency source the backend Docker build cannot reach" +_WINDOWS_ABSOLUTE_PATH = re.compile(r"^[A-Za-z]:[\\/]") + + +def _validate_locked_local_sources(lock_path: Path, backend_dir: Path) -> None: + """Reject lock entries that the stock backend Docker build cannot reproduce. + + The image build copies ``backend/`` and runs ``uv sync --locked`` inside + it, so every local reference in the lock must be a relative path that + resolves to the project itself, an exact workspace member, or a managed + snapshot under ``extensions/sources/``. Absolute paths, ``file:`` URLs, + and other local locations (for example a ``UV_FIND_LINKS`` wheelhouse + pulled in by environment configuration) install on this host but fail + the image build, so the caller rolls back the whole transaction when one + appears. + """ + with lock_path.open("rb") as stream: + document = tomllib.load(stream) + backend_root = backend_dir.resolve() + workspace_members = _workspace_member_dirs(backend_root) + managed_snapshots_root = backend_root / "extensions" / "sources" + + def is_reproducible_by_backend_builder(path: Path) -> bool: + if path == backend_root or path in workspace_members: + return True + return path.is_relative_to(managed_snapshots_root) + + def visit(value: object) -> None: + if isinstance(value, dict): + for key, child in value.items(): + if isinstance(child, str): + local_path: Path | None = None + if key in _LOCK_LOCAL_PATH_KEYS: + local_path = _resolve_locked_local_path(child, backend_root, path_only=True) + elif key in _LOCK_LOCAL_URL_KEYS: + local_path = _resolve_locked_local_path(child, backend_root, path_only=False) + if local_path is None and _is_loopback_reference(child): + # An explicit loopback source is an operator choice, + # unlike an environment-driven wheelhouse resolution, + # so it is reported rather than rolled back. + logger.warning("%s: %s", _LOCK_LOOPBACK_SOURCE_WARNING, child) + if local_path is not None and not is_reproducible_by_backend_builder(local_path): + raise ValueError(_LOCK_LOCAL_SOURCE_VIOLATION) + visit(child) + elif isinstance(value, list): + for child in value: + visit(child) + + visit(document) + + +def _is_loopback_reference(value: str) -> bool: + """Report whether a lock URL points back at the installing host. + + Only loopback is rejected. A private-network index (an internal mirror at + ``10.0.0.5``) is reachable from a builder on that network, but ``127.0.0.1`` + resolves to the builder itself, so the recorded source silently means + something different — or nothing — during ``make up``. + """ + candidate = value[4:] if value.lower().startswith("git+") else value + parsed = urllib.parse.urlsplit(candidate) + if not parsed.scheme: + return False + try: + host = parsed.hostname + except ValueError: + return False + if host is None: + return False + if host.lower() == "localhost": + return True + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return False + + +def _workspace_member_dirs(backend_root: Path) -> frozenset[Path]: + """Return the resolved directories of the backend uv workspace members.""" + pyproject_path = backend_root / "pyproject.toml" + if not pyproject_path.is_file(): + return frozenset() + with pyproject_path.open("rb") as stream: + document = tomllib.load(stream) + tool = document.get("tool") + uv_config = tool.get("uv") if isinstance(tool, dict) else None + workspace = uv_config.get("workspace") if isinstance(uv_config, dict) else None + members = workspace.get("members") if isinstance(workspace, dict) else None + if not isinstance(members, list): + return frozenset() + resolved: set[Path] = set() + for member in members: + if not isinstance(member, str) or Path(member).is_absolute(): + continue + for candidate in backend_root.glob(member): + if candidate.is_dir(): + resolved.add(candidate.resolve()) + return frozenset(resolved) + + +def _resolve_locked_local_path(value: str, backend_root: Path, *, path_only: bool) -> Path | None: + """Resolve one lock reference to its in-project path, or None for remote URLs. + + Raises :class:`ValueError` for absolute local references: the stock + backend image build copies ``backend/`` and cannot reproduce host-absolute + paths, even ones that point inside this checkout. + """ + if _WINDOWS_ABSOLUTE_PATH.match(value): + raise ValueError(_LOCK_LOCAL_SOURCE_VIOLATION) + parsed = urllib.parse.urlsplit(value) + scheme = parsed.scheme.lower() + if scheme == "file": + raise ValueError(_LOCK_LOCAL_SOURCE_VIOLATION) + if not path_only and scheme: + return None + path = Path(value).expanduser() + if path.is_absolute(): + raise ValueError(_LOCK_LOCAL_SOURCE_VIOLATION) + return (backend_root / path).resolve() + + +def _first_json_array(stdout: str | None) -> Any: + """Return the first JSON array printed by the probe interpreter. + + The child may emit unrelated startup output first — a `sitecustomize` or + `.pth` banner, a vendored import notice — so the payload is located rather + than assumed to occupy the first line. + """ + for line in (stdout or "").splitlines(): + candidate = line.strip() + if not candidate.startswith("["): + continue + try: + return json.loads(candidate) + except json.JSONDecodeError: + continue + return [] + + +def _discover_installed_entry_point(backend_dir: Path, distribution: str) -> tuple[str, str]: + python = backend_dir / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + script = f"""\ +import json +import sys +from importlib.metadata import distribution + +entry_points = [ + entry_point + for entry_point in distribution(sys.argv[1]).entry_points + if entry_point.group == {_ENTRY_POINT_GROUP!r} +] +print(json.dumps([[entry_point.name, entry_point.value] for entry_point in entry_points]), flush=True) +if len(entry_points) == 1 and not callable(entry_points[0].load()): + raise TypeError("extension entry point is not callable") +""" + completed = subprocess.run( + [str(python), "-c", script, distribution], + cwd=backend_dir, + check=False, + capture_output=True, + text=True, + ) + entry_points = _first_json_array(completed.stdout) + if not isinstance(entry_points, list) or len(entry_points) != 1: + raise ValueError(f"distribution {distribution!r} must expose exactly one {_ENTRY_POINT_GROUP!r} entry point") + name, use = entry_points[0] + if not isinstance(name, str) or not isinstance(use, str): + raise ValueError(f"distribution {distribution!r} has invalid extension entry point metadata") + _validate_entry_point(name, use) + if completed.returncode != 0: + raise ValueError(f"distribution {distribution!r} extension entry point could not be loaded") + return name, use + + +def _write_plugins_block(path: Path, original: str, plugins: list[Any]) -> None: + newline = "\r\n" if "\r\n" in original else "\n" + rendered = yaml.safe_dump( + {"plugins": plugins}, + allow_unicode=True, + sort_keys=False, + ).rstrip() + rendered = rendered.replace("\n", newline) + newline + span = _plugins_block_span(original) + if span is None: + separator = "" if not original or original.endswith(newline * 2) else newline + updated = original + separator + rendered + else: + start, end = span + updated = original[:start] + rendered + original[end:] + + mode = path.stat().st_mode + temporary: Path | None = None + try: + with tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + newline="", + dir=path.parent, + prefix=f".{path.name}.", + delete=False, + ) as stream: + stream.write(updated) + temporary = Path(stream.name) + os.chmod(temporary, mode) + os.replace(temporary, path) + temporary = None + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def _plugins_block_span(original: str) -> tuple[int, int] | None: + """Locate the character span of an existing top-level ``plugins:`` entry. + + Both boundaries come from the YAML parser rather than a key-shaped regex. + ``AppConfig`` allows extra top-level keys, so a following section may be + named anything YAML accepts — ``my.key``, ``2fa``, ``$schema``, a non-ASCII + word. A pattern that fails to recognize that key does not fail loudly: it + reports "no next section", and the rewrite then replaces the neighbour and + its whole subtree with the managed block. Trailing comments below a + file-final block are preserved for the same reason. + """ + root = yaml.compose(original) + if not isinstance(root, yaml.MappingNode): + return None + for index, (key, _value) in enumerate(root.value): + if not isinstance(key, yaml.ScalarNode) or key.value != "plugins": + continue + start = key.start_mark.index + line_end = original.find("\n", start) + content_start = len(original) if line_end < 0 else line_end + 1 + following = root.value[index + 1 :] + next_start = following[0][0].start_mark.index if following else len(original) + between = original[content_start:next_start] + return start, content_start + _trailing_section_comment_start(between) + return None + + +def _trailing_section_comment_start(text: str) -> int: + lines = text.splitlines(keepends=True) + index = len(lines) - 1 + saw_comment = False + while index >= 0: + content = lines[index].rstrip("\r\n") + if not content.strip(): + index -= 1 + continue + if content.startswith("#"): + saw_comment = True + index -= 1 + continue + break + if not saw_comment: + return len(text) + return sum(len(line) for line in lines[: index + 1]) + + +def _find_plugin(plugins: list[Any], identifier: str) -> dict[str, Any]: + matches = [plugin for plugin in plugins if isinstance(plugin, dict) and (plugin.get("name") == identifier or plugin.get("use") == identifier or _same_distribution(plugin.get("package"), identifier))] + if len(matches) != 1: + raise ValueError(f"expected exactly one configured extension matching {identifier!r}") + return matches[0] + + +def _controlled_uv_environment() -> dict[str, str]: + environment = os.environ.copy() + for name in _UV_ENV_OVERRIDES: + environment.pop(name, None) + return environment + + +def _run_uv(command: list[str], backend_dir: Path) -> None: + subprocess.run( + command, + cwd=backend_dir, + env=_controlled_uv_environment(), + check=True, + ) + + +def _require_supported_uv(backend_dir: Path) -> None: + completed = subprocess.run( + ["uv", "--version"], + cwd=backend_dir, + env=_controlled_uv_environment(), + check=True, + capture_output=True, + text=True, + ) + match = re.search(r"\b(\d+)\.(\d+)\.(\d+)\b", completed.stdout) + if match is None or tuple(int(part) for part in match.groups()) < (0, 8, 0): + raise RuntimeError("extension installation requires uv 0.8.0 or newer") + + +def _detect_extra_flags(project_root: Path, config_path: Path) -> list[str]: + detector = project_root / "scripts" / "detect_uv_extras.py" + if not detector.is_file(): + return [] + environment = _controlled_uv_environment() + environment["DEER_FLOW_CONFIG_PATH"] = str(config_path) + completed = subprocess.run( + [sys.executable, str(detector)], + cwd=project_root, + env=environment, + check=True, + stdout=subprocess.PIPE, + text=True, + ) + tokens = shlex.split(completed.stdout or "") + if len(tokens) % 2 or any(tokens[index] != "--extra" or not re.fullmatch(r"[A-Za-z][A-Za-z0-9_-]*", tokens[index + 1]) for index in range(0, len(tokens), 2)): + raise RuntimeError("extension dependency sync received invalid optional-dependency flags") + return tokens + + +def _sync_environment( + project_root: Path, + backend_dir: Path, + config_path: Path, + *, + locked: bool = True, +) -> None: + command = [ + "uv", + "sync", + "--project", + str(backend_dir), + "--all-packages", + ] + if locked: + command.append("--locked") + command.extend(_detect_extra_flags(project_root, config_path)) + _run_uv(command, backend_dir) + + +def _sync_restored_environment(project_root: Path, backend_dir: Path, config_path: Path) -> None: + try: + _sync_environment( + project_root, + backend_dir, + config_path, + locked=(backend_dir / "uv.lock").is_file(), + ) + except (OSError, RuntimeError, subprocess.CalledProcessError) as exc: + raise RuntimeError("extension operation failed and the restored environment could not be synchronized") from exc diff --git a/backend/packages/harness/deerflow/extensions/registry.py b/backend/packages/harness/deerflow/extensions/registry.py index 7dcbfb4f0..ae5882152 100644 --- a/backend/packages/harness/deerflow/extensions/registry.py +++ b/backend/packages/harness/deerflow/extensions/registry.py @@ -7,13 +7,14 @@ runtime projection. from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from contextlib import contextmanager from dataclasses import dataclass from typing import Any from deerflow_extension_api import ( ExtensionData, + ExtensionService, MiddlewareContributor, SystemModelCallObserver, TaskLifecycleContributor, @@ -35,6 +36,8 @@ class LoadedExtensions: middleware_contributors: tuple[tuple[str, MiddlewareContributor], ...] = () task_lifecycle: tuple[tuple[str, TaskLifecycleContributor], ...] = () system_model_observers: tuple[tuple[str, SystemModelCallObserver], ...] = () + services: tuple[tuple[str, ExtensionService], ...] = () + routers: tuple[tuple[str, Any], ...] = () # Precomputed attributes, not methods: hook sites read one attribute to # short-circuit, so the zero-extension path constructs nothing. @@ -57,6 +60,8 @@ class ExtensionRegistry(ExtensionRegistryContract): self._middlewares: list[_Entry] = [] self._task_lifecycle: list[_Entry] = [] self._system_model_observers: list[_Entry] = [] + self._services: list[_Entry] = [] + self._routers: list[_Entry] = [] self._current_source: str | None = None @contextmanager @@ -83,6 +88,13 @@ class ExtensionRegistry(ExtensionRegistryContract): def system_model_observer(self, observer: SystemModelCallObserver) -> None: self._system_model_observers.append((self._source(), observer)) + def service(self, service: ExtensionService) -> None: + self._services.append((self._source(), service)) + + def routers(self, routers: Sequence[Any]) -> None: + source = self._source() + self._routers.extend((source, router) for router in routers) + def discard(self, source: str) -> None: """Remove every entry registered by ``source``. @@ -100,18 +112,22 @@ class ExtensionRegistry(ExtensionRegistryContract): self._middlewares, self._task_lifecycle, self._system_model_observers, + self._services, + self._routers, ): bucket[:] = [entry for entry in bucket if entry[0] != source] - def mark(self) -> tuple[int, int, int]: + def mark(self) -> tuple[int, int, int, int, int]: """Snapshot bucket lengths so one install() can be undone positionally.""" return ( len(self._middlewares), len(self._task_lifecycle), len(self._system_model_observers), + len(self._services), + len(self._routers), ) - def rollback_to(self, mark: tuple[int, int, int]) -> None: + def rollback_to(self, mark: tuple[int, int, int, int, int]) -> None: """Undo every registration made since ``mark``. Positional rather than source-keyed: two specs may legitimately share @@ -123,6 +139,8 @@ class ExtensionRegistry(ExtensionRegistryContract): self._middlewares, self._task_lifecycle, self._system_model_observers, + self._services, + self._routers, ), mark, strict=True, @@ -135,6 +153,8 @@ class ExtensionRegistry(ExtensionRegistryContract): middleware_contributors=tuple(self._middlewares), task_lifecycle=tuple(self._task_lifecycle), system_model_observers=tuple(self._system_model_observers), + services=tuple(self._services), + routers=tuple(self._routers), has_middleware_contributors=bool(self._middlewares), has_task_lifecycle=bool(self._task_lifecycle), has_system_model_observers=bool(self._system_model_observers), diff --git a/backend/packages/harness/deerflow/tui/cli.py b/backend/packages/harness/deerflow/tui/cli.py index 31e7e826b..28985ec4b 100644 --- a/backend/packages/harness/deerflow/tui/cli.py +++ b/backend/packages/harness/deerflow/tui/cli.py @@ -49,6 +49,7 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="deerflow", description="DeerFlow terminal workbench — a TUI over the embedded DeerFlow harness.", + epilog="Extension management: deerflow extensions --help", add_help=True, ) parser.add_argument("message", nargs="*", help="initial prompt for the TUI, or message in --cli mode") @@ -204,6 +205,7 @@ deerflow — DeerFlow terminal workbench deerflow --json "question" stream newline-delimited JSON events deerflow --recursion-limit N --print "question" set the headless agent-loop super-step limit + deerflow extensions --help install and manage trusted Python extensions echo "question" | deerflow --print """ @@ -222,6 +224,10 @@ def _run_overrides(plan: LaunchPlan) -> dict[str, int]: def main(argv: Sequence[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) + if argv and argv[0] == "extensions": + from deerflow.extensions.cli import main as extensions_main + + return extensions_main(argv[1:]) plan = plan_launch( argv, stdin_isatty=sys.stdin.isatty(), diff --git a/backend/packages/harness/pyproject.toml b/backend/packages/harness/pyproject.toml index 8c8b5ef93..479fb3ba4 100644 --- a/backend/packages/harness/pyproject.toml +++ b/backend/packages/harness/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ # the contract version it implements, extensions declare ranges. A range # here would let pip resolve a newer contract package than this harness # implements, making newer extensions look supported at runtime. - "deerflow-extension-api==0.1.1", + "deerflow-extension-api==0.1.2", "dotenv>=0.9.9", "exa-py>=1.0.0", "httpx>=0.28.0", @@ -31,6 +31,7 @@ dependencies = [ "langgraph-runtime-inmem>=0.28.0", "markdownify>=1.2.2", "markitdown[all,xlsx]>=0.0.1a2", + "packaging>=24.2", "pydantic>=2.12.5", "pyyaml>=6.0.3", "readabilipy>=0.3.0", diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 991ac85af..aa91c2d81 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -10,6 +10,16 @@ dependencies = [ "httpx>=0.28.0", "python-multipart>=0.0.31", "sse-starlette>=2.1.0", + # Direct dependency on purpose, even though FastAPI already pulls it in: + # app/gateway/request_path.py imports the private + # `starlette._utils.get_route_path` so the auth and CSRF predicates + # classify the exact string Starlette's router matches on. A private + # import is the safest option here because it fails loudly (ImportError at + # startup) instead of silently drifting from the dispatcher at a security + # boundary -- but it does mean a Starlette bump is a security-relevant + # change. Declaring and bounding it here makes that bump visible in the + # diff; tests/test_gateway_request_path.py pins the agreement itself. + "starlette>=1.3.1,<2", "uvicorn[standard]>=0.34.0", "lark-oapi>=1.4.0", "slack-sdk>=3.33.0", @@ -34,6 +44,10 @@ browser = ["deerflow-harness[browser]"] memory-zh = ["deerflow-harness[memory-zh]"] [dependency-groups] +# Managed extension packages are added here by `deerflow extensions install`. +# Keeping them separate from development tooling lets every startup mode sync +# the same locked runtime set without promoting those packages to core deps. +extensions = [] dev = [ "blockbuster>=1.5.26,<1.6", "hypothesis>=6.100,<7", @@ -64,6 +78,7 @@ markers = [ [tool.uv] index-url = "https://pypi.org/simple" +default-groups = ["dev", "extensions"] # langgraph-sdk 0.4.2 (pulled in by langgraph 1.2.9 for DeltaChannel) pins # `websockets<16,>=14`, silently downgrading websockets 16.0 -> 15.0.1. The # pin is not grounded in any API incompatibility: websockets 16's only diff --git a/backend/tests/test_auth_middleware.py b/backend/tests/test_auth_middleware.py index b529415ac..38251dc3c 100644 --- a/backend/tests/test_auth_middleware.py +++ b/backend/tests/test_auth_middleware.py @@ -216,6 +216,58 @@ def test_public_auth_path_no_cookie(client): assert res.status_code == 200 +@pytest.mark.parametrize( + "encoded_path", + [ + "/api/v1/auth/setup-sta%0Atus", + "/api/v1/auth/setup-sta%0Dtus", + "/api/v1/auth/setup-sta%09tus", + "/api/v1/auth/setup-status%23private", + "/api/v1/auth/setup-status%3Fprivate", + ], +) +def test_url_reconstruction_cannot_turn_a_protected_route_path_public( + monkeypatch, + encoded_path, +): + from fastapi import FastAPI + + monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "") + app = FastAPI() + app.add_middleware(AuthMiddleware) + + @app.get("/api/v1/auth/setup-sta{gap}tus") + async def control_gap(gap: str): + return {"gap": gap} + + @app.get("/api/v1/auth/setup-status{suffix}") + async def delimiter_suffix(suffix: str): + return {"suffix": suffix} + + response = TestClient(app).get(encoded_path) + + assert response.status_code == 401 + + +def test_auth_uses_the_same_root_path_projection_as_the_router(monkeypatch): + from fastapi import FastAPI + + monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "") + child = FastAPI() + child.add_middleware(AuthMiddleware) + + @child.get("/health") + async def health(): + return {"ok": True} + + parent = FastAPI() + parent.mount("/prefix", child) + + response = TestClient(parent).get("/prefix/health") + + assert response.status_code == 200 + + def test_protected_auth_path_no_cookie(client): """/auth/me requires cookie even though it's under /api/v1/auth/.""" res = client.get("/api/v1/auth/me") diff --git a/backend/tests/test_auth_type_system.py b/backend/tests/test_auth_type_system.py index d82804350..de9b12b93 100644 --- a/backend/tests/test_auth_type_system.py +++ b/backend/tests/test_auth_type_system.py @@ -70,6 +70,7 @@ class _FakeRequest: def __init__(self, path: str, method: str = "POST"): self.method = method + self.scope = {"path": path, "root_path": ""} class _URL: def __init__(self, p): diff --git a/backend/tests/test_ci_uv_version_pin.py b/backend/tests/test_ci_uv_version_pin.py new file mode 100644 index 000000000..bcdb55a27 --- /dev/null +++ b/backend/tests/test_ci_uv_version_pin.py @@ -0,0 +1,111 @@ +"""Regression test pinning CI's uv binary to the version production ships. + +``ExtensionManager`` is not a consumer of uv the build tool -- it is a program +whose whole job is driving ``uv`` as a subprocess, depending on its CLI +behavior (``--no-workspace``, ``--no-sync``, what ``uv add`` writes into +``[dependency-groups] extensions``) and on the ``uv.lock`` serialization +format. uv is therefore closer to a runtime dependency with a contract than to +incidental tooling. + +``backend/Dockerfile`` pins that binary to an exact version, but every +``astral-sh/setup-uv`` step used to install whatever was latest at run time, so +CI exercised the manager against a uv that is not the uv production runs. The +sharpest failure that allows: a newer uv bumps ``uv.lock``'s ``revision``, +CI stays green because the same uv reads back what it wrote, and the pinned uv +in the production image then cannot read the committed lock. ``uv lock +--check`` is version-sensitive for the same reason -- it verifies the lock is +what *this* uv would produce, and two versions can emit equivalent but +non-identical output. + +Pinning is only half of it; without this test the two pins drift apart again +the next time someone bumps one of them. Upgrading uv should be one explicit +change that touches the Dockerfile, the workflows, and this test together. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +BACKEND_DOCKERFILE = REPO_ROOT / "backend" / "Dockerfile" +WORKFLOWS_DIR = REPO_ROOT / ".github" / "workflows" +COMPOSE_PATHS = ( + REPO_ROOT / "docker" / "docker-compose.yaml", + REPO_ROOT / "docker" / "docker-compose-dev.yaml", +) + +_UV_IMAGE_REFERENCE = re.compile(r"ghcr\.io/astral-sh/uv:(?P\d+\.\d+\.\d+)") +_SETUP_UV_ACTION = re.compile(r"^astral-sh/setup-uv@(?P[^\s]+)$") + + +def _pinned_uv_version() -> str: + """The single source of truth: the uv image the backend image builds from.""" + match = _UV_IMAGE_REFERENCE.search(BACKEND_DOCKERFILE.read_text(encoding="utf-8")) + assert match is not None, f"{BACKEND_DOCKERFILE} no longer pins a ghcr.io/astral-sh/uv version" + return match.group("version") + + +def _workflow_paths() -> list[Path]: + return sorted(path for path in WORKFLOWS_DIR.iterdir() if path.suffix in {".yml", ".yaml"}) + + +def _setup_uv_steps() -> list[tuple[str, str, dict]]: + """Return (workflow name, action ref, step mapping) for each setup-uv step.""" + steps: list[tuple[str, str, dict]] = [] + for path in _workflow_paths(): + workflow = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + for job in (workflow.get("jobs") or {}).values(): + if not isinstance(job, dict): + continue + for step in job.get("steps") or []: + if not isinstance(step, dict): + continue + uses = step.get("uses") + if not isinstance(uses, str): + continue + match = _SETUP_UV_ACTION.match(uses.strip()) + if match is not None: + steps.append((path.name, match.group("ref"), step)) + return steps + + +def test_the_repository_still_has_setup_uv_steps_to_check(): + """Guard against the other assertions silently passing on an empty list.""" + assert _setup_uv_steps(), "no astral-sh/setup-uv steps found; this test needs updating" + + +def test_every_setup_uv_step_pins_the_uv_version_production_ships(): + expected = _pinned_uv_version() + + unpinned: list[str] = [] + mismatched: list[str] = [] + for workflow_name, _ref, step in _setup_uv_steps(): + with_block = step.get("with") + version = with_block.get("version") if isinstance(with_block, dict) else None + if version is None: + unpinned.append(f"{workflow_name}: {step.get('name', '')}") + elif str(version) != expected: + mismatched.append(f"{workflow_name}: {version!r} != {expected!r}") + + assert not unpinned, f"setup-uv steps install whatever uv is latest, so CI would not exercise the uv production ships ({expected}): {unpinned}" + assert not mismatched, f"setup-uv steps pin a uv other than the one backend/Dockerfile ships ({expected}): {mismatched}" + + +def test_every_setup_uv_step_uses_the_same_action_version(): + refs = {ref for _workflow_name, ref, _step in _setup_uv_steps()} + + assert len(refs) == 1, f"astral-sh/setup-uv is referenced at mixed action versions, so the steps do not share caching or input behavior: {sorted(refs)}" + + +@pytest.mark.parametrize("compose_path", COMPOSE_PATHS, ids=lambda path: path.name) +def test_compose_uv_image_default_matches_the_backend_dockerfile(compose_path: Path): + """The compose override defaults must not drift from the image they build.""" + expected = _pinned_uv_version() + versions = set(_UV_IMAGE_REFERENCE.findall(compose_path.read_text(encoding="utf-8"))) + + assert versions, f"{compose_path.name} no longer references a pinned uv image" + assert versions == {expected}, f"{compose_path.name} defaults to uv {sorted(versions)} while backend/Dockerfile ships {expected!r}" diff --git a/backend/tests/test_csrf_middleware.py b/backend/tests/test_csrf_middleware.py index 94dd8db38..f9d4d48a0 100644 --- a/backend/tests/test_csrf_middleware.py +++ b/backend/tests/test_csrf_middleware.py @@ -22,9 +22,50 @@ def _make_app() -> FastAPI: async def protected_mutation(): return {"ok": True} + @app.post("/api/v1/auth/log{gap}in/local") + async def control_gap(gap: str): + return {"gap": gap} + + @app.post("/api/v1/auth/me{suffix}") + async def delimiter_suffix(suffix: str): + return {"suffix": suffix} + return app +def test_url_reconstruction_cannot_create_a_csrf_exemption(): + client = TestClient(_make_app(), base_url="https://deerflow.example") + + for encoded_path in ( + "/api/v1/auth/log%0Ain/local", + "/api/v1/auth/log%0Din/local", + "/api/v1/auth/log%09in/local", + "/api/v1/auth/me%23private", + "/api/v1/auth/me%3Fprivate", + ): + response = client.post(encoded_path) + assert response.status_code == 403, encoded_path + + +def test_csrf_uses_the_same_root_path_projection_as_the_router(): + child = FastAPI() + child.add_middleware(CSRFMiddleware) + + @child.post("/api/v1/auth/login/local") + async def login_local(): + return {"ok": True} + + parent = FastAPI() + parent.mount("/prefix", child) + + response = TestClient( + parent, + base_url="https://deerflow.example", + ).post("/prefix/api/v1/auth/login/local") + + assert response.status_code == 200 + + def test_auth_post_rejects_cross_origin_browser_request(): """CSRF-exempt auth routes must not accept hostile browser origins. diff --git a/backend/tests/test_deploy_uv_extras.py b/backend/tests/test_deploy_uv_extras.py index e80c56209..7ac6c36ce 100644 --- a/backend/tests/test_deploy_uv_extras.py +++ b/backend/tests/test_deploy_uv_extras.py @@ -49,6 +49,7 @@ def test_backend_dockerfile_expands_multiple_uv_extras(tmp_path): assert capture.read_text(encoding="utf-8").splitlines() == [ "sync", + "--locked", "--extra", "redis", "--extra", diff --git a/backend/tests/test_dev_entrypoint.py b/backend/tests/test_dev_entrypoint.py index 12bbe1898..e14b1881c 100644 --- a/backend/tests/test_dev_entrypoint.py +++ b/backend/tests/test_dev_entrypoint.py @@ -18,14 +18,26 @@ REPO_ROOT = Path(__file__).resolve().parents[2] ENTRYPOINT = REPO_ROOT / "docker" / "dev-entrypoint.sh" -def _run(uv_extras: str | None) -> subprocess.CompletedProcess[str]: - """Invoke `dev-entrypoint.sh --print-extras` with UV_EXTRAS set.""" +def _run( + uv_extras: str | None, + *, + config_path: Path | None = None, + stream_bridge_redis_url: str | None = None, +) -> subprocess.CompletedProcess[str]: + """Invoke the entrypoint's public extras-resolution dry run.""" env = os.environ.copy() env.pop("UV_EXTRAS", None) + env.pop("DEER_FLOW_CONFIG_PATH", None) + env.pop("DEER_FLOW_STREAM_BRIDGE_REDIS_URL", None) if uv_extras is not None: env["UV_EXTRAS"] = uv_extras + if config_path is not None: + env["DEER_FLOW_CONFIG_PATH"] = str(config_path) + if stream_bridge_redis_url is not None: + env["DEER_FLOW_STREAM_BRIDGE_REDIS_URL"] = stream_bridge_redis_url return subprocess.run( ["sh", str(ENTRYPOINT), "--print-extras"], + cwd=ENTRYPOINT.parent, env=env, capture_output=True, text=True, @@ -54,12 +66,32 @@ def test_entrypoint_excludes_runtime_state_from_uvicorn_reload(): assert "--reload-exclude=/app/backend/.deer-flow" in content +def test_failed_sync_recreates_a_clean_virtual_environment(): + content = ENTRYPOINT.read_text(encoding="utf-8") + + assert "uv venv --clear .venv" in content + assert "uv venv --allow-existing .venv" not in content + + def test_no_uv_extras_yields_empty_flags(): proc = _run(None) assert proc.returncode == 0 assert proc.stdout.strip() == "" +def test_no_explicit_extras_uses_the_runtime_selected_config(tmp_path: Path): + config_path = tmp_path / "deployment.yaml" + config_path.write_text( + "database:\n backend: postgres\ntools:\n - name: browser_navigate\n", + encoding="utf-8", + ) + + proc = _run(None, config_path=config_path) + + assert proc.returncode == 0, proc.stderr + assert proc.stdout.strip() == "--extra browser --extra postgres" + + def test_single_extra(): proc = _run("postgres") assert proc.returncode == 0 @@ -84,6 +116,26 @@ def test_multi_extra_mixed_separators(): assert proc.stdout.strip() == "--extra postgres --extra ollama" +def test_explicit_extras_override_config_and_are_deduplicated(tmp_path: Path): + config_path = tmp_path / "deployment.yaml" + config_path.write_text("database:\n backend: postgres\n", encoding="utf-8") + + proc = _run("redis,redis browser redis", config_path=config_path) + + assert proc.returncode == 0, proc.stderr + assert proc.stdout.strip() == "--extra redis --extra browser" + + +def test_explicit_extras_keep_runtime_required_redis_without_duplicates(): + proc = _run( + "postgres,postgres", + stream_bridge_redis_url="redis://redis:6379/0", + ) + + assert proc.returncode == 0, proc.stderr + assert proc.stdout.strip() == "--extra postgres --extra redis" + + def test_empty_string_yields_empty_flags(): proc = _run("") assert proc.returncode == 0 @@ -114,3 +166,97 @@ def test_underscores_and_hyphens_in_name_are_allowed(): proc = _run("post_gres,post-gres") assert proc.returncode == 0 assert proc.stdout.strip() == "--extra post_gres --extra post-gres" + + +# ── Dependency-sync failure branch ────────────────────────────────────────── +# +# The self-heal retry reuses `--locked`, so a lock that genuinely disagrees with +# the environment fails the same way twice. `set -e` already stops the script +# there -- these tests pin that the handoff to uvicorn is never reached, and +# that the operator is told what to do instead of reading a bare uv traceback. +# +# `/app/backend` only exists inside the container, so the sync block is sliced +# out of the real script and run against a stub `uv`. The block is read from +# the file rather than duplicated here: editing the script changes what runs. + +_SYNC_BLOCK_START = "# ── Sync dependencies (with self-heal) ──" +_SYNC_BLOCK_END = "# ── Hand off to uvicorn ──" + +_STUB_UV_ALWAYS_FAILS_SYNC = """#!/bin/sh +# `uv venv` succeeds so the retry is actually reached; every sync fails. +case "$1" in + sync) exit 1 ;; + *) exit 0 ;; +esac +""" + +_STUB_UV_SUCCEEDS = """#!/bin/sh +exit 0 +""" + +_STUB_UV_FAILS_THEN_SUCCEEDS = """#!/bin/sh +case "$1" in + sync) + if [ -f "$STUB_UV_STATE/first_sync_done" ]; then exit 0; fi + : > "$STUB_UV_STATE/first_sync_done" + exit 1 + ;; + *) exit 0 ;; +esac +""" + + +def _sync_block() -> str: + content = ENTRYPOINT.read_text(encoding="utf-8") + start = content.index(_SYNC_BLOCK_START) + end = content.index(_SYNC_BLOCK_END) + return content[start:end] + + +def _run_sync_block(tmp_path: Path, stub_uv: str) -> subprocess.CompletedProcess[str]: + """Execute the script's real sync block with a stubbed `uv` on PATH.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + uv_stub = bin_dir / "uv" + uv_stub.write_text(stub_uv, encoding="utf-8") + uv_stub.chmod(0o755) + + state_dir = tmp_path / "state" + state_dir.mkdir() + + # `cd` is shadowed because /app/backend does not exist outside the + # container; everything else in the block runs verbatim. + script = f'set -e\ncd() {{ :; }}\nEXTRAS_FLAGS=""\n{_sync_block()}\necho "REACHED_HANDOFF"\n' + + env = os.environ.copy() + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + env["STUB_UV_STATE"] = str(state_dir) + return subprocess.run(["sh", "-c", script], capture_output=True, text=True, check=False, env=env, cwd=tmp_path) + + +def test_successful_sync_reaches_the_uvicorn_handoff(tmp_path: Path): + proc = _run_sync_block(tmp_path, _STUB_UV_SUCCEEDS) + + assert proc.returncode == 0, proc.stderr + assert "REACHED_HANDOFF" in proc.stdout + + +def test_self_heal_retry_still_reaches_the_handoff(tmp_path: Path): + proc = _run_sync_block(tmp_path, _STUB_UV_FAILS_THEN_SUCCEEDS) + + assert proc.returncode == 0, proc.stderr + assert "recreating .venv" in proc.stdout + assert "REACHED_HANDOFF" in proc.stdout + + +def test_failed_retry_aborts_before_starting_uvicorn(tmp_path: Path): + proc = _run_sync_block(tmp_path, _STUB_UV_ALWAYS_FAILS_SYNC) + + assert proc.returncode != 0, "startup continued past an unsatisfied lock" + assert "REACHED_HANDOFF" not in proc.stdout, "uvicorn would have been started against a stale or missing environment" + + +def test_failed_retry_tells_the_operator_how_to_recover(tmp_path: Path): + proc = _run_sync_block(tmp_path, _STUB_UV_ALWAYS_FAILS_SYNC) + + assert "make install" in proc.stderr, f"no recovery guidance on stderr: {proc.stderr!r}" diff --git a/backend/tests/test_extension_api_contracts.py b/backend/tests/test_extension_api_contracts.py index 37a4c2bcb..e9ece1246 100644 --- a/backend/tests/test_extension_api_contracts.py +++ b/backend/tests/test_extension_api_contracts.py @@ -21,6 +21,8 @@ from deerflow_extension_api import ( ExtensionData, ExtensionInstall, ExtensionRegistry, + ExtensionRuntimeDeps, + ExtensionService, HostPolicySnapshot, MiddlewareContributor, MiddlewarePlacement, @@ -63,6 +65,7 @@ def test_middleware_placement_defaults(): "cls", [ HostPolicySnapshot, + ExtensionRuntimeDeps, AgentBuildContext, TaskInfo, SystemModelRequest, @@ -77,7 +80,7 @@ def test_every_dataclass_is_frozen(cls): @pytest.mark.parametrize( "cls", - [HostPolicySnapshot, TaskInfo, SystemModelRequest, SystemModelResult], + [HostPolicySnapshot, ExtensionRuntimeDeps, TaskInfo, SystemModelRequest, SystemModelResult], ) def test_additive_dataclasses_are_constructible_with_required_fields_only(cls): """Fields added later must carry defaults, or old extensions break on upgrade. @@ -114,6 +117,7 @@ def test_agent_build_context_optional_fields_keep_their_defaults(): "protocol", [ ExtensionRegistry, + ExtensionService, MiddlewareContributor, TaskLifecycleContributor, SystemModelCallObserver, @@ -195,16 +199,18 @@ def test_system_model_request_normalizes_messages_into_an_immutable_sequence(): assert SystemModelRequest(messages=("already", "a", "tuple")).messages == ("already", "a", "tuple") -def test_future_contribution_points_are_not_advertised_before_the_host_supports_them(): - """A merged slice must not silently accept registrations it cannot run.""" +def test_gateway_contribution_points_are_part_of_the_public_surface(): import deerflow_extension_api for name in ( "ExtensionRuntimeDeps", "ExtensionService", ): - assert name not in deerflow_extension_api.__all__ - assert not hasattr(deerflow_extension_api, name) + assert name in deerflow_extension_api.__all__ + assert hasattr(deerflow_extension_api, name) + assert callable(ExtensionRegistry.service) + assert callable(ExtensionRegistry.routers) + assert not hasattr(deerflow_extension_api, "RouterContributor") def test_task_store_from_runtime_reads_the_host_key(): @@ -263,6 +269,15 @@ def test_distribution_marks_the_contract_package_as_typed(): assert marker.is_file() +def test_contract_package_keeps_runtime_dependencies_empty(): + import tomllib + from pathlib import Path + + pyproject = Path(__file__).parent.parent / "packages" / "extension-api" / "pyproject.toml" + + assert tomllib.loads(pyproject.read_text())["project"]["dependencies"] == [] + + def test_harness_pins_the_contract_package_exactly(): """The version contract (extension-system design): the host pins the contract package exactly, extensions use ranges. A range here would let an @@ -289,5 +304,17 @@ def test_runtime_api_version_matches_the_installed_contract_package(): """Every additive contract slice bumps both gates together.""" from importlib.metadata import version - assert API_VERSION == "0.1.1" + assert API_VERSION == "0.1.2" assert API_VERSION == version("deerflow-extension-api") + + +def test_extension_service_contract_is_public_and_defaults_to_noop(): + class _Bare: + pass + + deps = ExtensionRuntimeDeps() + + assert deps.app_store is None + assert deps.session_factory is None + assert asyncio.run(ExtensionService.start(_Bare(), deps)) is None + assert asyncio.run(ExtensionService.stop(_Bare())) is None diff --git a/backend/tests/test_extension_app_loading.py b/backend/tests/test_extension_app_loading.py index a2b041c7b..a75dcb747 100644 --- a/backend/tests/test_extension_app_loading.py +++ b/backend/tests/test_extension_app_loading.py @@ -85,6 +85,53 @@ def test_create_app_exposes_one_canonical_live_diagnostics_list(monkeypatch): ] +def test_create_app_mounts_extension_routers_after_all_host_routes(monkeypatch): + from fastapi import APIRouter + from fastapi.testclient import TestClient + + import deerflow.extensions as extensions_module + + conflict = APIRouter() + good = APIRouter() + + @conflict.get("/health") + async def shadow_health(): + return {"status": "extension"} + + @good.get("/api/extension-test/ping") + async def ping(): + return {"ok": True} + + registry = ExtensionRegistry() + with registry.attributed_to("router:install"): + registry.routers((conflict, good)) + loaded = registry.build() + monkeypatch.setattr( + extensions_module, + "load_extensions", + lambda plugins: (loaded, []), + ) + + from app.gateway.app import create_app + + app = create_app() + paths = [getattr(route, "path", None) for route in app.routes] + + assert paths.count("/health") == 1 + assert "/api/extension-test/ping" in paths + assert len(app.state.extension_diagnostics) == 1 + assert app.state.extension_diagnostics == extensions_module.get_runtime_diagnostics() + assert app.state.extension_diagnostics[0].source == "router:install" + assert "host" in app.state.extension_diagnostics[0].message + + client = TestClient(app) + assert client.get("/health").json() == { + "status": "healthy", + "service": "deer-flow-gateway", + } + assert client.get("/api/extension-test/ping").status_code == 401 + + def test_create_app_fails_open_when_extension_loading_raises_unexpectedly(monkeypatch): import deerflow.extensions as extensions_module diff --git a/backend/tests/test_extension_dependency_sync.py b/backend/tests/test_extension_dependency_sync.py new file mode 100644 index 000000000..300eb8bc6 --- /dev/null +++ b/backend/tests/test_extension_dependency_sync.py @@ -0,0 +1,298 @@ +"""Contracts for installing declared Python extensions in every startup mode.""" + +from __future__ import annotations + +import os +import re +import subprocess +import tomllib +import zipfile +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +BACKEND_ROOT = REPO_ROOT / "backend" + + +def _make_recipe(path: Path, target: str) -> str: + content = path.read_text(encoding="utf-8") + match = re.search(rf"^{re.escape(target)}:[^\n]*\n(?P(?:\t[^\n]*\n)+)", content, re.MULTILINE) + assert match is not None, f"missing {target!r} target in {path}" + return match.group("recipe") + + +def test_extensions_dependency_group_is_part_of_the_default_sync() -> None: + project = tomllib.loads((BACKEND_ROOT / "pyproject.toml").read_text(encoding="utf-8")) + + assert isinstance(project["dependency-groups"]["extensions"], list) + assert set(project["tool"]["uv"]["default-groups"]) == {"dev", "extensions"} + + +def test_backend_make_targets_never_mutate_the_extension_lock() -> None: + makefile = BACKEND_ROOT / "Makefile" + + assert "uv sync --locked" in _make_recipe(makefile, "install") + for target in ("dev", "gateway"): + recipe = _make_recipe(makefile, target) + assert "uv run --locked uvicorn" in recipe, target + assert "uv sync" not in recipe, target + + +def test_root_local_startup_syncs_the_locked_backend_before_runtime() -> None: + root_install = _make_recipe(REPO_ROOT / "Makefile", "install") + serve = (REPO_ROOT / "scripts" / "serve.sh").read_text(encoding="utf-8") + + assert "cd backend && uv sync --locked" in root_install + sync_at = serve.find("uv sync --locked") + runtime_at = serve.find("uv run --no-sync uvicorn app.gateway.app:app") + assert sync_at != -1 + assert runtime_at > sync_at + + +def test_root_makefile_exposes_extension_management_commands() -> None: + makefile = REPO_ROOT / "Makefile" + + install = _make_recipe(makefile, "extension-install") + assert "deerflow extensions install" in install + assert "--source-env __deerflow_extension_source__" in install + assert "DEER_FLOW_EXTENSION_SOURCE" not in install + assert "$(SOURCE)" not in install + assert "uv run --frozen --no-group extensions" in install + assert "--yes" not in install + + for target, command in ( + ("extension-list", "deerflow extensions list"), + ("extension-enable", "deerflow extensions enable"), + ("extension-disable", "deerflow extensions disable"), + ("extension-remove", "deerflow extensions remove"), + ): + recipe = _make_recipe(makefile, target) + assert command in recipe + assert "uv run --frozen --no-group extensions" in recipe + + +def test_extension_management_bootstrap_does_not_resolve_a_broken_extension_source() -> None: + makefile = REPO_ROOT / "Makefile" + + for target in ( + "extension-install", + "extension-list", + "extension-enable", + "extension-disable", + "extension-remove", + ): + recipe = _make_recipe(makefile, target) + assert "uv run --frozen --no-group extensions" in recipe, target + assert "--locked" not in recipe, target + + +def test_frozen_management_bootstrap_installs_core_without_reading_a_missing_extension( + tmp_path: Path, +) -> None: + project = tmp_path / "backend" + wheels = tmp_path / "wheels" + project.mkdir() + wheels.mkdir() + + def _write_wheel(distribution: str, module: str) -> Path: + wheel = wheels / f"{distribution.replace('-', '_')}-1.0.0-py3-none-any.whl" + dist_info = f"{distribution.replace('-', '_')}-1.0.0.dist-info" + records = { + f"{module}/__init__.py": "VALUE = 'installed'\n", + f"{dist_info}/METADATA": f"Metadata-Version: 2.1\nName: {distribution}\nVersion: 1.0.0\n", + f"{dist_info}/WHEEL": "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + } + records[f"{dist_info}/RECORD"] = "".join(f"{name},,\n" for name in (*records, f"{dist_info}/RECORD")) + with zipfile.ZipFile(wheel, "w") as archive: + for name, content in records.items(): + archive.writestr(name, content) + return wheel + + core_wheel = _write_wheel("bootstrap-core", "bootstrap_core") + missing_extension = _write_wheel("broken-extension", "broken_extension") + (project / "pyproject.toml").write_text( + f"""\ +[project] +name = "bootstrap-host" +version = "0.0.0" +requires-python = ">=3.12" +dependencies = ["bootstrap-core @ {core_wheel.as_uri()}"] + +[dependency-groups] +extensions = ["broken-extension @ {missing_extension.as_uri()}"] +""", + encoding="utf-8", + ) + environment = os.environ.copy() + environment["UV_CACHE_DIR"] = str(tmp_path / "uv-cache") + subprocess.run(["uv", "lock"], cwd=project, env=environment, check=True, capture_output=True) + missing_extension.unlink() + + completed = subprocess.run( + [ + "uv", + "run", + "--frozen", + "--no-group", + "extensions", + "python", + "-c", + "import bootstrap_core; assert bootstrap_core.VALUE == 'installed'", + ], + cwd=project, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + + +def test_root_extension_shortcuts_are_cross_platform_and_keep_trust_confirmation() -> None: + makefile = REPO_ROOT / "Makefile" + + for target in ( + "extension-install", + "extension-enable", + "extension-disable", + "extension-remove", + ): + recipe = _make_recipe(makefile, target) + assert "test -n" not in recipe, target + assert "usage: make" in recipe, target + + assert "--yes" not in _make_recipe(makefile, "extension-install") + + +def test_root_extension_shortcuts_reject_ambient_environment_arguments() -> None: + environment = os.environ.copy() + + for target, variable in (("extension-install", "SOURCE"), ("extension-enable", "NAME")): + environment[variable] = "ambient-value" + result = subprocess.run( + ["make", "--no-print-directory", "-n", target], + cwd=REPO_ROOT, + env=environment, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 2, target + assert f"usage: make {target}" in result.stderr, target + + +@pytest.mark.parametrize( + ("target", "variable", "env_option"), + [ + ("extension-install", "SOURCE", "--source-env __deerflow_extension_source__"), + ("extension-enable", "NAME", "--name-env __deerflow_extension_name__"), + ("extension-disable", "NAME", "--name-env __deerflow_extension_name__"), + ("extension-remove", "NAME", "--name-env __deerflow_extension_name__"), + ], +) +def test_root_extension_shortcuts_keep_command_line_arguments_out_of_the_shell_recipe( + target: str, + variable: str, + env_option: str, +) -> None: + marker = "EXTENSION_WRAPPER_INJECTION" + malicious_value = f'"; printf {marker}; #$(shell printf {marker})' + + result = subprocess.run( + ["make", "--no-print-directory", "-n", target, f"{variable}={malicious_value}"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert marker not in result.stdout + assert env_option in result.stdout + assert "$DEER_FLOW_EXTENSION_" not in result.stdout + assert "%DEER_FLOW_EXTENSION_" not in result.stdout + + +@pytest.mark.parametrize( + ("target", "variable", "env_option"), + [ + ("extension-install", "SOURCE", "--source-env __deerflow_extension_source__"), + ("extension-enable", "NAME", "--name-env __deerflow_extension_name__"), + ("extension-disable", "NAME", "--name-env __deerflow_extension_name__"), + ("extension-remove", "NAME", "--name-env __deerflow_extension_name__"), + ], +) +def test_root_extension_shortcuts_keep_values_out_of_the_cmd_recipe_on_windows( + target: str, + variable: str, + env_option: str, +) -> None: + marker = "EXTENSION_WRAPPER_INJECTION" + malicious_value = f'"; printf {marker}; #$(shell printf {marker})' + + result = subprocess.run( + [ + "make", + "--no-print-directory", + "-n", + "OS=Windows_NT", + target, + f"{variable}={malicious_value}", + ], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert marker not in result.stdout + assert env_option in result.stdout + assert "$DEER_FLOW_EXTENSION_" not in result.stdout + assert "%DEER_FLOW_EXTENSION_" not in result.stdout + + +def test_docker_dev_entrypoint_syncs_the_lock_before_runtime() -> None: + entrypoint = (REPO_ROOT / "docker" / "dev-entrypoint.sh").read_text(encoding="utf-8") + + sync_at = entrypoint.find("uv sync --locked --all-packages") + runtime_at = entrypoint.find("uv run --no-sync uvicorn app.gateway.app:app") + assert sync_at != -1 + assert runtime_at > sync_at + + +def test_docker_image_builds_from_the_lock_and_never_syncs_at_runtime() -> None: + dockerfile = (BACKEND_ROOT / "Dockerfile").read_text(encoding="utf-8") + production_compose = (REPO_ROOT / "docker" / "docker-compose.yaml").read_text(encoding="utf-8") + + assert "uv sync --locked --extra redis" in dockerfile + assert "ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.1" in dockerfile + assert dockerfile.count("uv run --no-sync uvicorn app.gateway.app:app") == 2 + assert "uv run --no-sync uvicorn app.gateway.app:app" in production_compose + for compose_name in ("docker-compose.yaml", "docker-compose-dev.yaml"): + compose = (REPO_ROOT / "docker" / compose_name).read_text(encoding="utf-8") + assert "ghcr.io/astral-sh/uv:0.11.1" in compose + + +def test_docker_context_keeps_every_managed_extension_artifact() -> None: + dockerignore = (REPO_ROOT / ".dockerignore").read_text(encoding="utf-8") + + reinclusion = "!backend/extensions/sources/**" + assert reinclusion in dockerignore + assert dockerignore.rfind(reinclusion) > dockerignore.rfind("*.md") + assert dockerignore.rfind(reinclusion) > dockerignore.rfind("assets/") + assert dockerignore.rfind(reinclusion) > dockerignore.rfind("*.so") + + +def test_docker_builder_can_resolve_locked_git_extensions() -> None: + dockerfile = (BACKEND_ROOT / "Dockerfile").read_text(encoding="utf-8") + + builder_packages = re.search( + r"apt-get install -y \\\n(?P.*?) && mkdir -p /etc/apt/keyrings", + dockerfile, + re.DOTALL, + ) + assert builder_packages is not None + assert re.search(r"^\s*git\s+\\$", builder_packages.group("packages"), re.MULTILINE) diff --git a/backend/tests/test_extension_gateway_wiring.py b/backend/tests/test_extension_gateway_wiring.py new file mode 100644 index 000000000..411f08123 --- /dev/null +++ b/backend/tests/test_extension_gateway_wiring.py @@ -0,0 +1,985 @@ +"""Gateway binding tests for app-scoped extension contributions.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from types import SimpleNamespace + +import pytest + +from deerflow.extensions.registry import ExtensionRegistry + + +class _Service: + def __init__( + self, + name: str, + events: list[str], + *, + fail_start: bool = False, + fail_stop: bool = False, + ) -> None: + self.name = name + self.events = events + self.fail_start = fail_start + self.fail_stop = fail_stop + self.deps = None + + async def start(self, deps) -> None: + self.events.append(f"start:{self.name}") + self.deps = deps + if self.fail_start: + raise RuntimeError(f"{self.name} failed") + + async def stop(self) -> None: + self.events.append(f"stop:{self.name}") + if self.fail_stop: + raise RuntimeError(f"{self.name} failed") + + +@pytest.mark.asyncio +async def test_services_start_in_order_with_narrow_deps_and_fail_open(): + from deerflow.extensions.gateway import start_services + + events: list[str] = [] + first = _Service("first", events, fail_start=True) + second = _Service("second", events) + registry = ExtensionRegistry() + with registry.attributed_to("first:install"): + registry.service(first) + with registry.attributed_to("second:install"): + registry.service(second) + extensions = registry.build() + session_factory = object() + config = SimpleNamespace( + token_budget=SimpleNamespace(enabled=False, max_tokens=999), + subagents=SimpleNamespace(max_total_per_run=4), + ) + + diagnostics = await start_services(extensions, config, session_factory) + + assert events == ["start:first", "start:second"] + assert first.deps is second.deps + assert second.deps.app_store is extensions.app_store + assert second.deps.session_factory is session_factory + assert second.deps.policy.token_budget_enabled is False + assert second.deps.policy.max_total_tokens is None + assert second.deps.policy.max_subagents_per_run == 4 + assert [(diagnostic.source, diagnostic.level) for diagnostic in diagnostics] == [("first:install", "error")] + + +@pytest.mark.asyncio +async def test_service_originated_cancelled_error_does_not_abort_start_batch(): + from deerflow.extensions.gateway import start_services + + events: list[str] = [] + + class _CancelsItself(_Service): + async def start(self, deps) -> None: + self.events.append(f"start:{self.name}") + raise asyncio.CancelledError() + + first = _CancelsItself("first", events) + second = _Service("second", events) + registry = ExtensionRegistry() + with registry.attributed_to("first:install"): + registry.service(first) + with registry.attributed_to("second:install"): + registry.service(second) + + diagnostics = await start_services(registry.build(), SimpleNamespace(), None) + + assert events == ["start:first", "start:second"] + assert len(diagnostics) == 1 + assert diagnostics[0].source == "first:install" + assert "CancelledError" in diagnostics[0].message + + +@pytest.mark.asyncio +async def test_services_stop_in_reverse_order_and_fail_open(): + from deerflow.extensions.gateway import stop_services + + events: list[str] = [] + first = _Service("first", events) + second = _Service("second", events, fail_stop=True) + registry = ExtensionRegistry() + with registry.attributed_to("first:install"): + registry.service(first) + with registry.attributed_to("second:install"): + registry.service(second) + + diagnostics = await stop_services(registry.build()) + + assert events == ["stop:second", "stop:first"] + assert [(diagnostic.source, diagnostic.level) for diagnostic in diagnostics] == [("second:install", "error")] + + +@pytest.mark.asyncio +async def test_service_originated_cancelled_error_does_not_abort_stop_batch(): + from deerflow.extensions.gateway import stop_services + + events: list[str] = [] + + class _CancelsItself(_Service): + async def stop(self) -> None: + self.events.append(f"stop:{self.name}") + raise asyncio.CancelledError() + + first = _Service("first", events) + second = _CancelsItself("second", events) + registry = ExtensionRegistry() + with registry.attributed_to("first:install"): + registry.service(first) + with registry.attributed_to("second:install"): + registry.service(second) + + diagnostics = await stop_services(registry.build()) + + assert events == ["stop:second", "stop:first"] + assert len(diagnostics) == 1 + assert diagnostics[0].source == "second:install" + assert "CancelledError" in diagnostics[0].message + + +@pytest.mark.asyncio +async def test_each_service_stop_has_its_own_timeout_budget(): + from deerflow.extensions.gateway import stop_services + + events: list[str] = [] + + class _HangingService(_Service): + async def stop(self) -> None: + self.events.append(f"stop:{self.name}") + await asyncio.Event().wait() + + first = _Service("first", events) + second = _HangingService("second", events) + registry = ExtensionRegistry() + with registry.attributed_to("first:install"): + registry.service(first) + with registry.attributed_to("second:install"): + registry.service(second) + + diagnostics = await stop_services(registry.build(), timeout_seconds=0.01) + + assert events == ["stop:second", "stop:first"] + assert len(diagnostics) == 1 + assert diagnostics[0].source == "second:install" + assert "timed out" in diagnostics[0].message + + +@pytest.mark.asyncio +async def test_service_originated_timeout_error_is_reported_as_failure_not_budget_expiry(): + from deerflow.extensions.gateway import stop_services + + events: list[str] = [] + + class _RaisesTimeout(_Service): + async def stop(self) -> None: + self.events.append(f"stop:{self.name}") + raise TimeoutError("extension deadline") + + first = _Service("first", events) + second = _RaisesTimeout("second", events) + registry = ExtensionRegistry() + with registry.attributed_to("first:install"): + registry.service(first) + with registry.attributed_to("second:install"): + registry.service(second) + + diagnostics = await stop_services(registry.build(), timeout_seconds=1.0) + + assert events == ["stop:second", "stop:first"] + assert diagnostics[0].source == "second:install" + assert "failed" in diagnostics[0].message + assert "timed out" not in diagnostics[0].message + + +@pytest.mark.parametrize( + ("owner_path", "owner_protocol", "candidate_path", "candidate_protocol", "rejected"), + [ + ("/exact", "GET", "/exact", "GET", True), + ("/items/{item_id}", "GET", "/items/{id}", "GET", True), + ("/items/{item_id}", "GET", "/items/new", "GET", True), + ("/items/{item_id}", "GET", "/items/prefix-{id}", "GET", True), + ("/pre{tenant}", "GET", "/prefoo{id}", "GET", True), + ("/items/new", "GET", "/items/{id}", "GET", False), + ("/records/{value}", "GET", "/records/{id:int}", "GET", True), + ("/records/{value:int}", "GET", "/records/0{id:int}", "GET", True), + ("/records/{value:int}", "GET", "/records/new", "GET", False), + ("/files/{rest:path}", "GET", "/files/{id:int}", "GET", True), + ("/files/{rest:path}", "GET", "/files/{id}", "GET", False), + ("/x/{rest:path}/tail", "GET", "/x/a/{id}/tail", "GET", False), + ("/items/{item_id}", "GET", "/items/{id}", "POST", False), + ("/live/{item_id}", "WS", "/live/{id}", "GET", False), + ], + ids=[ + "exact", + "renamed-parameter", + "dynamic-shadows-static", + "dynamic-shadows-compound", + "compound-trailing-str-shadows-narrower-compound", + "static-does-not-shadow-dynamic", + "str-covers-int", + "int-shadows-digit-compound", + "int-does-not-cover-static", + "path-covers-descendant", + "path-does-not-cover-newline-capable-str", + "nonterminal-path-does-not-cover-newline-capable-str", + "disjoint-http-methods", + "websocket-does-not-shadow-http", + ], +) +def test_router_conflicts_follow_starlette_dispatch_order( + owner_path, + owner_protocol, + candidate_path, + candidate_protocol, + rejected, +): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + async def endpoint(): + return {"ok": True} + + app = FastAPI() + if owner_protocol == "WS": + app.add_api_websocket_route(owner_path, endpoint) + else: + app.add_api_route(owner_path, endpoint, methods=[owner_protocol]) + + router = APIRouter() + if candidate_protocol == "WS": + router.add_api_websocket_route(candidate_path, endpoint) + else: + router.add_api_route(candidate_path, endpoint, methods=[candidate_protocol]) + registry = ExtensionRegistry() + with registry.attributed_to("candidate:install"): + registry.routers((router,)) + + diagnostics = include_contributed_routers(app, registry.build()) + + assert bool(diagnostics) is rejected + if rejected: + assert diagnostics[0].source == "candidate:install" + assert "host" in diagnostics[0].message + assert candidate_path in diagnostics[0].message + else: + assert any(getattr(route, "path", None) == candidate_path for route in app.routes) + + +@pytest.mark.parametrize("convertor_name", ["flip", "int"]) +def test_re_registered_convertor_does_not_create_a_false_shadow( + monkeypatch, + convertor_name, +): + from fastapi import APIRouter, FastAPI + from starlette.convertors import CONVERTOR_TYPES, Convertor + from starlette.routing import Match + + from deerflow.extensions.gateway import include_contributed_routers + + class DigitsConvertor(Convertor[str]): + regex = "[0-9]+" + + def convert(self, value: str) -> str: + return value + + def to_string(self, value: str) -> str: + return value + + class LettersConvertor(Convertor[str]): + regex = "[A-Z]+" + + def convert(self, value: str) -> str: + return value + + def to_string(self, value: str) -> str: + return value + + async def endpoint(value: str): + return {"value": value} + + monkeypatch.setitem(CONVERTOR_TYPES, convertor_name, DigitsConvertor()) + route_path = f"/owned/{{value:{convertor_name}}}" + app = FastAPI() + app.add_api_route(route_path, endpoint, methods=["GET"]) + owner = app.routes[-1] + + monkeypatch.setitem(CONVERTOR_TYPES, convertor_name, LettersConvertor()) + router = APIRouter() + router.add_api_route(route_path, endpoint, methods=["GET"]) + registry = ExtensionRegistry() + with registry.attributed_to("candidate:install"): + registry.routers((router,)) + + diagnostics = include_contributed_routers(app, registry.build()) + + assert diagnostics == [] + candidate = app.routes[-1] + scope = { + "type": "http", + "path": "/owned/A", + "method": "GET", + "root_path": "", + } + assert owner.matches(scope)[0] is Match.NONE + assert candidate.matches(scope)[0] is Match.FULL + + +def test_router_claim_uses_converter_semantics_at_include_time(monkeypatch): + from fastapi import APIRouter, FastAPI + from starlette.convertors import CONVERTOR_TYPES, Convertor + from starlette.routing import Match + + from deerflow.extensions.gateway import include_contributed_routers + + class DigitsConvertor(Convertor[str]): + regex = "[0-9]+" + + def convert(self, value: str) -> str: + return value + + def to_string(self, value: str) -> str: + return value + + class LettersConvertor(Convertor[str]): + regex = "[A-Z]+" + + def convert(self, value: str) -> str: + return value + + def to_string(self, value: str) -> str: + return value + + async def endpoint(value: str): + return {"value": value} + + monkeypatch.setitem(CONVERTOR_TYPES, "flip", DigitsConvertor()) + route_path = "/owned/{value:flip}" + app = FastAPI() + app.add_api_route(route_path, endpoint, methods=["GET"]) + owner = app.routes[-1] + router = APIRouter() + router.add_api_route(route_path, endpoint, methods=["GET"]) + + monkeypatch.setitem(CONVERTOR_TYPES, "flip", LettersConvertor()) + registry = ExtensionRegistry() + with registry.attributed_to("candidate:install"): + registry.routers((router,)) + + diagnostics = include_contributed_routers(app, registry.build()) + + assert diagnostics == [] + candidate = app.routes[-1] + scope = { + "type": "http", + "path": "/owned/A", + "method": "GET", + "root_path": "", + } + assert owner.matches(scope)[0] is Match.NONE + assert candidate.matches(scope)[0] is Match.FULL + + +def test_recompiled_converter_cannot_enter_a_public_namespace(monkeypatch): + from fastapi import APIRouter, FastAPI + from starlette.convertors import CONVERTOR_TYPES, Convertor + + from deerflow.extensions.gateway import include_contributed_routers + + class PublicPathConvertor(Convertor[str]): + regex = r"webhooks/.+" + + def convert(self, value: str) -> str: + return value + + def to_string(self, value: str) -> str: + return value + + async def endpoint(value: str): + return {"value": value} + + router = APIRouter() + router.add_api_route("/api/{value:int}", endpoint, methods=["GET"]) + monkeypatch.setitem(CONVERTOR_TYPES, "int", PublicPathConvertor()) + registry = ExtensionRegistry() + with registry.attributed_to("candidate:install"): + registry.routers((router,)) + + app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None) + diagnostics = include_contributed_routers(app, registry.build()) + + assert [diagnostic.source for diagnostic in diagnostics] == ["candidate:install"] + assert "public namespace" in diagnostics[0].message + assert not any(getattr(route, "path", None) == "/api/{value:int}" for route in app.routes) + + +def test_host_mount_claims_descendant_http_paths(): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + app = FastAPI() + app.mount("/assets", FastAPI()) + router = APIRouter() + + @router.get("/assets/{name:int}") + async def asset(name: int): + return {"name": name} + + registry = ExtensionRegistry() + with registry.attributed_to("assets:install"): + registry.routers((router,)) + + diagnostics = include_contributed_routers(app, registry.build()) + + assert len(diagnostics) == 1 + assert diagnostics[0].source == "assets:install" + assert "host" in diagnostics[0].message + + +def test_dynamic_host_mount_claims_matching_descendants(): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + app = FastAPI() + app.mount("/pre{tenant}", FastAPI()) + router = APIRouter() + + @router.get("/prefoo/{item_id:int}") + async def item(item_id: int): + return {"item_id": item_id} + + registry = ExtensionRegistry() + with registry.attributed_to("mount:install"): + registry.routers((router,)) + + diagnostics = include_contributed_routers(app, registry.build()) + + assert [diagnostic.source for diagnostic in diagnostics] == ["mount:install"] + assert "host" in diagnostics[0].message + + +@pytest.mark.parametrize( + ("mount_path", "candidate_path", "witness"), + [ + ("/assets", "/assets/{name}", "/assets/a\nb"), + ("/pre{tenant}", "/prefoo{id}/{child}", "/prefoo1/a\nb"), + ], +) +def test_host_mount_does_not_claim_newline_capable_str_descendants( + mount_path, + candidate_path, + witness, +): + from fastapi import APIRouter, FastAPI + from starlette.routing import Match, Mount + + from deerflow.extensions.gateway import include_contributed_routers + + async def endpoint(): + return {"ok": True} + + app = FastAPI() + app.mount(mount_path, FastAPI()) + host_mount = next(route for route in app.routes if isinstance(route, Mount) and route.path == mount_path) + router = APIRouter() + router.add_api_route(candidate_path, endpoint, methods=["GET"]) + registry = ExtensionRegistry() + with registry.attributed_to("mount:install"): + registry.routers((router,)) + + diagnostics = include_contributed_routers(app, registry.build()) + + assert diagnostics == [] + candidate_route = next(route for route in app.routes if getattr(route, "path", None) == candidate_path) + scope = { + "type": "http", + "path": witness, + "method": "GET", + "root_path": "", + } + assert host_mount.matches(scope)[0] is Match.NONE + assert candidate_route.matches(scope)[0] is Match.FULL + + +def test_contributed_websocket_route_is_rejected_until_host_auth_wraps_it(): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + async def websocket_endpoint(websocket): + await websocket.close() + + router = APIRouter() + router.add_api_websocket_route("/extension-ws", websocket_endpoint) + registry = ExtensionRegistry() + with registry.attributed_to("websocket:install"): + registry.routers((router,)) + + app = FastAPI() + diagnostics = include_contributed_routers(app, registry.build()) + + assert [diagnostic.source for diagnostic in diagnostics] == ["websocket:install"] + assert "WebSocket" in diagnostics[0].message + assert not any(getattr(route, "path", None) == "/extension-ws" for route in app.routes) + + +@pytest.mark.parametrize( + "path", + [ + "/health-extension", + "/docs-private", + "/redoc-private", + "/api/webhooks/extension", + "/api/{rest:path}", + "/api/{section}/extension", + ], +) +def test_extension_routes_cannot_enter_host_public_namespaces(path): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + async def endpoint(): + return {"ok": True} + + router = APIRouter() + router.add_api_route(path, endpoint, methods=["GET"]) + registry = ExtensionRegistry() + with registry.attributed_to("public:install"): + registry.routers((router,)) + + app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None) + diagnostics = include_contributed_routers(app, registry.build()) + + assert [diagnostic.source for diagnostic in diagnostics] == ["public:install"] + assert "public namespace" in diagnostics[0].message + assert not any(getattr(route, "path", None) == path for route in app.routes) + + +@pytest.mark.parametrize( + "path", + [ + "/api", + "/heal", + "/api/{item_id:int}", + "/api/{item_id}", + ], +) +def test_extension_routes_that_cannot_enter_a_public_namespace_are_allowed(path): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + async def endpoint(): + return {"ok": True} + + router = APIRouter() + router.add_api_route(path, endpoint, methods=["GET"]) + registry = ExtensionRegistry() + with registry.attributed_to("private:install"): + registry.routers((router,)) + + app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None) + diagnostics = include_contributed_routers(app, registry.build()) + + assert diagnostics == [] + assert any(getattr(route, "path", None) == path for route in app.routes) + + +def test_unknown_convertor_near_a_public_namespace_fails_closed(monkeypatch): + from fastapi import APIRouter, FastAPI + from starlette.convertors import CONVERTOR_TYPES, Convertor + + from deerflow.extensions.gateway import include_contributed_routers + + class UppercaseConvertor(Convertor[str]): + regex = "[A-Z]+" + + def convert(self, value: str) -> str: + return value + + def to_string(self, value: str) -> str: + return value + + monkeypatch.setitem(CONVERTOR_TYPES, "uppercase", UppercaseConvertor()) + router = APIRouter() + router.add_api_route( + "/api/{value:uppercase}", + lambda: {"ok": True}, + methods=["GET"], + ) + registry = ExtensionRegistry() + with registry.attributed_to("custom-public:install"): + registry.routers((router,)) + + diagnostics = include_contributed_routers( + FastAPI(docs_url=None, redoc_url=None, openapi_url=None), + registry.build(), + ) + + assert [diagnostic.source for diagnostic in diagnostics] == ["custom-public:install"] + assert "public namespace" in diagnostics[0].message + + +def test_private_custom_convertor_with_named_backreference_is_allowed(monkeypatch): + from fastapi import APIRouter, FastAPI + from starlette.convertors import CONVERTOR_TYPES, Convertor + + from deerflow.extensions.gateway import include_contributed_routers + + class DoubledLetterConvertor(Convertor[str]): + regex = r"(?P[A-Z])(?P=char)" + + def convert(self, value: str) -> str: + return value + + def to_string(self, value: str) -> str: + return value + + monkeypatch.setitem(CONVERTOR_TYPES, "doubled", DoubledLetterConvertor()) + router = APIRouter() + router.add_api_route( + "/private/{value:doubled}", + lambda value: {"value": value}, + methods=["GET"], + ) + registry = ExtensionRegistry() + with registry.attributed_to("custom-private:install"): + registry.routers((router,)) + + app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None) + diagnostics = include_contributed_routers(app, registry.build()) + + assert diagnostics == [] + assert any(getattr(route, "path", None) == "/private/{value:doubled}" for route in app.routes) + + +def test_extension_public_paths_track_auth_middleware_public_paths(): + from app.gateway.auth_middleware import ( + _PUBLIC_EXACT_PATHS, + _PUBLIC_PATH_PREFIXES, + _is_public, + ) + from deerflow.extensions.gateway import ( + _HOST_PUBLIC_EXACT_PATHS, + _HOST_PUBLIC_PATH_PREFIXES, + ) + + assert _HOST_PUBLIC_PATH_PREFIXES == _PUBLIC_PATH_PREFIXES + assert _HOST_PUBLIC_EXACT_PATHS == _PUBLIC_EXACT_PATHS + assert all(_is_public(f"{path}//") for path in _HOST_PUBLIC_EXACT_PATHS) + + +@pytest.mark.parametrize( + ("host_path", "host_method", "candidate_path", "candidate_method"), + [ + ( + "/api/v1/auth/login/local", + "POST", + "/api/v1/auth/login/local", + "GET", + ), + ( + "/api/v1/auth/login/local", + "POST", + "/api/v1/auth/login/local/", + "GET", + ), + ( + "/api/v1/auth/login/local", + "POST", + "/api/v1/auth/login/local//", + "GET", + ), + ("/api/v1/auth/me", "GET", "/api/v1/auth/me", "POST"), + ("/api/v1/auth/me", "GET", "/api/v1/auth/me/", "POST"), + ], +) +def test_extension_routes_cannot_claim_reserved_exact_paths_with_a_disjoint_method( + host_path, + host_method, + candidate_path, + candidate_method, +): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + async def endpoint(): + return {"ok": True} + + app = FastAPI() + app.add_api_route(host_path, endpoint, methods=[host_method]) + router = APIRouter() + router.add_api_route(candidate_path, endpoint, methods=[candidate_method]) + registry = ExtensionRegistry() + with registry.attributed_to("public-exact:install"): + registry.routers((router,)) + + diagnostics = include_contributed_routers(app, registry.build()) + + assert [diagnostic.source for diagnostic in diagnostics] == ["public-exact:install"] + assert "reserved" in diagnostics[0].message + assert not any(getattr(route, "path", None) == candidate_path and getattr(route, "methods", set()) == {candidate_method} for route in app.routes) + + +@pytest.mark.parametrize( + "path", + [ + "/api/v1/auth/me", + "/api/v1/auth/me/", + "/api/v1/auth/me//", + ], +) +def test_extension_csrf_reserved_exact_paths_track_csrf_exemption( + monkeypatch, + path, +): + from starlette.requests import Request + + from app.gateway import csrf_middleware + from deerflow.extensions.gateway import ( + _CSRF_STATE_CHANGING_METHODS, + _HOST_CSRF_EXEMPT_EXACT_PATHS, + _HOST_PUBLIC_EXACT_PATHS, + ) + + monkeypatch.setattr(csrf_middleware, "is_auth_disabled", lambda: False) + request = Request( + { + "type": "http", + "method": "POST", + "scheme": "http", + "path": path, + "raw_path": path.encode(), + "query_string": b"", + "headers": [], + "server": ("testserver", 80), + } + ) + + assert path.rstrip("/") in _HOST_CSRF_EXEMPT_EXACT_PATHS + assert _HOST_CSRF_EXEMPT_EXACT_PATHS == csrf_middleware._CSRF_EXEMPT_EXACT_PATHS + assert _CSRF_STATE_CHANGING_METHODS == csrf_middleware._CSRF_STATE_CHANGING_METHODS + assert csrf_middleware._AUTH_EXEMPT_PATHS <= _HOST_PUBLIC_EXACT_PATHS + assert csrf_middleware.should_check_csrf(request) is False + + +def test_safe_method_at_csrf_exempt_exact_path_is_not_reserved(): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + router = APIRouter() + router.add_api_route( + "/api/v1/auth/me", + lambda: {"ok": True}, + methods=["GET"], + ) + registry = ExtensionRegistry() + with registry.attributed_to("safe-csrf:install"): + registry.routers((router,)) + + app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None) + diagnostics = include_contributed_routers(app, registry.build()) + + assert diagnostics == [] + assert any(getattr(route, "path", None) == "/api/v1/auth/me" for route in app.routes) + + +def test_router_with_one_conflict_is_rejected_atomically_and_names_first_owner(): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + async def endpoint(): + return {"ok": True} + + first = APIRouter() + first.add_api_route("/shared", endpoint, methods=["GET"]) + second = APIRouter() + second.add_api_route("/would-have-been-reachable", endpoint, methods=["GET"]) + second.add_api_route("/shared", endpoint, methods=["GET"]) + registry = ExtensionRegistry() + with registry.attributed_to("first:install"): + registry.routers((first,)) + with registry.attributed_to("second:install"): + registry.routers((second,)) + + app = FastAPI() + diagnostics = include_contributed_routers(app, registry.build()) + paths = [getattr(route, "path", None) for route in app.routes] + + assert "/shared" in paths + assert "/would-have-been-reachable" not in paths + assert len(diagnostics) == 1 + assert diagnostics[0].source == "second:install" + assert "first:install" in diagnostics[0].message + + +def test_router_is_rejected_when_its_own_earlier_route_shadows_a_later_one(): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + async def endpoint(): + return {"ok": True} + + router = APIRouter() + router.add_api_route("/same/{value}", endpoint, methods=["GET"]) + router.add_api_route("/same/fixed", endpoint, methods=["GET"]) + registry = ExtensionRegistry() + with registry.attributed_to("self-shadow:install"): + registry.routers((router,)) + + app = FastAPI() + diagnostics = include_contributed_routers(app, registry.build()) + + assert [diagnostic.source for diagnostic in diagnostics] == ["self-shadow:install"] + assert "self-shadow:install" in diagnostics[0].message + assert not any(getattr(route, "path", "").startswith("/same/") for route in app.routes) + + +def test_contributed_mount_is_rejected_but_does_not_starve_later_router(): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + bad = APIRouter() + bad.mount("/nested", FastAPI()) + good = APIRouter() + + @good.get("/extension-good") + async def extension_good(): + return {"ok": True} + + registry = ExtensionRegistry() + with registry.attributed_to("bad:install"): + registry.routers((bad,)) + with registry.attributed_to("good:install"): + registry.routers((good,)) + + app = FastAPI() + diagnostics = include_contributed_routers(app, registry.build()) + + assert [diagnostic.source for diagnostic in diagnostics] == ["bad:install"] + assert "Mount" in diagnostics[0].message + assert any(getattr(route, "path", None) == "/extension-good" for route in app.routes) + + +def test_router_with_unsupported_route_item_is_rejected_atomically(): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + async def endpoint(): + return {"ok": True} + + router = APIRouter() + router.add_api_route("/otherwise-valid", endpoint, methods=["GET"]) + router.routes.append(object()) + registry = ExtensionRegistry() + with registry.attributed_to("unsupported:install"): + registry.routers((router,)) + + app = FastAPI() + diagnostics = include_contributed_routers(app, registry.build()) + + assert [diagnostic.source for diagnostic in diagnostics] == ["unsupported:install"] + assert "unsupported route" in diagnostics[0].message + assert not any(getattr(route, "path", None) == "/otherwise-valid" for route in app.routes) + + +def test_include_router_failure_rolls_back_partial_routes_before_continuing(): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + async def endpoint(): + return {"ok": True} + + broken = APIRouter() + broken.add_api_route("/partial", endpoint, methods=["GET"]) + broken.add_api_route("/explodes", endpoint, methods=["GET"]) + broken.routes[-1].endpoint = None + later = APIRouter() + later.add_api_route("/partial", endpoint, methods=["GET"]) + + registry = ExtensionRegistry() + with registry.attributed_to("broken:install"): + registry.routers((broken,)) + with registry.attributed_to("later:install"): + registry.routers((later,)) + + app = FastAPI() + diagnostics = include_contributed_routers(app, registry.build()) + partial_routes = [route for route in app.routes if getattr(route, "path", None) == "/partial"] + + assert [diagnostic.source for diagnostic in diagnostics] == ["broken:install"] + assert len(partial_routes) == 1 + + +def test_router_lifecycle_hooks_are_rejected_in_favor_of_extension_service(): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + async def endpoint(): + return {"ok": True} + + async def startup_hook(): + raise RuntimeError("must never be installed") + + bad = APIRouter() + bad.add_api_route("/has-lifecycle", endpoint, methods=["GET"]) + bad.add_event_handler("startup", startup_hook) + good = APIRouter() + good.add_api_route("/after-lifecycle", endpoint, methods=["GET"]) + registry = ExtensionRegistry() + with registry.attributed_to("lifecycle:install"): + registry.routers((bad,)) + with registry.attributed_to("good:install"): + registry.routers((good,)) + + app = FastAPI() + diagnostics = include_contributed_routers(app, registry.build()) + paths = [getattr(route, "path", None) for route in app.routes] + + assert [diagnostic.source for diagnostic in diagnostics] == ["lifecycle:install"] + assert "ExtensionService" in diagnostics[0].message + assert "/has-lifecycle" not in paths + assert "/after-lifecycle" in paths + assert startup_hook not in app.router.on_startup + + +def test_router_custom_lifespan_is_rejected_in_favor_of_extension_service(): + from fastapi import APIRouter, FastAPI + + from deerflow.extensions.gateway import include_contributed_routers + + @asynccontextmanager + async def lifespan(_app): + yield + + async def endpoint(): + return {"ok": True} + + router = APIRouter(lifespan=lifespan) + router.add_api_route("/custom-lifespan", endpoint, methods=["GET"]) + registry = ExtensionRegistry() + with registry.attributed_to("lifespan:install"): + registry.routers((router,)) + + app = FastAPI() + diagnostics = include_contributed_routers(app, registry.build()) + + assert [diagnostic.source for diagnostic in diagnostics] == ["lifespan:install"] + assert "ExtensionService" in diagnostics[0].message + assert not any(getattr(route, "path", None) == "/custom-lifespan" for route in app.routes) diff --git a/backend/tests/test_extension_loader.py b/backend/tests/test_extension_loader.py index b14b47146..5deea5471 100644 --- a/backend/tests/test_extension_loader.py +++ b/backend/tests/test_extension_loader.py @@ -28,6 +28,30 @@ def test_no_specs_yields_empty_result(): assert loaded.has_middleware_contributors is False +def test_host_disabled_required_extension_is_skipped_before_resolution(monkeypatch): + def _must_not_resolve(path: str): + raise AssertionError(f"disabled extension was resolved: {path}") + + monkeypatch.setattr("deerflow.extensions.loader.resolve_variable", _must_not_resolve) + + loaded, diagnostics = load_extensions([ExtensionSpec(use="missing_extension:install", enabled=False, required=True)]) + + assert diagnostics == [] + assert loaded.has_middleware_contributors is False + + +def test_manager_metadata_is_accepted_without_reaching_the_install_hook(): + spec = ExtensionSpec( + name="demo", + package="deerflow-extension-demo", + use=f"{_FIXTURE}:install_ok", + enabled=False, + ) + + assert spec.name == "demo" + assert spec.package == "deerflow-extension-demo" + + def test_successful_install_registers_and_attributes(): spec = ExtensionSpec(use=f"{_FIXTURE}:install_ok") loaded, diagnostics = load_extensions([spec]) @@ -88,6 +112,9 @@ def test_install_failure_rolls_back_partial_registration(): assert sources == {f"{_FIXTURE}:install_ok"} assert len(loaded.middleware_contributors) == 1, "rollback must clear every partial registration" assert loaded.task_lifecycle == (), "rollback must clear partial lifecycle registrations too" + assert loaded.system_model_observers == () + assert loaded.services == () + assert loaded.routers == () def test_rollback_does_not_remove_a_different_specs_registrations_sharing_the_same_use(): diff --git a/backend/tests/test_extension_manager.py b/backend/tests/test_extension_manager.py new file mode 100644 index 000000000..b3fd0bc05 --- /dev/null +++ b/backend/tests/test_extension_manager.py @@ -0,0 +1,2302 @@ +from __future__ import annotations + +import functools +import http.server +import os +import re +import shutil +import subprocess +import threading +import tomllib +import zipfile +from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from pathlib import Path + +import pytest +import yaml + +from deerflow.extensions.cli import find_project_root +from deerflow.extensions.loader import ExtensionSpec +from deerflow.extensions.manager import ( + ExtensionManager, + _controlled_uv_environment, + _detect_extra_flags, + _retry_until_locked, + _validate_locked_local_sources, + _validate_remote_source, +) +from deerflow.tui.cli import main as deerflow_main + + +def _write_local_extension( + source: Path, + *, + with_entry_point: bool = True, + distribution: str = "deerflow-extension-demo", + entry_target: str = "demo_extension:install", +) -> None: + package = source / "demo_extension" + package.mkdir(parents=True) + (package / "__init__.py").write_text( + "def install(registry, config):\n return None\n", + encoding="utf-8", + ) + entry_point = ( + f"""\ +[project.entry-points."deerflow.extensions"] +demo = "{entry_target}" +""" + if with_entry_point + else "" + ) + (source / "pyproject.toml").write_text( + f"""\ +[project] +name = "{distribution}" +version = "1.0.0" +requires-python = ">=3.12" + +{entry_point} + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["demo_extension"] +""", + encoding="utf-8", + ) + + +def _write_host_project(root: Path) -> None: + backend = root / "backend" + backend.mkdir() + (backend / "pyproject.toml").write_text( + """\ +[project] +name = "extension-manager-test-host" +version = "0.0.0" +requires-python = ">=3.12" +dependencies = [] + +[dependency-groups] +extensions = [] + +[tool.uv] +default-groups = ["extensions"] +""", + encoding="utf-8", + ) + (root / "config.yaml").write_text("config_version: 1\n", encoding="utf-8") + + +def _commit_local_extension(source: Path) -> str: + subprocess.run(["git", "init", "-q"], cwd=source, check=True) + subprocess.run(["git", "config", "user.name", "Extension Test"], cwd=source, check=True) + subprocess.run(["git", "config", "user.email", "extension-test@example.com"], cwd=source, check=True) + subprocess.run(["git", "add", "."], cwd=source, check=True) + subprocess.run(["git", "commit", "-qm", "initial extension"], cwd=source, check=True) + return subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=source, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +class _QuietFileHandler(http.server.SimpleHTTPRequestHandler): + def log_message(self, _format: str, *args: object) -> None: + return + + +@contextmanager +def _serve_directory(directory: Path) -> Iterator[str]: + handler = functools.partial(_QuietFileHandler, directory=str(directory)) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, port = server.server_address[:2] + yield f"http://{host}:{port}" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def _assert_demo_entry_point_loads(backend: Path) -> None: + completed = subprocess.run( + [ + str(backend / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python")), + "-c", + "from importlib.metadata import entry_points; eps=entry_points(group='deerflow.extensions'); assert [(e.name, e.value) for e in eps] == [('demo', 'demo_extension:install')]; assert callable(next(iter(eps)).load())", + ], + check=False, + capture_output=True, + text=True, + ) + assert completed.returncode == 0, completed.stderr + + +def _write_demo_wheel(directory: Path) -> Path: + directory.mkdir() + wheel = directory / "deerflow_extension_demo-1.0.0-py3-none-any.whl" + dist_info = "deerflow_extension_demo-1.0.0.dist-info" + records = { + "demo_extension/__init__.py": "def install(registry, config):\n return None\n", + f"{dist_info}/METADATA": ("Metadata-Version: 2.1\nName: deerflow-extension-demo\nVersion: 1.0.0\nRequires-Python: >=3.12\n"), + f"{dist_info}/WHEEL": ("Wheel-Version: 1.0\nGenerator: deerflow-extension-test\nRoot-Is-Purelib: true\nTag: py3-none-any\n"), + f"{dist_info}/entry_points.txt": ("[deerflow.extensions]\ndemo = demo_extension:install\n"), + } + records[f"{dist_info}/RECORD"] = "".join(f"{name},,\n" for name in (*records, f"{dist_info}/RECORD")) + with zipfile.ZipFile(wheel, "w") as archive: + for name, content in records.items(): + archive.writestr(name, content) + return wheel + + +def test_install_local_directory_makes_it_deployable_and_enabled(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + + result = ExtensionManager(root).install(str(source), yes=True) + + assert result.name == "demo" + assert result.distribution == "deerflow-extension-demo" + assert result.use == "demo_extension:install" + + managed_source = root / "backend" / "extensions" / "sources" / "deerflow-extension-demo" + assert (managed_source / "demo_extension" / "__init__.py").is_file() + project = tomllib.loads((root / "backend" / "pyproject.toml").read_text(encoding="utf-8")) + assert project["dependency-groups"]["extensions"] == ["deerflow-extension-demo"] + assert project["tool"]["uv"]["sources"]["deerflow-extension-demo"] == {"path": "extensions/sources/deerflow-extension-demo"} + assert "workspace" not in project["tool"]["uv"] + + config = yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8")) + assert config["plugins"] == [ + { + "name": "demo", + "package": "deerflow-extension-demo", + "use": "demo_extension:install", + "enabled": True, + "required": False, + "config": {}, + } + ] + + _assert_demo_entry_point_loads(root / "backend") + + +def test_install_defaults_to_a_fail_open_plugin_record(tmp_path: Path) -> None: + """A managed install must not silently choose the fail-closed side: with + `required: true`, a later broken extension aborts Gateway startup entirely, + and recovery needs shell access to run `extensions disable`.""" + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + + ExtensionManager(root).install(str(source), yes=True) + + config = yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8")) + assert config["plugins"][0]["required"] is False + + +def test_install_records_required_when_the_operator_opts_in(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + + ExtensionManager(root).install(str(source), yes=True, required=True) + + config = yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8")) + assert config["plugins"][0]["required"] is True + + +def test_cli_install_exposes_the_required_opt_in(tmp_path: Path, monkeypatch, capsys) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + + assert deerflow_main(["extensions", "install", str(source), "--yes", "--required"]) == 0 + + capsys.readouterr() + config = yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8")) + assert config["plugins"][0]["required"] is True + + +def test_contended_lock_waits_instead_of_failing() -> None: + """Windows' blocking lock mode gives up after ~10 seconds, far shorter than + a real `uv add` + `uv sync`, so the manager retries a non-blocking + acquisition rather than turning contention into an error.""" + attempts: list[int] = [] + delays: list[float] = [] + + def _acquire() -> None: + attempts.append(len(attempts)) + if len(attempts) < 3: + raise OSError(13, "Permission denied") + + _retry_until_locked(_acquire, sleep=delays.append) + + assert len(attempts) == 3 + assert len(delays) == 2 + assert all(delay > 0 for delay in delays) + + +def test_mutating_operations_are_serialized_for_one_checkout(tmp_path: Path, monkeypatch) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + first_entered = threading.Event() + release_first = threading.Event() + second_entered = threading.Event() + + def _fake_install(self, source: str, *, yes: bool, required: bool): + if source == "first": + first_entered.set() + assert release_first.wait(timeout=5) + else: + second_entered.set() + return source + + monkeypatch.setattr(ExtensionManager, "_install", _fake_install) + first_manager = ExtensionManager(root) + second_manager = ExtensionManager(root) + + with ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(first_manager.install, "first", yes=True) + assert first_entered.wait(timeout=5) + second = pool.submit(second_manager.install, "second", yes=True) + assert not second_entered.wait(timeout=0.2) + release_first.set() + assert first.result(timeout=5) == "first" + assert second.result(timeout=5) == "second" + + assert second_entered.is_set() + + +def test_deerflow_extensions_install_exposes_the_local_install_flow( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + + exit_code = deerflow_main(["extensions", "install", str(source), "--yes"]) + + assert exit_code == 0 + assert "Installed and enabled demo" in capsys.readouterr().out + config = yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8")) + assert config["plugins"][0]["use"] == "demo_extension:install" + + +def test_hidden_source_env_option_reads_the_install_source_outside_the_shell_recipe( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + monkeypatch.setenv("DEER_FLOW_EXTENSION_SOURCE", str(source)) + + exit_code = deerflow_main( + [ + "extensions", + "install", + "--source-env", + "__deerflow_extension_source__", + "--yes", + ] + ) + + assert exit_code == 0 + config = yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8")) + assert config["plugins"][0]["name"] == "demo" + + +def test_explicit_invalid_project_root_does_not_fall_back_to_current_checkout( + tmp_path: Path, + monkeypatch, +) -> None: + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(tmp_path / "not-a-checkout")) + monkeypatch.chdir(Path(__file__).resolve().parents[2]) + + with pytest.raises(FileNotFoundError, match="DEER_FLOW_PROJECT_ROOT"): + find_project_root() + + +def test_install_git_source_discovers_and_enables_its_packaging_entry_point(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-git-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + revision = _commit_local_extension(source) + bare_repository = tmp_path / "demo.git" + subprocess.run(["git", "clone", "-q", "--bare", str(source), str(bare_repository)], check=True) + subprocess.run(["git", "--git-dir", str(bare_repository), "update-server-info"], check=True) + + with _serve_directory(tmp_path) as base_url: + result = ExtensionManager(root).install(f"git+{base_url}/demo.git@{revision}", yes=True) + + shutil.rmtree(root / "backend" / ".venv") + subprocess.run(["uv", "sync", "--locked"], cwd=root / "backend", check=True) + _assert_demo_entry_point_loads(root / "backend") + + assert result == result.__class__( + name="demo", + distribution="deerflow-extension-demo", + use="demo_extension:install", + ) + assert not (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo").exists() + assert revision in (root / "backend" / "uv.lock").read_text(encoding="utf-8") + config = yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8")) + assert config["plugins"][0]["name"] == "demo" + + +def test_install_rejects_a_pypi_requirement_resolved_from_an_external_local_wheel( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + wheels = tmp_path / "wheels" + root.mkdir() + _write_host_project(root) + _write_demo_wheel(wheels) + monkeypatch.setenv("UV_FIND_LINKS", str(wheels)) + monkeypatch.setenv("UV_NO_INDEX", "1") + pyproject_path = root / "backend" / "pyproject.toml" + config_path = root / "config.yaml" + before = (pyproject_path.read_bytes(), config_path.read_bytes()) + + with pytest.raises(ValueError, match="build context"): + ExtensionManager(root).install("deerflow-extension-demo==1.0.0", yes=True) + + assert (pyproject_path.read_bytes(), config_path.read_bytes()) == before + assert not (root / "backend" / "uv.lock").exists() + + +@pytest.mark.parametrize("relative_wheels", ["wheels", "packages/harness/wheels"]) +def test_install_rejects_a_local_wheel_directory_ignored_by_the_docker_context( + tmp_path: Path, + monkeypatch, + relative_wheels: str, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + wheels = root / "backend" / relative_wheels + wheels.parent.mkdir(parents=True, exist_ok=True) + _write_demo_wheel(wheels) + monkeypatch.setenv("UV_FIND_LINKS", str(wheels)) + monkeypatch.setenv("UV_NO_INDEX", "1") + pyproject_path = root / "backend" / "pyproject.toml" + config_path = root / "config.yaml" + before = (pyproject_path.read_bytes(), config_path.read_bytes()) + + with pytest.raises(ValueError, match="build context"): + ExtensionManager(root).install("deerflow-extension-demo==1.0.0", yes=True) + + assert (pyproject_path.read_bytes(), config_path.read_bytes()) == before + assert not (root / "backend" / "uv.lock").exists() + + +def test_install_rejects_a_relative_find_links_wheelhouse_outside_the_build_context( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + _write_demo_wheel(root / "backend" / "wheelhouse") + # uv resolves a relative UV_FIND_LINKS against its working directory (the + # backend project), so the lock records a relative registry that only + # exists on this host. + monkeypatch.setenv("UV_FIND_LINKS", "wheelhouse") + monkeypatch.setenv("UV_NO_INDEX", "1") + pyproject_path = root / "backend" / "pyproject.toml" + config_path = root / "config.yaml" + before = (pyproject_path.read_bytes(), config_path.read_bytes()) + + with pytest.raises(ValueError, match="build context"): + ExtensionManager(root).install("deerflow-extension-demo==1.0.0", yes=True) + + assert (pyproject_path.read_bytes(), config_path.read_bytes()) == before + assert not (root / "backend" / "uv.lock").exists() + + +def _write_audit_host(backend: Path) -> None: + (backend / "packages" / "harness").mkdir(parents=True) + (backend / "packages" / "extension-api").mkdir(parents=True) + (backend / "extensions" / "sources" / "deerflow-extension-demo").mkdir(parents=True) + (backend / "pyproject.toml").write_text( + '[tool.uv.workspace]\nmembers = ["packages/harness", "packages/extension-api"]\n', + encoding="utf-8", + ) + + +def test_locked_local_source_audit_accepts_deployable_relative_references(tmp_path: Path) -> None: + backend = tmp_path / "backend" + backend.mkdir() + _write_audit_host(backend) + lock_path = backend / "uv.lock" + lock_path.write_text( + """\ +version = 1 +requires-python = ">=3.12" + +[[package]] +name = "host" +version = "0.0.0" +source = { virtual = "." } + +[package.metadata.requires-dev] +extensions = [{ name = "deerflow-extension-demo", directory = "extensions/sources/deerflow-extension-demo" }] + +[[package]] +name = "deerflow-harness" +version = "0.0.0" +source = { editable = "packages/harness" } + +[[package]] +name = "deerflow-extension-api" +version = "0.0.0" +source = { editable = "packages/extension-api" } + +[[package]] +name = "deerflow-extension-demo" +version = "1.0.0" +source = { directory = "extensions/sources/deerflow-extension-demo" } + +[[package]] +name = "git-extension" +version = "1.0.0" +source = { git = "https://github.com/acme/git-extension.git?rev=0123456789012345678901234567890123456789#0123456789012345678901234567890123456789" } + +[[package]] +name = "requests" +version = "2.32.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/requests-2.32.3.tar.gz", hash = "sha256:aaaa" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/requests-2.32.3-py3-none-any.whl", hash = "sha256:bbbb" }, +] +""", + encoding="utf-8", + ) + + _validate_locked_local_sources(lock_path, backend) + + +@pytest.mark.parametrize( + "source_line", + [ + 'source = { registry = "wheels" }', + 'source = { registry = "packages/harness/wheels" }', + 'source = { registry = "/srv/wheels" }', + 'source = { registry = "file:///srv/wheels" }', + 'source = { registry = "C:/srv/wheels" }', + 'source = { path = "vendor/demo.whl" }', + 'source = { directory = "../outside" }', + 'source = { editable = "packages/harness/../extension-api/../../wheels" }', + ], + ids=[ + "relative-wheelhouse", + "wheelhouse-inside-a-workspace-member", + "absolute-wheelhouse", + "file-url-wheelhouse", + "windows-absolute-wheelhouse", + "direct-wheel-path", + "escaping-directory", + "dotdot-through-a-workspace-member", + ], +) +def test_locked_local_source_audit_rejects_non_deployable_local_references( + tmp_path: Path, + source_line: str, +) -> None: + backend = tmp_path / "backend" + backend.mkdir() + _write_audit_host(backend) + lock_path = backend / "uv.lock" + lock_path.write_text( + f"""\ +version = 1 +requires-python = ">=3.12" + +[[package]] +name = "host" +version = "0.0.0" +source = {{ virtual = "." }} + +[[package]] +name = "smuggled" +version = "1.0.0" +{source_line} +""", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="build context"): + _validate_locked_local_sources(lock_path, backend) + + +@pytest.mark.parametrize( + "source_line", + [ + 'source = { registry = "http://127.0.0.1:8000/simple" }', + 'source = { registry = "http://localhost:8000/simple" }', + 'source = { url = "http://[::1]:8000/demo-1.0-py3-none-any.whl" }', + 'source = { git = "http://127.0.0.1:9418/demo.git?rev=0123456789012345678901234567890123456789" }', + ], + ids=[ + "loopback-ipv4-registry", + "localhost-registry", + "loopback-ipv6-wheel-url", + "loopback-git-remote", + ], +) +def test_locked_local_source_audit_warns_about_loopback_references( + tmp_path: Path, + source_line: str, + caplog, +) -> None: + """`_validate_remote_source` allows loopback HTTP for local tooling, so a + loopback URL can legitimately land in the lock — but inside the backend + image build `127.0.0.1` is a different machine. Unlike an environment-driven + wheelhouse resolution, this is an explicit operator choice, so it warns + instead of failing the transaction.""" + backend = tmp_path / "backend" + backend.mkdir() + _write_audit_host(backend) + lock_path = backend / "uv.lock" + lock_path.write_text( + f"""\ +version = 1 +requires-python = ">=3.12" + +[[package]] +name = "host" +version = "0.0.0" +source = {{ virtual = "." }} + +[[package]] +name = "smuggled" +version = "1.0.0" +{source_line} +""", + encoding="utf-8", + ) + + with caplog.at_level("WARNING", logger="deerflow.extensions.manager"): + _validate_locked_local_sources(lock_path, backend) + + assert "loopback" in caplog.text + + +def test_locked_local_source_audit_allows_a_private_network_index(tmp_path: Path, caplog) -> None: + """A builder on the same network can reach a private index host, so only + loopback is rejected — blocking RFC1918 would break internal mirrors.""" + backend = tmp_path / "backend" + backend.mkdir() + _write_audit_host(backend) + lock_path = backend / "uv.lock" + lock_path.write_text( + """\ +version = 1 +requires-python = ">=3.12" + +[[package]] +name = "host" +version = "0.0.0" +source = { virtual = "." } + +[[package]] +name = "internal" +version = "1.0.0" +source = { registry = "https://10.0.0.5/simple" } +wheels = [ + { url = "https://10.0.0.5/packages/internal-1.0.0-py3-none-any.whl", hash = "sha256:bbbb" }, +] +""", + encoding="utf-8", + ) + + with caplog.at_level("WARNING", logger="deerflow.extensions.manager"): + _validate_locked_local_sources(lock_path, backend) + + assert caplog.text == "" + + +def test_locked_local_source_audit_rejects_absolute_paths_even_inside_allowed_roots(tmp_path: Path) -> None: + backend = tmp_path / "backend" + backend.mkdir() + _write_audit_host(backend) + lock_path = backend / "uv.lock" + workspace_member = backend.resolve() / "packages" / "harness" + lock_path.write_text( + f"""\ +version = 1 +requires-python = ">=3.12" + +[[package]] +name = "host" +version = "0.0.0" +source = {{ virtual = "." }} + +[[package]] +name = "deerflow-harness" +version = "0.0.0" +source = {{ editable = "{workspace_member.as_posix()}" }} +""", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="build context"): + _validate_locked_local_sources(lock_path, backend) + + +def test_file_urls_are_rejected_because_they_cannot_enter_the_docker_build_context(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + + with pytest.raises(ValueError, match="local directory"): + ExtensionManager(root).install("git+file:///outside/demo.git@deadbeef", yes=True) + + +def test_install_rolls_back_when_the_declared_entry_point_cannot_be_imported(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source, entry_target="missing_demo_extension:install") + pyproject = root / "backend" / "pyproject.toml" + original = pyproject.read_bytes() + + with pytest.raises(ValueError, match="could not be loaded"): + ExtensionManager(root).install(str(source), yes=True) + + assert pyproject.read_bytes() == original + assert not (root / "backend" / "uv.lock").exists() + assert not (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo").exists() + assert yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8")).get("plugins") is None + + +@pytest.mark.parametrize( + "source", + [ + "deerflow-extension-demo @ ../outside", + "../outside/deerflow-extension-demo", + "deerflow-extension-demo @ /outside/demo.whl", + ], +) +def test_relative_or_absolute_direct_paths_must_use_the_managed_directory_snapshot( + tmp_path: Path, + source: str, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + pyproject = root / "backend" / "pyproject.toml" + original = pyproject.read_bytes() + + with pytest.raises(ValueError, match="local directory"): + ExtensionManager(root).install(source, yes=True) + + assert pyproject.read_bytes() == original + assert not (root / "backend" / "uv.lock").exists() + + +def test_install_preserves_unrelated_config_comments_and_layout(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + config_path = root / "config.yaml" + config_path.write_text( + """\ +# operator notes must survive extension management +config_version: 1 + +models: + # keep the carefully documented model + - name: demo-model + use: provider:model + +database: + url: sqlite:///data.db +""", + encoding="utf-8", + ) + + ExtensionManager(root).install(str(source), yes=True) + + updated = config_path.read_text(encoding="utf-8") + assert "# operator notes must survive extension management" in updated + assert " # keep the carefully documented model" in updated + assert "models:\n # keep the carefully documented model\n - name: demo-model\n use: provider:model" in updated + assert "database:\n url: sqlite:///data.db" in updated + + +def test_toggle_preserves_the_next_section_header_and_crlf_style(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + config_path = root / "config.yaml" + config_path.write_bytes( + b"config_version: 1\r\n" + b"plugins:\r\n" + b" - name: demo\r\n" + b" package: deerflow-extension-demo\r\n" + b" use: demo_extension:install\r\n" + b" enabled: true\r\n" + b" config:\r\n" + b" label: 'keep value'\r\n" + b"\r\n" + b"# Database settings must stay with the next section.\r\n" + b"database:\r\n" + b" url: sqlite:///data.db\r\n" + ) + + ExtensionManager(root).set_enabled("demo", enabled=False) + + updated = config_path.read_bytes() + assert b"# Database settings must stay with the next section.\r\n" in updated + assert b"database:\r\n url: sqlite:///data.db\r\n" in updated + assert b"\n" not in updated.replace(b"\r\n", b"") + parsed = yaml.safe_load(updated) + assert parsed["plugins"][0]["config"] == {"label": "keep value"} + assert parsed["plugins"][0]["enabled"] is False + + +def test_deerflow_extensions_disable_keeps_the_plugin_configuration( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + config_path = root / "config.yaml" + config_path.write_text( + """\ +# keep me +config_version: 1 +plugins: + - name: demo + package: deerflow-extension-demo + use: demo_extension:install + enabled: true + required: true + config: + label: production +""", + encoding="utf-8", + ) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + + exit_code = deerflow_main(["extensions", "disable", "demo"]) + + assert exit_code == 0 + assert "Disabled demo" in capsys.readouterr().out + updated = yaml.safe_load(config_path.read_text(encoding="utf-8")) + assert updated["plugins"] == [ + { + "name": "demo", + "package": "deerflow-extension-demo", + "use": "demo_extension:install", + "enabled": False, + "required": True, + "config": {"label": "production"}, + } + ] + assert "# keep me" in config_path.read_text(encoding="utf-8") + + +def test_hidden_name_env_option_reads_the_extension_name_outside_the_shell_recipe( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + config_path = root / "config.yaml" + config_path.write_text( + "plugins:\n - name: demo\n package: deerflow-extension-demo\n use: demo_extension:install\n enabled: true\n", + encoding="utf-8", + ) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + monkeypatch.setenv("DEER_FLOW_EXTENSION_NAME", "demo") + + exit_code = deerflow_main( + [ + "extensions", + "disable", + "--name-env", + "__deerflow_extension_name__", + ] + ) + + assert exit_code == 0 + assert yaml.safe_load(config_path.read_text(encoding="utf-8"))["plugins"][0]["enabled"] is False + + +def test_deerflow_extensions_enable_reactivates_a_configured_plugin( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + config_path = root / "config.yaml" + config_path.write_text( + """\ +config_version: 1 +plugins: + - name: demo + package: deerflow-extension-demo + use: demo_extension:install + enabled: false + required: true + config: {} +""", + encoding="utf-8", + ) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + + exit_code = deerflow_main(["extensions", "enable", "demo"]) + + assert exit_code == 0 + assert "Enabled demo" in capsys.readouterr().out + updated = yaml.safe_load(config_path.read_text(encoding="utf-8")) + assert updated["plugins"][0]["enabled"] is True + + +def test_distribution_identifier_uses_pep_503_normalization(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + config_path = root / "config.yaml" + config_path.write_text( + "plugins:\n - name: demo\n package: DeerFlow_Extension.Demo\n use: demo_extension:install\n enabled: true\n", + encoding="utf-8", + ) + + ExtensionManager(root).set_enabled("deerflow-extension-demo", enabled=False) + + assert yaml.safe_load(config_path.read_text(encoding="utf-8"))["plugins"][0]["enabled"] is False + + +def test_deerflow_extensions_list_reports_activation_and_package( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + (root / "config.yaml").write_text( + """\ +config_version: 1 +plugins: + - name: demo + package: deerflow-extension-demo + use: demo_extension:install + enabled: true + required: true + config: {} +""", + encoding="utf-8", + ) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + + exit_code = deerflow_main(["extensions", "list"]) + + assert exit_code == 0 + output = capsys.readouterr().out + assert "demo" in output + assert "enabled" in output + assert "deerflow-extension-demo" in output + assert "demo_extension:install" in output + + +def test_cli_reports_invalid_config_without_a_traceback( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + (root / "config.yaml").write_text("plugins: [\n", encoding="utf-8") + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + + exit_code = deerflow_main(["extensions", "list"]) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "invalid DeerFlow config YAML" in captured.err + assert "Traceback" not in captured.err + + +@pytest.mark.parametrize( + "malformed_plugin", + [42, {"name": "missing-use"}], + ids=["non-mapping", "missing-use"], +) +def test_deerflow_extensions_list_rejects_entries_the_runtime_schema_rejects( + tmp_path: Path, + monkeypatch, + capsys, + malformed_plugin: object, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + (root / "config.yaml").write_text( + yaml.safe_dump({"config_version": 1, "plugins": [malformed_plugin]}, sort_keys=False), + encoding="utf-8", + ) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + + exit_code = deerflow_main(["extensions", "list"]) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "NAME\tSTATE\tPACKAGE\tENTRY POINT" not in captured.out + assert "extension command failed:" in captured.err + assert "Traceback" not in captured.err + + +def test_deerflow_extensions_remove_uninstalls_dependency_source_and_activation( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + ExtensionManager(root).install(str(source), yes=True) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + + exit_code = deerflow_main(["extensions", "remove", "demo"]) + + assert exit_code == 0 + output = capsys.readouterr().out + assert "Removed demo" in output + assert "Restart DeerFlow" in output + config = yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8")) + assert config["plugins"] == [] + assert not (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo").exists() + pyproject = (root / "backend" / "pyproject.toml").read_text(encoding="utf-8") + assert "deerflow-extension-demo" not in pyproject + + +def test_remove_one_configured_instance_keeps_its_shared_distribution_runnable( + tmp_path: Path, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + manager = ExtensionManager(root) + manager.install(str(source), yes=True) + config_path = root / "config.yaml" + installed = yaml.safe_load(config_path.read_text(encoding="utf-8"))["plugins"][0] + first = {**installed, "name": "first", "config": {"instance": 1}} + second = { + **installed, + "name": "second", + "package": "DeerFlow_Extension.Demo", + "config": {"instance": 2}, + } + config_path.write_text( + yaml.safe_dump({"config_version": 1, "plugins": [first, second]}, sort_keys=False), + encoding="utf-8", + ) + pyproject_path = root / "backend" / "pyproject.toml" + lock_path = root / "backend" / "uv.lock" + dependency_files_before = (pyproject_path.read_bytes(), lock_path.read_bytes()) + + removed = manager.remove("first") + + assert removed == "first" + assert yaml.safe_load(config_path.read_text(encoding="utf-8"))["plugins"] == [second] + assert (pyproject_path.read_bytes(), lock_path.read_bytes()) == dependency_files_before + assert (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo").is_dir() + _assert_demo_entry_point_loads(root / "backend") + + +def test_install_prompts_for_trust_when_yes_is_not_supplied( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + monkeypatch.setattr("builtins.input", lambda _prompt: "yes") + + exit_code = deerflow_main(["extensions", "install", str(source)]) + + assert exit_code == 0 + assert "executes code with Gateway privileges" in capsys.readouterr().out + + +def test_failed_entry_point_discovery_rolls_back_dependency_and_lock(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "broken-git-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source, with_entry_point=False) + revision = _commit_local_extension(source) + bare_repository = tmp_path / "broken.git" + subprocess.run(["git", "clone", "-q", "--bare", str(source), str(bare_repository)], check=True) + subprocess.run(["git", "--git-dir", str(bare_repository), "update-server-info"], check=True) + pyproject_path = root / "backend" / "pyproject.toml" + config_path = root / "config.yaml" + original_pyproject = pyproject_path.read_bytes() + original_config = config_path.read_bytes() + + with _serve_directory(tmp_path) as base_url: + with pytest.raises(ValueError, match="exactly one"): + ExtensionManager(root).install(f"git+{base_url}/broken.git@{revision}", yes=True) + + assert pyproject_path.read_bytes() == original_pyproject + assert config_path.read_bytes() == original_config + assert not (root / "backend" / "uv.lock").exists() + absent = subprocess.run( + [ + str(root / "backend" / ".venv" / "bin" / "python"), + "-c", + "from importlib.metadata import PackageNotFoundError, version; \ntry: version('deerflow-extension-demo')\nexcept PackageNotFoundError: raise SystemExit(0)\nraise SystemExit(1)", + ], + check=False, + ) + assert absent.returncode == 0 + + +def test_failed_install_does_not_overwrite_a_concurrent_operator_config_edit( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + config_path = root / "config.yaml" + operator_edit = "config_version: 1\nlog_level: debug # edited during install\n" + from deerflow.extensions import manager as manager_module + + original_sync = manager_module._sync_environment + calls = 0 + + def _fail_after_operator_edit(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 1: + config_path.write_text(operator_edit, encoding="utf-8") + raise RuntimeError("simulated dependency sync failure") + return original_sync(*args, **kwargs) + + monkeypatch.setattr("deerflow.extensions.manager._sync_environment", _fail_after_operator_edit) + + with pytest.raises(RuntimeError, match="sync failure"): + ExtensionManager(root).install(str(source), yes=True) + + assert config_path.read_text(encoding="utf-8") == operator_edit + + +def test_failed_install_preserves_a_concurrent_dependency_file_edit( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + pyproject_path = root / "backend" / "pyproject.toml" + + def _fail_after_operator_edit(*_args, **_kwargs): + pyproject_path.write_text( + pyproject_path.read_text(encoding="utf-8") + "\n# operator edit during install\n", + encoding="utf-8", + ) + raise RuntimeError("simulated dependency sync failure") + + monkeypatch.setattr("deerflow.extensions.manager._sync_environment", _fail_after_operator_edit) + + with pytest.raises(RuntimeError, match="recovery.*dependency"): + ExtensionManager(root).install(str(source), yes=True) + + assert "# operator edit during install" in pyproject_path.read_text(encoding="utf-8") + assert (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo").is_dir() + + +def test_uv_add_partial_writes_are_rolled_back_when_the_command_fails( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + pyproject_path = root / "backend" / "pyproject.toml" + lock_path = root / "backend" / "uv.lock" + config_path = root / "config.yaml" + before = (pyproject_path.read_bytes(), config_path.read_bytes()) + uv_commands: list[str] = [] + + def _partially_write_then_fail(command, _backend_dir): + uv_commands.append(command[1]) + if command[1] == "add": + pyproject_path.write_text( + pyproject_path.read_text(encoding="utf-8") + "\n# partial uv add write\n", + encoding="utf-8", + ) + lock_path.write_text("partial uv lock write\n", encoding="utf-8") + raise subprocess.CalledProcessError(1, command) + + monkeypatch.setattr("deerflow.extensions.manager._run_uv", _partially_write_then_fail) + + with pytest.raises(subprocess.CalledProcessError): + ExtensionManager(root).install(str(source), yes=True) + + assert (pyproject_path.read_bytes(), config_path.read_bytes()) == before + assert not lock_path.exists() + assert not (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo").exists() + assert uv_commands == ["add", "sync"] + + +def test_local_install_rejects_symlinks_before_copying_or_resolving(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + outside = tmp_path / "operator-secret.txt" + outside.write_text("do not vendor me", encoding="utf-8") + (source / "linked-secret.txt").symlink_to(outside) + original_pyproject = (root / "backend" / "pyproject.toml").read_bytes() + + with pytest.raises(ValueError, match="symbolic links"): + ExtensionManager(root).install(str(source), yes=True) + + assert (root / "backend" / "pyproject.toml").read_bytes() == original_pyproject + assert not (root / "backend" / "extensions").exists() + + +@pytest.mark.skipif(os.name == "nt", reason="named pipes are POSIX-specific") +def test_local_install_rejects_special_files_before_snapshotting(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + os.mkfifo(source / "runtime.pipe") + + with pytest.raises(ValueError, match="regular files"): + ExtensionManager(root).install(str(source), yes=True) + + assert not (root / "backend" / "extensions").exists() + + +@pytest.mark.parametrize( + "secret_name", + [".env.local", "deploy.pem", "credentials.json", ".npmrc", ".pypirc"], +) +def test_local_install_rejects_likely_secret_files( + tmp_path: Path, + secret_name: str, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + (source / secret_name).write_text("credential", encoding="utf-8") + + with pytest.raises(ValueError, match="sensitive file"): + ExtensionManager(root).install(str(source), yes=True) + + assert not (root / "backend" / "extensions").exists() + + +@pytest.mark.parametrize("distribution", ["../outside", "/tmp/outside", "C:/outside"]) +def test_local_install_rejects_distribution_names_that_escape_the_managed_root( + tmp_path: Path, + distribution: str, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source, distribution=distribution) + + with pytest.raises(ValueError, match="distribution name"): + ExtensionManager(root).install(str(source), yes=True) + + assert not (root / "backend" / "extensions").exists() + + +def test_install_adopts_an_existing_manual_plugin_instead_of_loading_it_twice(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + config_path = root / "config.yaml" + config_path.write_text( + """\ +config_version: 1 +plugins: + - use: demo_extension:install + required: false + config: + label: keep-this +""", + encoding="utf-8", + ) + + ExtensionManager(root).install(str(source), yes=True) + + plugins = yaml.safe_load(config_path.read_text(encoding="utf-8"))["plugins"] + assert plugins == [ + { + "use": "demo_extension:install", + "required": False, + "config": {"label": "keep-this"}, + "name": "demo", + "package": "deerflow-extension-demo", + "enabled": True, + } + ] + + +@pytest.mark.parametrize( + "configured_plugin", + [ + { + "name": "demo", + "use": "other_extension:install", + "config": {"keep": True}, + }, + { + "package": "deerflow-extension-demo", + "use": "other_extension:install", + "config": {"keep": True}, + }, + { + "package": "deerflow_extension.demo", + "use": "other_extension:install", + "config": {"keep": True}, + }, + ], +) +def test_install_rejects_identity_collisions_with_a_different_entry_point( + tmp_path: Path, + configured_plugin: dict[str, object], +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + config_path = root / "config.yaml" + original = yaml.safe_dump( + {"config_version": 1, "plugins": [configured_plugin]}, + sort_keys=False, + ) + config_path.write_text(original, encoding="utf-8") + + with pytest.raises(ValueError, match="conflict"): + ExtensionManager(root).install(str(source), yes=True) + + assert config_path.read_text(encoding="utf-8") == original + assert not (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo").exists() + assert "deerflow-extension-demo" not in (root / "backend" / "pyproject.toml").read_text(encoding="utf-8") + + +def test_install_replaces_inline_empty_plugins_with_one_schema_valid_block(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + config_path = root / "config.yaml" + config_path.write_text( + "config_version: 1\nplugins: [] # managed plugins\nlog_level: info\n", + encoding="utf-8", + ) + + ExtensionManager(root).install(str(source), yes=True) + + updated = config_path.read_text(encoding="utf-8") + assert updated.count("plugins:") == 1 + config = yaml.safe_load(updated) + assert config["log_level"] == "info" + parsed = ExtensionSpec.model_validate(config["plugins"][0]) + assert parsed.name == "demo" + assert parsed.package == "deerflow-extension-demo" + assert parsed.enabled is True + + +@pytest.mark.parametrize( + "plugins_key", + ["plugins", '"plugins"', "'plugins'"], +) +def test_disable_replaces_nonempty_flow_style_plugins_without_duplicate_key( + tmp_path: Path, + plugins_key: str, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + config_path = root / "config.yaml" + config_path.write_text( + f'{plugins_key}: [{{name: demo, package: deerflow-extension-demo, use: "demo_extension:install", enabled: true}}]\nlog_level: info\n', + encoding="utf-8", + ) + + ExtensionManager(root).set_enabled("demo", enabled=False) + + updated = config_path.read_text(encoding="utf-8") + assert len(re.findall(r"(?m)^(?:plugins|[\'\"]plugins[\'\"])[ \t]*:", updated)) == 1 + config = yaml.safe_load(updated) + assert config["plugins"][0]["enabled"] is False + assert config["log_level"] == "info" + + +def test_toggle_rejects_duplicate_top_level_plugins_keys_without_mutating_config( + tmp_path: Path, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + config_path = root / "config.yaml" + original = """\ +plugins: [] +log_level: info +"plugins": + - name: demo + package: deerflow-extension-demo + use: demo_extension:install + enabled: true +""" + config_path.write_text(original, encoding="utf-8") + + with pytest.raises(ValueError, match="duplicate top-level plugins"): + ExtensionManager(root).set_enabled("demo", enabled=False) + + assert config_path.read_text(encoding="utf-8") == original + + +@pytest.mark.parametrize("next_key", ["log_level", '"log_level"', "'log_level'"]) +def test_plugins_rewrite_preserves_the_next_quoted_or_plain_top_level_section( + tmp_path: Path, + next_key: str, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + config_path = root / "config.yaml" + config_path.write_text( + f'plugins: [{{name: demo, use: "demo_extension:install", enabled: true}}]\n{next_key}: info\n', + encoding="utf-8", + ) + + ExtensionManager(root).set_enabled("demo", enabled=False) + + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + assert config["plugins"][0]["enabled"] is False + assert config["log_level"] == "info" + + +@pytest.mark.parametrize("next_key", ["my.key", "2fa", "$schema", "日本", "my key"]) +def test_plugins_rewrite_preserves_a_following_section_with_an_unconventional_key( + tmp_path: Path, + next_key: str, +) -> None: + """`AppConfig` allows extra top-level keys, so the managed rewrite must not + assume the next section is named like a Python identifier.""" + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + config_path = root / "config.yaml" + config_path.write_text( + f'plugins: [{{name: demo, use: "demo_extension:install", enabled: true}}]\n{next_key}:\n nested: keep-me\n', + encoding="utf-8", + ) + + ExtensionManager(root).set_enabled("demo", enabled=False) + + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + assert config["plugins"][0]["enabled"] is False + assert config[next_key] == {"nested": "keep-me"} + + +def test_plugins_rewrite_preserves_trailing_content_below_a_final_plugins_block(tmp_path: Path) -> None: + """The manager appends `plugins:` at end of file, so the steady-state shape + has no following key; trailing operator notes still must survive a toggle.""" + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + config_path = root / "config.yaml" + config_path.write_text( + 'log_level: info\nplugins: [{name: demo, use: "demo_extension:install", enabled: true}]\n\n# operator note kept below the managed block\n', + encoding="utf-8", + ) + + ExtensionManager(root).set_enabled("demo", enabled=False) + + updated = config_path.read_text(encoding="utf-8") + assert "# operator note kept below the managed block" in updated + config = yaml.safe_load(updated) + assert config["plugins"][0]["enabled"] is False + assert config["log_level"] == "info" + + +def test_null_plugins_is_treated_as_the_runtime_default_and_can_be_managed(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + config_path = root / "config.yaml" + config_path.write_text("plugins: # no extensions yet\nlog_level: info\n", encoding="utf-8") + manager = ExtensionManager(root) + + assert manager.list_configured() == () + + +def test_list_uses_the_same_boolean_coercion_as_the_runtime_loader(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + (root / "config.yaml").write_text( + """\ +plugins: + - name: numeric + package: deerflow-extension-numeric + use: numeric_extension:install + enabled: 0 + required: 1 + - name: yaml-booleans + package: deerflow-extension-yaml-booleans + use: yaml_boolean_extension:install + enabled: yes + required: no +""", + encoding="utf-8", + ) + + configured = ExtensionManager(root).list_configured() + + assert [(item.enabled, item.required) for item in configured] == [ + (False, True), + (True, False), + ] + + +def test_cli_install_updates_the_runtime_selected_config_file( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + runtime_config = tmp_path / "deployment.yaml" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + root_config = root / "config.yaml" + original_root_config = root_config.read_bytes() + runtime_config.write_text("config_version: 1\n", encoding="utf-8") + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + monkeypatch.setenv("DEER_FLOW_CONFIG_PATH", str(runtime_config)) + + assert deerflow_main(["extensions", "install", str(source), "--yes"]) == 0 + + assert root_config.read_bytes() == original_root_config + runtime = yaml.safe_load(runtime_config.read_text(encoding="utf-8")) + assert runtime["plugins"][0]["name"] == "demo" + + +def test_manager_falls_back_to_the_legacy_backend_config_path(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + (root / "config.yaml").unlink() + backend_config = root / "backend" / "config.yaml" + backend_config.write_text("config_version: 1\nplugins: []\n", encoding="utf-8") + + assert ExtensionManager(root).list_configured() == () + + +def test_remove_rolls_back_package_lock_config_source_and_environment_when_config_write_fails( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + manager = ExtensionManager(root) + manager.install(str(source), yes=True) + pyproject_path = root / "backend" / "pyproject.toml" + lock_path = root / "backend" / "uv.lock" + config_path = root / "config.yaml" + managed_source = root / "backend" / "extensions" / "sources" / "deerflow-extension-demo" + before = ( + pyproject_path.read_bytes(), + lock_path.read_bytes(), + config_path.read_bytes(), + ) + + def _fail_replace(_source, _target): + raise OSError("simulated config replacement failure") + + monkeypatch.setattr("deerflow.extensions.manager.os.replace", _fail_replace) + + with pytest.raises(OSError, match="replacement failure"): + manager.remove("demo") + + assert (pyproject_path.read_bytes(), lock_path.read_bytes(), config_path.read_bytes()) == before + assert managed_source.is_dir() + present = subprocess.run( + [ + str(root / "backend" / ".venv" / "bin" / "python"), + "-c", + "from importlib.metadata import version; assert version('deerflow-extension-demo') == '1.0.0'", + ], + check=False, + ) + assert present.returncode == 0 + + +def test_failed_remove_preserves_a_concurrent_operator_config_edit( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + manager = ExtensionManager(root) + manager.install(str(source), yes=True) + config_path = root / "config.yaml" + operator_edit = "config_version: 1\nplugins: []\nlog_level: debug # edited during remove\n" + from deerflow.extensions import manager as manager_module + + original_sync = manager_module._sync_environment + calls = 0 + + def _fail_after_operator_edit(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 1: + config_path.write_text(operator_edit, encoding="utf-8") + raise RuntimeError("simulated dependency sync failure") + return original_sync(*args, **kwargs) + + monkeypatch.setattr("deerflow.extensions.manager._sync_environment", _fail_after_operator_edit) + + with pytest.raises( + RuntimeError, + match="recovery.*config", + ): + manager.remove("demo") + + assert config_path.read_text(encoding="utf-8") == operator_edit + assert (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo").is_dir() + + +def test_failed_remove_preserves_a_concurrent_dependency_file_edit( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + manager = ExtensionManager(root) + manager.install(str(source), yes=True) + pyproject_path = root / "backend" / "pyproject.toml" + + def _fail_after_operator_edit(*_args, **_kwargs): + pyproject_path.write_text( + pyproject_path.read_text(encoding="utf-8") + "\n# operator edit during remove\n", + encoding="utf-8", + ) + raise RuntimeError("simulated dependency sync failure") + + monkeypatch.setattr("deerflow.extensions.manager._sync_environment", _fail_after_operator_edit) + + with pytest.raises(RuntimeError, match="recovery.*dependency"): + manager.remove("demo") + + assert "# operator edit during remove" in pyproject_path.read_text(encoding="utf-8") + assert (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo").is_dir() + assert yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8"))["plugins"] == [] + + +def test_uv_remove_partial_writes_are_rolled_back_when_the_command_fails( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + manager = ExtensionManager(root) + manager.install(str(source), yes=True) + pyproject_path = root / "backend" / "pyproject.toml" + lock_path = root / "backend" / "uv.lock" + config_path = root / "config.yaml" + managed_source = root / "backend" / "extensions" / "sources" / "deerflow-extension-demo" + before = (pyproject_path.read_bytes(), lock_path.read_bytes(), config_path.read_bytes()) + uv_commands: list[str] = [] + + def _partially_write_then_fail(command, _backend_dir): + uv_commands.append(command[1]) + if command[1] == "remove": + pyproject_path.write_text( + pyproject_path.read_text(encoding="utf-8") + "\n# partial uv remove write\n", + encoding="utf-8", + ) + lock_path.write_text( + lock_path.read_text(encoding="utf-8") + "\n# partial uv remove lock write\n", + encoding="utf-8", + ) + raise subprocess.CalledProcessError(1, command) + + monkeypatch.setattr("deerflow.extensions.manager._run_uv", _partially_write_then_fail) + + with pytest.raises(subprocess.CalledProcessError): + manager.remove("demo") + + assert (pyproject_path.read_bytes(), lock_path.read_bytes(), config_path.read_bytes()) == before + assert managed_source.is_dir() + assert uv_commands == ["remove", "sync"] + + +def test_cli_reports_uv_install_failure_without_traceback_or_partial_state( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + pyproject_path = root / "backend" / "pyproject.toml" + config_path = root / "config.yaml" + original = (pyproject_path.read_bytes(), config_path.read_bytes()) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + + exit_code = deerflow_main(["extensions", "install", "not a valid @ requirement @@", "--yes"]) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "extension command failed" in captured.err + assert "Traceback" not in captured.err + assert (pyproject_path.read_bytes(), config_path.read_bytes()) == original + + +@pytest.mark.parametrize( + "source", + [ + "git+https://token@example.com/acme/demo.git@0123456789012345678901234567890123456789", + "https://user:password@example.com/demo.whl", + "deerflow-extension-demo @ https://user:password@example.com/demo.whl", + ], +) +def test_remote_sources_with_embedded_credentials_are_rejected_before_uv( + tmp_path: Path, + source: str, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + original = (root / "backend" / "pyproject.toml").read_bytes() + + with pytest.raises(ValueError, match="credentials"): + ExtensionManager(root).install(source, yes=True) + + assert (root / "backend" / "pyproject.toml").read_bytes() == original + + +@pytest.mark.parametrize( + "source", + [ + "https://packages.example/demo.whl?token=do-not-store", + "demo @ https://packages.example/demo.whl?X-Amz-Signature=do-not-store", + "https://packages.example/demo.whl#api_key=do-not-store", + "https://packages.example/demo.whl?authToken=do-not-store", + "https://packages.example/demo.whl?clientSecret=do-not-store", + "https://packages.example/demo.whl?AWSAccessKeyId=do-not-store", + "https://packages.example/demo.whl?Authorization=Bearer-do-not-store", + "https://packages.example/demo.whl?X-Authorization=Bearer-do-not-store", + ], +) +def test_remote_sources_with_secret_query_parameters_are_rejected(source: str) -> None: + with pytest.raises(ValueError, match="credential"): + _validate_remote_source(source) + + +@pytest.mark.parametrize( + "source", + [ + "https://packages.example/demo.whl?accesstoken=do-not-store", + "https://packages.example/demo.whl?ACCESSTOKEN=do-not-store", + "https://packages.example/demo.whl?apikey=do-not-store", + "https://packages.example/demo.whl?key=do-not-store", + "https://packages.example/demo.whl?pw=do-not-store", + "https://packages.example/demo.whl?sas=do-not-store", + "https://packages.example/demo.whl?code=do-not-store", + ], +) +def test_run_together_secret_query_parameters_are_rejected(source: str) -> None: + """The camel-case splitter only fires on case transitions, so run-together + and all-caps spellings need to be recognized directly.""" + with pytest.raises(ValueError, match="credential"): + _validate_remote_source(source) + + +@pytest.mark.parametrize( + "source", + [ + "git+https://github.com/acme/demo.git@main#subdirectory=packages/demo", + "https://packages.example/demo.whl?keyword=demo", + "https://packages.example/demo.whl?monkeypatch=1", + "https://packages.example/demo.whl?rev=0123456789", + ], +) +def test_benign_query_parameters_remain_installable(source: str) -> None: + _validate_remote_source(source) + + +@pytest.mark.parametrize( + "source", + [ + "git+ssh://git@github.com/acme/deerflow-extension-demo.git@main", + "deerflow-extension-demo @ git+ssh://git@github.com/acme/deerflow-extension-demo.git@main", + "ssh://git@github.com/acme/deerflow-extension-demo.git@main", + ], +) +def test_remote_git_ssh_sources_are_rejected_before_uv( + tmp_path: Path, + source: str, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + pyproject_path = root / "backend" / "pyproject.toml" + original = pyproject_path.read_bytes() + + with pytest.raises(ValueError, match="public HTTPS"): + ExtensionManager(root).install(source, yes=True) + + assert pyproject_path.read_bytes() == original + assert not (root / "backend" / "uv.lock").exists() + + +@pytest.mark.parametrize( + "source", + [ + "git@github.com:acme/deerflow-extension-demo.git", + "git+git@github.com:acme/deerflow-extension-demo.git", + "deerflow-extension-demo @ git+git@github.com:acme/deerflow-extension-demo.git", + "deploy@internal.example:acme/deerflow-extension-demo.git", + ], +) +def test_git_ssh_shorthand_points_at_the_https_correction(source: str) -> None: + """SCP-like shorthand carries no scheme, so it reaches validation looking + like a bare path. The operator asked for a remote source, so the actionable + correction is the HTTPS spelling, not a local directory snapshot.""" + with pytest.raises(ValueError, match="public HTTPS") as excinfo: + _validate_remote_source(source) + + message = str(excinfo.value) + assert "git+https://" in message + assert "snapshot" not in message + + +def test_cli_rejects_git_ssh_without_traceback_or_partial_state( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + pyproject_path = root / "backend" / "pyproject.toml" + original = pyproject_path.read_bytes() + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + + exit_code = deerflow_main( + [ + "extensions", + "install", + "git+ssh://git@github.com/acme/deerflow-extension-demo.git@main", + "--yes", + ] + ) + + captured = capsys.readouterr() + assert exit_code == 1 + assert "public HTTPS" in captured.err + assert "Traceback" not in captured.err + assert pyproject_path.read_bytes() == original + assert not (root / "backend" / "uv.lock").exists() + + +@pytest.mark.parametrize( + "source", + [ + "git+https://github.com/acme/deerflow-extension-demo.git@0123456789012345678901234567890123456789", + "deerflow-extension-demo @ git+https://github.com/acme/deerflow-extension-demo.git@0123456789012345678901234567890123456789", + ], +) +def test_public_git_https_sources_remain_allowed(source: str) -> None: + _validate_remote_source(source) + + +@pytest.mark.parametrize( + "source", + [ + "http://packages.example/demo.whl", + "git+git://github.com/acme/deerflow-extension-demo.git@main", + "ftp://packages.example/demo.whl", + ], +) +def test_remote_sources_require_https(source: str) -> None: + with pytest.raises(ValueError, match="HTTPS"): + _validate_remote_source(source) + + +def test_cli_never_echoes_rejected_source_credentials( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + source = "https://operator:super-secret@example.com/extension.whl" + + assert deerflow_main(["extensions", "install", source, "--yes"]) == 1 + + output = capsys.readouterr() + assert "super-secret" not in output.out + assert "super-secret" not in output.err + assert "embedded credentials" in output.err + + +def test_install_uses_one_controlled_uv_project_and_deferred_sync( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + for key in ("UV_PROJECT", "UV_WORKING_DIR", "UV_NO_SYNC", "UV_FROZEN", "UV_LOCKED"): + monkeypatch.setenv(key, "must-not-reach-uv") + monkeypatch.setenv("UV_INDEX_URL", "https://packages.example/simple") + monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example:8080") + uv_calls: list[tuple[list[str], dict[str, object]]] = [] + + def _record_run(command, **kwargs): + if command[:2] == ["uv", "--version"]: + return subprocess.CompletedProcess(command, 0, stdout="uv 0.11.1\n") + if command[0] == "uv": + uv_calls.append((list(command), kwargs)) + if command[1] == "add": + (root / "backend" / "uv.lock").write_text('version = 1\nrequires-python = ">=3.12"\n', encoding="utf-8") + return subprocess.CompletedProcess(command, 0) + return subprocess.CompletedProcess(command, 0, stdout='[["demo", "demo_extension:install"]]\n') + + monkeypatch.setattr("deerflow.extensions.manager.subprocess.run", _record_run) + + ExtensionManager(root).install(str(source), yes=True) + + assert [command[1] for command, _ in uv_calls] == ["add", "sync"] + backend = str(root / "backend") + add, sync = (uv_calls[0][0], uv_calls[1][0]) + assert ["--project", backend] == add[add.index("--project") : add.index("--project") + 2] + assert "--no-sync" in add + assert "--no-workspace" in add + assert add[-2:] == ["--", "extensions/sources/deerflow-extension-demo"] + assert ["--project", backend] == sync[sync.index("--project") : sync.index("--project") + 2] + assert "--locked" in sync + assert "--no-sync" not in sync + for _, kwargs in uv_calls: + child_env = kwargs["env"] + assert isinstance(child_env, dict) + assert not { + "UV_PROJECT", + "UV_WORKING_DIR", + "UV_NO_SYNC", + "UV_FROZEN", + "UV_LOCKED", + }.intersection(child_env) + assert child_env["UV_INDEX_URL"] == "https://packages.example/simple" + assert child_env["HTTPS_PROXY"] == "http://proxy.example:8080" + + +@pytest.mark.parametrize( + "variable", + ["UV_PYTHON", "UV_INSECURE_HOST", "UV_CONSTRAINT", "UV_NO_BUILD_ISOLATION"], +) +def test_controlled_uv_environment_drops_interpreter_and_trust_overrides(monkeypatch, variable: str) -> None: + """These redirect the target environment rather than index/proxy/cache + settings: `UV_PYTHON` swaps the interpreter that later loads the extension + entry point, and `UV_INSECURE_HOST` removes the TLS validation that the + HTTPS-only source rule depends on.""" + monkeypatch.setenv(variable, "must-not-reach-uv") + + assert variable not in _controlled_uv_environment() + + +@pytest.mark.parametrize( + ("config_text", "expected_error", "message"), + [ + ( + 'plugins: []\nlog_level: info\n"plugins":\n - use: demo_extension:install\n', + ValueError, + "duplicate top-level plugins", + ), + (None, FileNotFoundError, "config not found"), + ("- not-a-mapping\n", ValueError, "must be a mapping"), + ], + ids=["duplicate-plugins-keys", "missing-config", "non-mapping-root"], +) +def test_install_validates_the_config_before_running_third_party_build_hooks( + tmp_path: Path, + monkeypatch, + config_text: str | None, + expected_error: type[Exception], + message: str, +) -> None: + """`uv add`/`uv sync` execute the package's build backend, so a config the + manager can never write to must be rejected before that code runs — not + after it, via rollback.""" + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + config_path = root / "config.yaml" + if config_text is None: + config_path.unlink() + else: + config_path.write_text(config_text, encoding="utf-8") + commands: list[list[str]] = [] + + def _record(command, **_kwargs): + commands.append(list(command)) + return subprocess.CompletedProcess(command, 0, stdout="uv 0.11.1\n") + + monkeypatch.setattr("deerflow.extensions.manager.subprocess.run", _record) + + with pytest.raises(expected_error, match=message): + ExtensionManager(root).install(str(source), yes=True) + + assert commands == [] + assert not (root / "backend" / "extensions").exists() + + +def test_failed_recovery_sync_still_restores_the_dependency_files( + tmp_path: Path, + monkeypatch, +) -> None: + """The recovery `uv sync` runs without `--locked` when the checkout had no + lock, so uv writes one while resolving. If that sync then fails, the + operator must not be left holding a lock file they never had.""" + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + backend = root / "backend" + pyproject_path = backend / "pyproject.toml" + lock_path = backend / "uv.lock" + original_pyproject = pyproject_path.read_text(encoding="utf-8") + + def _run(command, **_kwargs): + if command[:2] == ["uv", "--version"]: + return subprocess.CompletedProcess(command, 0, stdout="uv 0.11.1\n") + if command[0] != "uv": + return subprocess.CompletedProcess(command, 0, stdout='[["demo", "demo_extension:install"]]\n') + if command[1] == "add": + pyproject_path.write_text( + original_pyproject.replace("extensions = []", 'extensions = ["deerflow-extension-demo"]'), + encoding="utf-8", + ) + lock_path.write_text('version = 1\nrequires-python = ">=3.12"\n', encoding="utf-8") + return subprocess.CompletedProcess(command, 0) + if "--locked" not in command: + lock_path.write_text("version = 1\n# written by the recovery resolve\n", encoding="utf-8") + raise subprocess.CalledProcessError(1, command) + + monkeypatch.setattr("deerflow.extensions.manager.subprocess.run", _run) + + with pytest.raises(RuntimeError, match="original failure"): + ExtensionManager(root).install(str(source), yes=True) + + assert not lock_path.exists() + assert pyproject_path.read_text(encoding="utf-8") == original_pyproject + + +def test_interrupt_during_install_restores_files_without_a_recovery_resolve( + tmp_path: Path, + monkeypatch, +) -> None: + """Ctrl-C must not be answered by blocking on a full dependency resolve: a + second interrupt during that sync would escape the handler and strand the + checkout mid-transaction.""" + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + backend = root / "backend" + pyproject_path = backend / "pyproject.toml" + lock_path = backend / "uv.lock" + original_pyproject = pyproject_path.read_text(encoding="utf-8") + syncs: list[list[str]] = [] + + def _run(command, **_kwargs): + if command[:2] == ["uv", "--version"]: + return subprocess.CompletedProcess(command, 0, stdout="uv 0.11.1\n") + if command[0] != "uv": + return subprocess.CompletedProcess(command, 0, stdout='[["demo", "demo_extension:install"]]\n') + if command[1] == "add": + pyproject_path.write_text( + original_pyproject.replace("extensions = []", 'extensions = ["deerflow-extension-demo"]'), + encoding="utf-8", + ) + lock_path.write_text('version = 1\nrequires-python = ">=3.12"\n', encoding="utf-8") + return subprocess.CompletedProcess(command, 0) + syncs.append(list(command)) + raise KeyboardInterrupt + + monkeypatch.setattr("deerflow.extensions.manager.subprocess.run", _run) + + with pytest.raises(KeyboardInterrupt): + ExtensionManager(root).install(str(source), yes=True) + + assert len(syncs) == 1 + assert not lock_path.exists() + assert pyproject_path.read_text(encoding="utf-8") == original_pyproject + assert not (backend / "extensions" / "sources" / "deerflow-extension-demo").exists() + + +def test_entry_point_discovery_tolerates_interpreter_startup_output( + tmp_path: Path, + monkeypatch, +) -> None: + """A `sitecustomize`/`.pth` banner on the child interpreter's stdout must + not roll back an otherwise-successful install with a JSON parse error.""" + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + + def _run(command, **_kwargs): + if command[:2] == ["uv", "--version"]: + return subprocess.CompletedProcess(command, 0, stdout="uv 0.11.1\n") + if command[0] == "uv": + if command[1] == "add": + (root / "backend" / "uv.lock").write_text('version = 1\nrequires-python = ">=3.12"\n', encoding="utf-8") + return subprocess.CompletedProcess(command, 0) + return subprocess.CompletedProcess( + command, + 0, + stdout='vendor sitecustomize loaded\n[["demo", "demo_extension:install"]]\n', + ) + + monkeypatch.setattr("deerflow.extensions.manager.subprocess.run", _run) + + result = ExtensionManager(root).install(str(source), yes=True) + + assert (result.name, result.use) == ("demo", "demo_extension:install") + + +def test_install_rejects_uv_versions_without_no_workspace_support( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + commands: list[list[str]] = [] + + def _old_uv(command, **_kwargs): + commands.append(list(command)) + if command[:2] == ["uv", "--version"]: + return subprocess.CompletedProcess(command, 0, stdout="uv 0.7.20\n") + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr("deerflow.extensions.manager.subprocess.run", _old_uv) + + with pytest.raises(RuntimeError, match="uv 0.8.0 or newer"): + ExtensionManager(root).install(str(source), yes=True) + + assert commands == [["uv", "--version"]] + assert not (root / "backend" / "extensions").exists() + + +def test_remove_uses_deferred_uv_mutation_then_the_same_controlled_sync( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + (root / "config.yaml").write_text( + """\ +config_version: 1 +plugins: + - name: demo + package: deerflow-extension-demo + use: demo_extension:install + enabled: true + required: true + config: {} +""", + encoding="utf-8", + ) + for key in ("UV_PROJECT", "UV_WORKING_DIR", "UV_NO_SYNC", "UV_FROZEN", "UV_LOCKED"): + monkeypatch.setenv(key, "must-not-reach-uv") + monkeypatch.setenv("UV_DEFAULT_INDEX", "https://packages.example/simple") + monkeypatch.setenv("HTTP_PROXY", "http://proxy.example:8080") + uv_calls: list[tuple[list[str], dict[str, object]]] = [] + + def _record_run(command, **kwargs): + if command[0] == "uv": + uv_calls.append((list(command), kwargs)) + if command[1] == "remove": + (root / "backend" / "uv.lock").write_text('version = 1\nrequires-python = ">=3.12"\n', encoding="utf-8") + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr("deerflow.extensions.manager.subprocess.run", _record_run) + + ExtensionManager(root).remove("demo") + + assert [command[1] for command, _ in uv_calls] == ["remove", "sync"] + backend = str(root / "backend") + remove, sync = (uv_calls[0][0], uv_calls[1][0]) + assert ["--project", backend] == remove[remove.index("--project") : remove.index("--project") + 2] + assert "--no-sync" in remove + assert remove[-2:] == ["--", "deerflow-extension-demo"] + assert ["--project", backend] == sync[sync.index("--project") : sync.index("--project") + 2] + assert "--locked" in sync + assert "--no-sync" not in sync + for _, kwargs in uv_calls: + child_env = kwargs["env"] + assert isinstance(child_env, dict) + assert not { + "UV_PROJECT", + "UV_WORKING_DIR", + "UV_NO_SYNC", + "UV_FROZEN", + "UV_LOCKED", + }.intersection(child_env) + assert child_env["UV_DEFAULT_INDEX"] == "https://packages.example/simple" + assert child_env["HTTP_PROXY"] == "http://proxy.example:8080" + + +def test_dependency_sync_uses_the_same_configured_optional_extras_as_startup( + tmp_path: Path, + monkeypatch, +) -> None: + repository_root = Path(__file__).resolve().parents[2] + config_path = tmp_path / "deployment.yaml" + config_path.write_text( + "database:\n backend: postgres\ntools:\n - name: browser_navigate\n", + encoding="utf-8", + ) + monkeypatch.delenv("UV_EXTRAS", raising=False) + monkeypatch.delenv("DEER_FLOW_STREAM_BRIDGE_REDIS_URL", raising=False) + monkeypatch.delenv("DEER_FLOW_SANDBOX_OWNERSHIP_REDIS_URL", raising=False) + + assert _detect_extra_flags(repository_root, config_path) == [ + "--extra", + "browser", + "--extra", + "postgres", + ] diff --git a/backend/tests/test_extension_registry.py b/backend/tests/test_extension_registry.py index 296d0ba5c..633c91f4b 100644 --- a/backend/tests/test_extension_registry.py +++ b/backend/tests/test_extension_registry.py @@ -80,6 +80,34 @@ def test_system_model_observers_are_attributed_and_require_task_storage(): assert loaded.needs_task_store is True +def test_services_are_attributed_without_allocating_task_storage(): + registry = ExtensionRegistry() + service = _Contributor("service") + with registry.attributed_to("service_ext:install"): + registry.service(service) + + loaded = registry.build() + + assert loaded.services == (("service_ext:install", service),) + assert loaded.needs_task_store is False + + +def test_routers_are_flattened_in_registration_order_without_task_storage(): + registry = ExtensionRegistry() + first = _Contributor("first-router") + second = _Contributor("second-router") + with registry.attributed_to("router_ext:install"): + registry.routers((first, second)) + + loaded = registry.build() + + assert loaded.routers == ( + ("router_ext:install", first), + ("router_ext:install", second), + ) + assert loaded.needs_task_store is False + + def test_rollback_restores_all_registration_buckets_positionally(): registry = ExtensionRegistry() with registry.attributed_to("keep:install"): @@ -89,6 +117,8 @@ def test_rollback_restores_all_registration_buckets_positionally(): registry.middlewares(_Contributor("drop-middleware")) registry.task_lifecycle(_Contributor("drop-lifecycle")) registry.system_model_observer(_Contributor("drop-observer")) + registry.service(_Contributor("drop-service")) + registry.routers((_Contributor("drop-router"),)) registry.rollback_to(mark) loaded = registry.build() @@ -96,6 +126,8 @@ def test_rollback_restores_all_registration_buckets_positionally(): assert [contributor.tag for _, contributor in loaded.middleware_contributors] == ["keep"] assert loaded.task_lifecycle == () assert loaded.system_model_observers == () + assert loaded.services == () + assert loaded.routers == () def test_registration_order_is_preserved(): @@ -120,11 +152,15 @@ def test_discard_removes_every_entry_of_one_source(): with registry.attributed_to("bad:install"): registry.middlewares(drop) registry.system_model_observer(drop) + registry.service(drop) + registry.routers((drop,)) registry.discard("bad:install") loaded = registry.build() assert loaded.middleware_contributors == (("good:install", keep),) assert loaded.task_lifecycle == (("good:install", keep),) assert loaded.system_model_observers == () + assert loaded.services == () + assert loaded.routers == () def test_registering_outside_attributed_to_raises(): diff --git a/backend/tests/test_gateway_extension_service_lifecycle.py b/backend/tests/test_gateway_extension_service_lifecycle.py new file mode 100644 index 000000000..23c7181a5 --- /dev/null +++ b/backend/tests/test_gateway_extension_service_lifecycle.py @@ -0,0 +1,203 @@ +"""Regression tests for Gateway ownership of extension services.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from types import SimpleNamespace + +import pytest +from fastapi import FastAPI + +from deerflow.extensions import ( + get_runtime_diagnostics, + initialize_runtime_diagnostics, + reset_runtime_diagnostics, +) +from deerflow.extensions.registry import ExtensionRegistry + + +@pytest.fixture(autouse=True) +def _isolate_runtime_diagnostics(): + reset_runtime_diagnostics() + yield + reset_runtime_diagnostics() + + +def _database_config() -> SimpleNamespace: + return SimpleNamespace( + backend="memory", + checkpoint_channel_mode="full", + checkpoint_delta=SimpleNamespace(snapshot_frequency=10), + ) + + +@asynccontextmanager +async def _resource(value): + yield value + + +def _patch_runtime_resources(monkeypatch, events: list[str]) -> None: + async def init_engine(_database) -> None: + return None + + async def close_engine() -> None: + events.append("engine_close") + + monkeypatch.setattr("deerflow.runtime.make_stream_bridge", lambda _config: _resource(object())) + monkeypatch.setattr("deerflow.runtime.make_store", lambda _config: _resource(object())) + monkeypatch.setattr("deerflow.runtime.checkpointer.async_provider.make_checkpointer", lambda _config: _resource(object())) + monkeypatch.setattr("deerflow.persistence.engine.init_engine_from_config", init_engine) + monkeypatch.setattr("deerflow.persistence.engine.close_engine", close_engine) + monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: None) + + +@pytest.mark.asyncio +async def test_runtime_owns_engine_cleanup_before_initialization(monkeypatch): + from app.gateway.deps import langgraph_runtime + + events: list[str] = [] + + async def fail_engine_init(_database) -> None: + events.append("engine_init") + raise RuntimeError("schema bootstrap failed") + + async def close_engine() -> None: + events.append("engine_close") + + monkeypatch.setattr("deerflow.runtime.make_stream_bridge", lambda _config: _resource(object())) + monkeypatch.setattr("deerflow.persistence.engine.init_engine_from_config", fail_engine_init) + monkeypatch.setattr("deerflow.persistence.engine.close_engine", close_engine) + + with pytest.raises(RuntimeError, match="schema bootstrap failed"): + async with langgraph_runtime( + FastAPI(), + SimpleNamespace(database=_database_config()), + ): + pytest.fail("runtime must not yield") + + assert events == ["engine_init", "engine_close"] + + +@pytest.mark.asyncio +async def test_later_startup_failure_stops_same_snapshot_and_appends_diagnostics(monkeypatch): + import deerflow.extensions as extensions_module + from app.gateway.deps import langgraph_runtime + + events: list[str] = [] + + class _Service: + def __init__(self, name: str, *, fail_start: bool = False, fail_stop: bool = False) -> None: + self.name = name + self.fail_start = fail_start + self.fail_stop = fail_stop + + async def start(self, _deps) -> None: + events.append(f"start:{self.name}") + if self.fail_start: + raise RuntimeError("start exploded") + + async def stop(self) -> None: + events.append(f"stop:{self.name}") + if self.fail_stop: + raise RuntimeError("stop exploded") + + registry = ExtensionRegistry() + with registry.attributed_to("bad-start:install"): + registry.service(_Service("bad-start", fail_start=True)) + with registry.attributed_to("bad-stop:install"): + registry.service(_Service("bad-stop", fail_stop=True)) + snapshot = registry.build() + + other_registry = ExtensionRegistry() + with other_registry.attributed_to("other:install"): + other_registry.service(_Service("other")) + monkeypatch.setattr(extensions_module, "_loaded", other_registry.build()) + + _patch_runtime_resources(monkeypatch, events) + monkeypatch.setattr( + "deerflow.persistence.thread_meta.make_thread_store", + lambda _sf, _store: (_ for _ in ()).throw(RuntimeError("thread store failed")), + ) + + app = FastAPI() + app.state.extensions = snapshot + live_diagnostics = initialize_runtime_diagnostics([]) + app.state.extension_diagnostics = live_diagnostics + + with pytest.raises(RuntimeError, match="thread store failed"): + async with langgraph_runtime( + app, + SimpleNamespace(database=_database_config()), + ): + pytest.fail("runtime must not yield") + + assert events == [ + "start:bad-start", + "start:bad-stop", + "stop:bad-stop", + "stop:bad-start", + "engine_close", + ] + assert all("other" not in event for event in events) + assert app.state.extension_diagnostics is live_diagnostics + assert app.state.extension_diagnostics == get_runtime_diagnostics() + assert [diagnostic.source for diagnostic in live_diagnostics] == [ + "bad-start:install", + "bad-stop:install", + ] + + +@pytest.mark.asyncio +async def test_cancellation_during_service_start_propagates_after_cleanup(monkeypatch): + from app.gateway.deps import langgraph_runtime + + events: list[str] = [] + blocking_start_entered = asyncio.Event() + + class _Service: + def __init__(self, name: str, *, block: bool = False) -> None: + self.name = name + self.block = block + + async def start(self, _deps) -> None: + events.append(f"start:{self.name}") + if self.block: + blocking_start_entered.set() + await asyncio.Event().wait() + + async def stop(self) -> None: + events.append(f"stop:{self.name}") + + registry = ExtensionRegistry() + with registry.attributed_to("first:install"): + registry.service(_Service("first")) + with registry.attributed_to("blocking:install"): + registry.service(_Service("blocking", block=True)) + with registry.attributed_to("never-started:install"): + registry.service(_Service("never-started")) + + _patch_runtime_resources(monkeypatch, events) + app = FastAPI() + app.state.extensions = registry.build() + + async def run_runtime() -> None: + async with langgraph_runtime( + app, + SimpleNamespace(database=_database_config()), + ): + pytest.fail("runtime must not yield while service start is blocked") + + task = asyncio.create_task(run_runtime()) + await asyncio.wait_for(blocking_start_entered.wait(), timeout=1.0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert events == [ + "start:first", + "start:blocking", + "stop:blocking", + "stop:first", + "engine_close", + ] diff --git a/backend/tests/test_gateway_request_path.py b/backend/tests/test_gateway_request_path.py new file mode 100644 index 000000000..3a3d533bb --- /dev/null +++ b/backend/tests/test_gateway_request_path.py @@ -0,0 +1,180 @@ +"""Conformance tests for the shared request-path projection. + +``app.gateway.request_path.get_request_route_path()`` exists to answer one +question: *which path string is Starlette's router matching right now?* Auth +and CSRF classify that value, so the security predicates and the dispatcher +must agree on it exactly. When they disagree, a route mounted under a +public-looking prefix can be classified public while the router dispatches to +a protected handler. + +These tests pin the agreement itself rather than the mechanism that produces +it, so they stay meaningful whether the projection keeps delegating to +Starlette or is ever reimplemented. Nothing here asserts *how* the value is +computed -- only that middleware and router see the same string. +""" + +import pytest +from fastapi import FastAPI +from starlette.requests import Request +from starlette.testclient import TestClient + +from app.gateway.auth_middleware import AuthMiddleware +from app.gateway.csrf_middleware import CSRFMiddleware, is_auth_endpoint, should_check_csrf +from app.gateway.request_path import get_request_route_path +from deerflow.config.authorization_config import AuthorizationConfig + + +@pytest.fixture(autouse=True) +def _default_route_authorization_config(monkeypatch): + """Keep minimal middleware apps independent of a repository config.yaml.""" + monkeypatch.setattr( + "app.gateway.authz._get_route_authorization_config", + lambda: AuthorizationConfig(), + ) + + +@pytest.fixture(autouse=True) +def _auth_enabled(monkeypatch): + """Every case here is about the enabled-auth path.""" + monkeypatch.delenv("DEER_FLOW_AUTH_DISABLED", raising=False) + + +def _request(path: str, root_path: str = "", method: str = "GET") -> Request: + return Request( + { + "type": "http", + "method": method, + "path": path, + "root_path": root_path, + "query_string": b"", + "headers": [], + } + ) + + +# ── Projection edge cases ──────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("path", "root_path", "expected"), + [ + # No mount: the raw ASGI path is already what the router matches. + ("/api/models", "", "/api/models"), + # Mounted: root_path is stripped on the segment boundary. + ("/prefix/api/models", "/prefix", "/api/models"), + # Nested mounts accumulate into root_path; strip all of it. + ("/outer/inner/health", "/outer/inner", "/health"), + # The mount itself was requested; the router matches the empty path. + ("/prefix", "/prefix", ""), + # root_path is not a prefix at all -- never strip. + ("/other/models", "/prefix", "/other/models"), + # Prefix collides mid-segment. Stripping here would produce "foo", + # a string the router would never match. + ("/apifoo/models", "/api", "/apifoo/models"), + ("/apifoo", "/api", "/apifoo"), + ], +) +def test_projection_strips_root_path_only_on_segment_boundaries(path: str, root_path: str, expected: str): + assert get_request_route_path(_request(path, root_path)) == expected + + +# ── Agreement with the router ──────────────────────────────────────────────── + + +def test_projection_matches_the_path_the_router_dispatched_on(): + """The middleware-visible string equals the route's declared path.""" + seen: dict[str, str] = {} + + child = FastAPI() + + @child.get("/health") + async def health(request: Request): + seen["projection"] = get_request_route_path(request) + return {"ok": True} + + middle = FastAPI() + middle.mount("/inner", child) + parent = FastAPI() + parent.mount("/outer", middle) + + assert TestClient(parent).get("/outer/inner/health").status_code == 200 + # The router matched the declared "/health", not the wire path. + assert seen["projection"] == "/health" + + +def test_public_route_stays_public_under_nested_mounts(): + """Availability direction: a mounted /health must not start 401-ing.""" + child = FastAPI() + child.add_middleware(AuthMiddleware) + + @child.get("/health") + async def health(): + return {"ok": True} + + middle = FastAPI() + middle.mount("/inner", child) + parent = FastAPI() + parent.mount("/outer", middle) + + assert TestClient(parent).get("/outer/inner/health").status_code == 200 + + +# ── The bypass these predicates exist to prevent ───────────────────────────── + + +def test_mounting_under_a_public_prefix_does_not_expose_protected_routes(): + """Security direction: the mount prefix must not leak into classification. + + Classifying the raw wire path would see "/health/api/models", match the + "/health" public prefix, and skip authentication entirely -- while the + router dispatches to the protected "/api/models" handler. + """ + child = FastAPI() + child.add_middleware(AuthMiddleware) + + @child.get("/api/models") + async def models(): + return {"models": []} + + parent = FastAPI() + parent.mount("/health", child) + + assert TestClient(parent).get("/health/api/models").status_code == 401 + + +def test_csrf_is_enforced_for_routes_mounted_under_the_webhook_prefix(): + """CSRF's webhook exemption keys off the projection, not the wire path.""" + child = FastAPI() + child.add_middleware(CSRFMiddleware) + + @child.post("/action") + async def action(): + return {"ok": True} + + parent = FastAPI() + parent.mount("/api/webhooks", child) + + response = TestClient(parent).post("/api/webhooks/action") + + assert response.status_code == 403 + assert "CSRF token missing" in response.json()["detail"] + + +# ── CSRF predicates read the same projection ───────────────────────────────── + + +def test_csrf_exemptions_follow_the_projection(): + # Genuine host webhook: no mount, exempt. + assert should_check_csrf(_request("/api/webhooks/github", method="POST")) is False + # Same wire path, but the router is matching "/github" inside a mount -- + # not the host's webhook namespace, so CSRF still applies. + assert should_check_csrf(_request("/api/webhooks/github", "/api/webhooks", method="POST")) is True + + +def test_auth_endpoint_detection_follows_the_projection(): + assert is_auth_endpoint(_request("/api/v1/auth/login/local", method="POST")) is True + # A mount whose prefix is itself the auth namespace: the router matches + # "/local", which is not the host's exempt endpoint. + assert is_auth_endpoint(_request("/api/v1/auth/login/local", "/api/v1/auth/login", method="POST")) is False + # The mount point itself was requested; the router matches the empty path. + assert is_auth_endpoint(_request("/api/v1/auth/login/local", "/api/v1/auth/login/local", method="POST")) is False diff --git a/backend/tests/test_gateway_run_drain_shutdown.py b/backend/tests/test_gateway_run_drain_shutdown.py index 7e7bdf704..22a72b490 100644 --- a/backend/tests/test_gateway_run_drain_shutdown.py +++ b/backend/tests/test_gateway_run_drain_shutdown.py @@ -148,7 +148,7 @@ async def test_shutdown_is_noop_without_inflight_runs(): @pytest.mark.asyncio async def test_langgraph_runtime_drains_runs_before_closing_checkpointer(monkeypatch): - """The wiring order lock for #3373: drain in-flight runs, THEN close the pool. + """Drain runs before services, then close runtime resources in stack order. Patches every ``langgraph_runtime`` collaborator down to trivial stand-ins so only the bootstrap/teardown ordering runs. The checkpointer probe records when @@ -158,6 +158,7 @@ async def test_langgraph_runtime_drains_runs_before_closing_checkpointer(monkeyp from fastapi import FastAPI from app.gateway.deps import langgraph_runtime + from deerflow.extensions.registry import ExtensionRegistry events: list[str] = [] @@ -170,17 +171,27 @@ async def test_langgraph_runtime_drains_runs_before_closing_checkpointer(monkeyp @asynccontextmanager async def fake_stream_bridge(_config): - yield object() + try: + yield object() + finally: + events.append("stream_bridge_closed") @asynccontextmanager async def fake_store(_config): - yield object() + try: + yield object() + finally: + events.append("store_closed") async def fake_init_engine(_db): + events.append("engine_initialized") + + def fake_session_factory(): + events.append("session_factory_resolved") return None async def fake_close_engine(): - return None + events.append("engine_closed") async def spy_shutdown(self, *, timeout): # noqa: ANN001 events.append("runs_drained") @@ -197,7 +208,7 @@ async def test_langgraph_runtime_drains_runs_before_closing_checkpointer(monkeyp monkeypatch.setattr("deerflow.runtime.make_store", fake_store) monkeypatch.setattr("deerflow.persistence.engine.init_engine_from_config", fake_init_engine) monkeypatch.setattr("deerflow.persistence.engine.close_engine", fake_close_engine) - monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: None) + monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", fake_session_factory) monkeypatch.setattr("deerflow.runtime.events.store.make_run_event_store", lambda _cfg: object()) monkeypatch.setattr("deerflow.persistence.thread_meta.make_thread_store", lambda _sf, _store: object()) monkeypatch.setattr(RunManager, "shutdown", spy_shutdown, raising=False) @@ -205,16 +216,36 @@ async def test_langgraph_runtime_drains_runs_before_closing_checkpointer(monkeyp monkeypatch.setattr("deerflow.extensions.notify.reset_extension_notify_loop", spy_reset_extension_notify_loop) app = FastAPI() + registry = ExtensionRegistry() + + class _Service: + async def start(self, _deps): + events.append("service_started") + + async def stop(self): + events.append("service_stopped") + + with registry.attributed_to("service:install"): + registry.service(_Service()) + app.state.extensions = registry.build() startup_config = SimpleNamespace(database=SimpleNamespace(backend="memory", checkpoint_channel_mode="full", checkpoint_delta=SimpleNamespace(snapshot_frequency=10)), run_events=None) async with langgraph_runtime(app, startup_config): pass assert "runs_drained" in events, "langgraph_runtime never drained in-flight runs on shutdown" + assert "service_started" in events + assert "service_stopped" in events assert "checkpointer_closed" in events - assert events.index("runs_drained") < events.index("checkpointer_closed"), f"runs must be drained before the checkpointer pool is closed; got order {events}" + assert events.index("engine_initialized") < events.index("session_factory_resolved") + assert events.index("session_factory_resolved") < events.index("service_started") + assert events.index("runs_drained") < events.index("service_stopped") + assert events.index("service_stopped") < events.index("store_closed") + assert events.index("store_closed") < events.index("checkpointer_closed") + assert events.index("checkpointer_closed") < events.index("engine_closed") + assert events.index("engine_closed") < events.index("stream_bridge_closed") assert events[0] == "extension_loop_set" - assert events.index("checkpointer_closed") < events.index("extension_loop_reset"), f"extension loop reset must be the final runtime teardown; got order {events}" + assert events.index("stream_bridge_closed") < events.index("extension_loop_reset"), f"extension loop reset must be the final runtime teardown; got order {events}" @pytest.mark.asyncio diff --git a/backend/tests/test_gateway_runtime_cleanup.py b/backend/tests/test_gateway_runtime_cleanup.py index 17ee6470a..a4797baec 100644 --- a/backend/tests/test_gateway_runtime_cleanup.py +++ b/backend/tests/test_gateway_runtime_cleanup.py @@ -105,7 +105,10 @@ def test_backend_make_dev_gateway_reload_excludes_runtime_state_with_absolute_di assert "DEER_FLOW_HOME := $(abspath $(DEER_FLOW_HOME))" in makefile assert "BACKEND_SANDBOX_HOME := $(abspath $(CURDIR)/sandbox)" in makefile assert 'mkdir -p "$(DEER_FLOW_HOME)" "$(BACKEND_SANDBOX_HOME)"' in makefile - assert 'DEER_FLOW_HOME="$(DEER_FLOW_HOME)" uv run uvicorn' in makefile + # The launch line may carry runtime-only uv flags (`--locked` pins the + # extension lock); what this guards is that DEER_FLOW_HOME is exported on it, + # so the reload-excludes below resolve to the same absolute directories. + assert re.search(r'DEER_FLOW_HOME="\$\(DEER_FLOW_HOME\)" uv run(?: --(?:locked|no-sync))? uvicorn', makefile) assert '--reload-exclude="$(DEER_FLOW_HOME)"' in makefile assert '--reload-exclude="$(BACKEND_SANDBOX_HOME)"' in makefile diff --git a/backend/tests/test_tui_cli.py b/backend/tests/test_tui_cli.py index 6cdacca27..00a153238 100644 --- a/backend/tests/test_tui_cli.py +++ b/backend/tests/test_tui_cli.py @@ -2,13 +2,17 @@ import pytest -from deerflow.tui.cli import LaunchPlan, plan_launch +from deerflow.tui.cli import LaunchPlan, build_parser, plan_launch def plan(argv, *, stdin_tty=True, stdout_tty=True, env=None): return plan_launch(argv, stdin_isatty=stdin_tty, stdout_isatty=stdout_tty, env=env or {}) +def test_top_level_help_points_to_extension_management(): + assert "deerflow extensions --help" in build_parser().format_help() + + def test_bare_command_on_tty_launches_tui(): p = plan([]) assert p.mode == "tui" diff --git a/backend/tests/test_uvicorn_reload_exclude.py b/backend/tests/test_uvicorn_reload_exclude.py index 9302c1c63..f4c8654cc 100644 --- a/backend/tests/test_uvicorn_reload_exclude.py +++ b/backend/tests/test_uvicorn_reload_exclude.py @@ -157,14 +157,15 @@ def test_sandbox_mkdir_precedes_uvicorn_launch(name): ``_mkdir_dirs`` only proves the mkdir is present somewhere; this pins script order so a future edit can't move (or guard) the mkdir below the launch and - silently reintroduce the #3454 crash on a fresh checkout. ``uv run uvicorn`` - matches the launch but not serve.sh's ``stop_all`` kill line. + silently reintroduce the #3454 crash on a fresh checkout. The ``uv run`` + matcher allows runtime-only flags while still excluding serve.sh's + ``stop_all`` kill line. """ lines = LAUNCHERS[name].read_text(encoding="utf-8").splitlines() - launch_idx = next((i for i, ln in enumerate(lines) if "uv run uvicorn" in ln), None) + launch_idx = next((i for i, ln in enumerate(lines) if re.search(r"\buv run(?: --(?:no-sync|locked))? uvicorn\b", ln)), None) mkdir_idx = next((i for i, ln in enumerate(lines) if re.search(r"\bmkdir\b", ln) and "sandbox" in ln.lower()), None) - assert launch_idx is not None, f"{name}: could not locate the 'uv run uvicorn' launch line" + assert launch_idx is not None, f"{name}: could not locate the uvicorn launch line" assert mkdir_idx is not None, f"{name}: could not locate the sandbox mkdir line" assert mkdir_idx < launch_idx, f"{name}: sandbox mkdir (line {mkdir_idx + 1}) must precede uvicorn launch (line {launch_idx + 1})" diff --git a/backend/uv.lock b/backend/uv.lock index 3fe1033df..a4a9aa18c 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -821,6 +821,7 @@ dependencies = [ { name = "python-telegram-bot" }, { name = "slack-sdk" }, { name = "sse-starlette" }, + { name = "starlette" }, { name = "uvicorn", extra = ["standard"] }, { name = "wecom-aibot-python-sdk" }, ] @@ -886,6 +887,7 @@ requires-dist = [ { name = "python-telegram-bot", specifier = ">=21.0" }, { name = "slack-sdk", specifier = ">=3.33.0" }, { name = "sse-starlette", specifier = ">=2.1.0" }, + { name = "starlette", specifier = ">=1.3.1,<2" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" }, { name = "wecom-aibot-python-sdk", specifier = ">=0.1.6" }, ] @@ -904,10 +906,11 @@ dev = [ { name = "ruff", specifier = ">=0.14.11" }, { name = "textual", specifier = ">=0.80" }, ] +extensions = [] [[package]] name = "deerflow-extension-api" -version = "0.1.1" +version = "0.1.2" source = { editable = "packages/extension-api" } [[package]] @@ -946,6 +949,7 @@ dependencies = [ { name = "langgraph-sdk" }, { name = "markdownify" }, { name = "markitdown", extra = ["all", "xlsx"] }, + { name = "packaging" }, { name = "pydantic" }, { name = "pyyaml" }, { name = "readabilipy" }, @@ -1028,6 +1032,7 @@ requires-dist = [ { name = "markdownify", specifier = ">=1.2.2" }, { name = "markitdown", extras = ["all", "xlsx"], specifier = ">=0.0.1a2" }, { name = "monocle-apptrace", marker = "extra == 'monocle'", specifier = ">=0.8.8" }, + { name = "packaging", specifier = ">=24.2" }, { name = "playwright", marker = "extra == 'browser'", specifier = ">=1.40" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'", specifier = ">=3.3.3" }, { name = "psycopg-pool", marker = "extra == 'postgres'", specifier = ">=3.3.0" }, @@ -4514,8 +4519,8 @@ name = "standard-aifc" version = "3.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "audioop-lts" }, - { name = "standard-chunk" }, + { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, + { name = "standard-chunk", marker = "python_full_version >= '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/53/6050dc3dde1671eb3db592c13b55a8005e5040131f7509cef0215212cb84/standard_aifc-3.13.0.tar.gz", hash = "sha256:64e249c7cb4b3daf2fdba4e95721f811bde8bdfc43ad9f936589b7bb2fae2e43", size = 15240, upload-time = "2024-10-30T16:01:31.772Z" } wheels = [ diff --git a/config.example.yaml b/config.example.yaml index 7d8a0105f..62e5c7865 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -41,7 +41,8 @@ logging: # lead-only vs subagent-only configuration is not expressible yet. Treat these # files as trusted operator config because middleware classes execute code. # Uncomment this block to define middlewares in config.yaml. Leaving it commented -# lets extensions_config.json remain the source of truth for extension packages. +# lets extensions_config.json remain the source of truth for this legacy +# config-declared middleware list. Packaged plugins use the `plugins:` block below. # extensions: # middlewares: # - my_company.deerflow_middlewares:DomainGuardMiddleware @@ -2531,8 +2532,27 @@ authorization: # entry names an install entry point as "module.path:install"; the `config` # block is handed to that package verbatim and validated by the package itself. # -# Loading is explicit on purpose: an installed package does nothing until it is -# listed here, and the list order is what fixes middleware ordering. +# Prefer the extension manager over editing this list by hand: +# make extension-install SOURCE="deerflow-extension-acme==1.2.3" +# make extension-install SOURCE="git+https://github.com/acme/deerflow-extension-acme.git@" +# make extension-install SOURCE="/absolute/path/to/local-extension" +# make extension-list +# make extension-enable NAME=acme +# make extension-disable NAME=acme +# make extension-remove NAME=acme +# The installer updates backend/pyproject.toml, backend/uv.lock, and this block +# together. It also asks the operator to confirm that the source is trusted; +# extensions and their build systems execute with Gateway privileges. +# +# Every install, enable, disable, remove, or manual edit requires a Gateway +# restart. Development launchers may fetch missing locked artifacts before +# handoff; a built production Gateway never installs plugins during startup. +# Local directories are copied to backend/extensions/sources as deployable +# snapshots; use a pinned version or public HTTPS Git commit for reproducible +# remote sources. SSH Git URLs are rejected because the stock Docker builder +# does not forward host SSH credentials. +# Loading remains explicit: a package installed outside the manager does nothing +# until it is listed here, and list order fixes contribution ordering. # # This is deliberately separate from the `extensions:` block above (MCP servers, # skills, config-declared middlewares). That one is backed by @@ -2540,20 +2560,16 @@ authorization: # endpoint; a list that causes code to be imported must stay in this # operator-controlled file only. # -# This initial extension-system slice accepts middleware contributors. Their -# install functions register semantically placed middlewares for the lead and/or -# subagent stack; the examples below intentionally cover only that surface. +# Plugins can contribute semantically placed middleware, task-lifecycle hooks, +# system-model observers, Gateway-lifetime services, and eager FastAPI HTTP routers. +# See examples/deerflow-extension-example for a package that exercises all five. # # plugins: -# - use: acme_request_observer:install +# - name: example # PEP 621 entry-point name +# package: deerflow-extension-example # Python distribution managed by uv +# use: deerflow_extension_example:install +# enabled: true # false skips import and registration +# required: false # true makes load failure abort Gateway startup +# # (install --required opts in) # config: -# enabled: true -# placement: model_logical -# - use: acme_tool_observer:install -# config: -# enabled: true -# placement: tool_raw -# # `required: true` turns a load failure into a startup failure. Use it for -# # packages whose absence changes behaviour rather than just observability. -# - use: acme_required_middleware:install -# required: true +# label: example # extension-private values, if any diff --git a/docker/dev-entrypoint.sh b/docker/dev-entrypoint.sh index b6dc11e0e..e6cb1d04e 100755 --- a/docker/dev-entrypoint.sh +++ b/docker/dev-entrypoint.sh @@ -5,13 +5,17 @@ # (PR #2767, addressing review on Issue #2754). # # Responsibilities: -# 1. Resolve `--extra X` flags from UV_EXTRAS (comma- or whitespace-separated, -# mirroring scripts/detect_uv_extras.py for parity with local `make dev`). +# 1. Resolve `--extra X` flags through scripts/detect_uv_extras.py, using the +# selected config plus explicit UV_EXTRAS and runtime-required backends. # 2. Validate each extra against [A-Za-z][A-Za-z0-9_-]* so a stray shell # metacharacter in `.env` cannot reach `uv sync`. -# 3. `uv sync --all-packages` so workspace member extras (deerflow-harness's +# 3. `uv sync --locked --all-packages` so the declared extension group and +# workspace member extras (deerflow-harness's # postgres extra in particular) are installed — see PR #2584. -# 4. Self-heal: if the first sync fails, recreate .venv and retry once. +# 4. Self-heal: if the first sync fails, recreate .venv and retry once. The +# retry stays `--locked`, so it repairs a broken .venv but not a stale +# lock; a second failure aborts with recovery instructions rather than +# starting uvicorn against an environment that does not match the lock. # 5. Hand off to uvicorn with reload, replacing this shell so uvicorn becomes # PID 1 inside the container. # @@ -38,22 +42,78 @@ fi # ── Resolve extras ────────────────────────────────────────────────────────── EXTRAS_FLAGS="" +EXTRA_NAMES="" +set -f + +append_extra() { + extra_name="$1" + case "$extra_name" in + [!A-Za-z]* | *[!A-Za-z0-9_-]*) + echo "[startup] UV_EXTRAS entry '$extra_name' is invalid (must match [A-Za-z][A-Za-z0-9_-]*) — aborting" >&2 + exit 1 + ;; + esac + case " $EXTRA_NAMES " in + *" $extra_name "*) return ;; + esac + EXTRA_NAMES="$EXTRA_NAMES $extra_name" + EXTRAS_FLAGS="$EXTRAS_FLAGS --extra $extra_name" +} + +# Validate explicit input before the detector normalizes it. The shared +# detector deliberately drops invalid names with a warning, while container +# startup fails closed so malformed .env input cannot be silently ignored. if [ -n "${UV_EXTRAS:-}" ]; then - # Normalize comma → space, then split on whitespace via the unquoted `for`. for raw in $(printf '%s' "$UV_EXTRAS" | tr ',' ' '); do - [ -z "$raw" ] && continue - # Reject anything that does not look like an identifier. - # Two patterns: leading non-letter, or any non-[A-Za-z0-9_-] character. - case "$raw" in - [!A-Za-z]* | *[!A-Za-z0-9_-]*) - echo "[startup] UV_EXTRAS entry '$raw' is invalid (must match [A-Za-z][A-Za-z0-9_-]*) — aborting" >&2 - exit 1 - ;; - esac - EXTRAS_FLAGS="$EXTRAS_FLAGS --extra $raw" + [ -z "$raw" ] || append_extra "$raw" done fi +# Docker dev mounts the host checkout at /app/project while +# DEER_FLOW_PROJECT_ROOT points at /app for runtime path translation. Prefer +# both locations, then the checkout-relative path used by direct invocations. +ENTRYPOINT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +DETECTOR_PATH="" +for candidate in \ + "${DEER_FLOW_PROJECT_ROOT:+$DEER_FLOW_PROJECT_ROOT/scripts/detect_uv_extras.py}" \ + /app/project/scripts/detect_uv_extras.py \ + "$ENTRYPOINT_DIR/../scripts/detect_uv_extras.py" +do + if [ -n "$candidate" ] && [ -f "$candidate" ]; then + DETECTOR_PATH="$candidate" + break + fi +done +if [ -z "$DETECTOR_PATH" ]; then + echo "[startup] scripts/detect_uv_extras.py is unavailable" >&2 + exit 1 +fi +if command -v python3 >/dev/null 2>&1; then + DETECTOR_PYTHON=python3 +elif command -v python >/dev/null 2>&1; then + DETECTOR_PYTHON=python +else + echo "[startup] Python is required to resolve optional dependencies" >&2 + exit 1 +fi +if ! DETECTED_FLAGS=$("$DETECTOR_PYTHON" "$DETECTOR_PATH"); then + echo "[startup] detect_uv_extras.py failed" >&2 + exit 1 +fi + +# The detector emits only validated `--extra NAME` pairs. Parse that small +# interface instead of evaluating shell text, and validate again at the final +# shell boundary before any value can reach uv. +set -- $DETECTED_FLAGS +while [ "$#" -gt 0 ]; do + if [ "$1" != "--extra" ] || [ "$#" -lt 2 ]; then + echo "[startup] detect_uv_extras.py returned invalid output" >&2 + exit 1 + fi + append_extra "$2" + shift 2 +done + if [ "$PRINT_EXTRAS_ONLY" = "1" ]; then # Trim leading space for tidier output, then exit. printf '%s\n' "${EXTRAS_FLAGS# }" @@ -78,22 +138,31 @@ mkdir -p "$DEER_FLOW_HOME" /app/backend/.deer-flow /app/backend/sandbox cd /app/backend # `--all-packages` propagates extras into workspace members (PR #2584). -# `--extra redis` is always installed because docker-compose-dev defaults the -# stream bridge to Redis (DEER_FLOW_STREAM_BRIDGE_REDIS_URL); redis is an -# optional extra elsewhere. It is kept out of EXTRAS_FLAGS so the --print-extras -# contract (UV_EXTRAS-derived flags only) stays unchanged. +# docker-compose-dev's default DEER_FLOW_STREAM_BRIDGE_REDIS_URL is translated +# to `--extra redis` by the shared detector, alongside config and UV_EXTRAS. # `$EXTRAS_FLAGS` intentionally unquoted so each `--extra X` becomes its own arg. # shellcheck disable=SC2086 # word-splitting is intentional here -if ! uv sync --all-packages --extra redis $EXTRAS_FLAGS; then +if ! uv sync --locked --all-packages $EXTRAS_FLAGS; then echo "[startup] uv sync failed; recreating .venv and retrying once" - uv venv --allow-existing .venv + uv venv --clear .venv + # The retry keeps `--locked` on purpose: it repairs a corrupt or partial + # .venv, not a lock that disagrees with pyproject.toml. Startup must never + # silently resolve dependencies, so a second failure is fatal rather than + # something uvicorn limps past and reports later as an import error. + # `set -e` would already stop here; abort explicitly so the operator gets + # the fix instead of a bare uv exit code. # shellcheck disable=SC2086 - uv sync --all-packages --extra redis $EXTRAS_FLAGS + if ! uv sync --locked --all-packages $EXTRAS_FLAGS; then + echo "[startup] uv sync --locked failed again after recreating .venv." >&2 + echo "[startup] backend/uv.lock does not match backend/pyproject.toml, or a locked artifact is unreachable." >&2 + echo "[startup] Run 'make install' on the host to refresh the lock, then restart this container." >&2 + exit 1 + fi fi # ── Hand off to uvicorn ───────────────────────────────────────────────────── -PYTHONPATH=. exec uv run uvicorn app.gateway.app:app \ +PYTHONPATH=. exec uv run --no-sync uvicorn app.gateway.app:app \ --host 0.0.0.0 --port 8001 \ --reload \ --reload-include='*.yaml' \ diff --git a/docker/docker-compose-dev.yaml b/docker/docker-compose-dev.yaml index 35a9dd9e1..c4dfa801e 100644 --- a/docker/docker-compose-dev.yaml +++ b/docker/docker-compose-dev.yaml @@ -153,7 +153,7 @@ services: # cache_from disabled - requires manual setup: mkdir -p /tmp/docker-cache-gateway args: APT_MIRROR: ${APT_MIRROR:-} - UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:0.7.20} + UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:0.11.1} UV_INDEX_URL: ${UV_INDEX_URL:-https://pypi.org/simple} NPM_REGISTRY: ${NPM_REGISTRY:-} LARK_CLI_NPM_VERSION: ${LARK_CLI_NPM_VERSION:-1.0.65} diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 3744d12f2..fedd0762e 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -90,7 +90,7 @@ services: dockerfile: backend/Dockerfile args: APT_MIRROR: ${APT_MIRROR:-} - UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:0.7.20} + UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:0.11.1} UV_INDEX_URL: ${UV_INDEX_URL:-https://pypi.org/simple} UV_EXTRAS: ${UV_EXTRAS:-} NPM_REGISTRY: ${NPM_REGISTRY:-} @@ -102,7 +102,7 @@ services: # reconnect, but run cancel, request dedup, and per-worker IM channel # services remain worker-local. Override GATEWAY_WORKERS only when the # Redis stream bridge is enabled and those limitations are acceptable. - command: sh -c "cd backend && PYTHONPATH=. uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 --workers ${GATEWAY_WORKERS:-1}" + command: sh -c "cd backend && PYTHONPATH=. uv run --no-sync uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 --workers ${GATEWAY_WORKERS:-1}" volumes: - ${DEER_FLOW_CONFIG_PATH}:/app/backend/config.yaml:ro - ${DEER_FLOW_EXTENSIONS_CONFIG_PATH}:/app/backend/extensions_config.json:ro diff --git a/examples/deerflow-extension-example/README.md b/examples/deerflow-extension-example/README.md new file mode 100644 index 000000000..5890fa7b2 --- /dev/null +++ b/examples/deerflow-extension-example/README.md @@ -0,0 +1,149 @@ +# DeerFlow extension example + +This directory is a compact, standalone Python package showing all five DeerFlow +extension contribution kinds. It depends on the public +`deerflow-extension-api` contract and never imports `deerflow.*` or `app.*`. + +The contract package intentionally has no framework dependencies. An extension +must therefore declare every framework it imports itself; this example explicitly +depends on FastAPI, LangChain, and LangGraph in `pyproject.toml`. + +## What it demonstrates + +| Contribution | Example behavior | +| --- | --- | +| Middleware | Counts tool calls through one `TOOL_VISIBLE` middleware for lead agents and subagents | +| Task lifecycle | Creates task-scoped stats on start and folds them into app scope on stop | +| System-model observer | Counts DeerFlow-owned model calls, including failures | +| Service | Binds `ExtensionRuntimeDeps` only while the Gateway is running | +| Router | Eagerly declares `GET /api/extension-example/stats` during `install()` | + +The middleware reads task scope only through `task_store_from_runtime()`. It +passes through unchanged when no task store exists. The router and service use +the same `ExampleService` object: its FastAPI dependency returns `503` before +`start()`, after `stop()`, or when no app store was bound. This keeps the route +topology stable while runtime capabilities arrive later. + +## Run the package tests + +`deerflow-extension-api` is currently sourced from this checkout. Install it +first, then install this independent package: + +```bash +cd examples/deerflow-extension-example +uv venv --python 3.12 +uv pip install -e ../../backend/packages/extension-api +uv pip install -e ".[dev]" +uv run --no-project pytest -q +uv run --no-project ruff check . +uv run --no-project ruff format --check . +``` + +The tests use only the public contract plus this package's declared dependencies; +the DeerFlow harness and Gateway application are not imported. + +## Install and load it in DeerFlow + +From the DeerFlow checkout root, install this directory through the extension +manager. Use an absolute path because the Make wrapper invokes the manager from +`backend/`: + +```bash +make extension-install SOURCE="$PWD/examples/deerflow-extension-example" +make extension-list +``` + +After the trust prompt is accepted, the manager: + +- copies a deployable snapshot to + `backend/extensions/sources/deerflow-extension-example/`; +- adds that snapshot to `backend/pyproject.toml`'s `extensions` dependency + group and updates `backend/uv.lock`; +- installs the locked environment; and +- adds and enables this startup-only entry in the selected `config.yaml`. + +```yaml +plugins: + - name: example + package: deerflow-extension-example + use: deerflow_extension_example:install + enabled: true + required: false + config: {} +``` + +Start or restart DeerFlow after installation: + +```bash +make dev +``` + +The Gateway imports extensions only while constructing the application. Install, +enable, disable, remove, and manual `plugins:` changes therefore take effect only +after a restart. These commands manage the example afterward: + +```bash +make extension-disable NAME=example +make extension-enable NAME=example +make extension-remove NAME=example +``` + +The manager can also install a PyPI requirement or a pinned public HTTPS Git +URL. SSH Git URLs are rejected because the stock Docker builder does not +forward host SSH credentials. The direct CLI surface, run from `backend/`, is: + +```text +uv run --frozen --no-group extensions deerflow extensions install [--yes] [--required] +uv run --frozen --no-group extensions deerflow extensions list +uv run --frozen --no-group extensions deerflow extensions enable +uv run --frozen --no-group extensions deerflow extensions disable +uv run --frozen --no-group extensions deerflow extensions remove +``` + +`--yes` is intended only for automation that has already reviewed and trusted +the source: extension build hooks and runtime code execute with Gateway +privileges. `--required` records `required: true`, which turns any later load +failure into a Gateway startup abort; leave it off unless the application is +wrong without this extension. + +The local snapshot is included in Docker builds. Local `make dev`, Docker dev, +and the production Gateway image all consume the same `backend/uv.lock`. +Development launchers may download missing locked artifacts before handing off +to the Gateway; a built production container does not. Rebuild the production +image with `make up` after changing the installed set. + +After one or more runs, request the extension route: + +```bash +curl -s http://localhost:2026/api/extension-example/stats +``` + +The response contains aggregated task outcomes, tool-call counts, system-model +call counts, the app scope id, and a small projection of the host policy. The +route passes through the Gateway's normal authentication middleware; use an +authenticated browser session when authentication is enabled. + +## Packaging entry point + +Managed packages expose exactly one standard PEP 621 entry point in the +`deerflow.extensions` group. This example declares: + +```toml +[project.entry-points."deerflow.extensions"] +example = "deerflow_extension_example:install" +``` + +The entry-point name (`example`) is the stable operator-facing name accepted by +`enable`, `disable`, and `remove`; those commands also accept the distribution +name or the `module:install` value. + +## Package layout + +```text +deerflow_extension_example/ +├── __init__.py # version-stamped install() entry point +└── plugin.py # state plus all five small contribution implementations +tests/ +├── test_entry_point.py +└── test_plugin.py +``` diff --git a/examples/deerflow-extension-example/deerflow_extension_example/__init__.py b/examples/deerflow-extension-example/deerflow_extension_example/__init__.py new file mode 100644 index 000000000..eb14d3025 --- /dev/null +++ b/examples/deerflow-extension-example/deerflow_extension_example/__init__.py @@ -0,0 +1,35 @@ +"""A compact, standalone DeerFlow extension exercising every contribution kind.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from deerflow_extension_api import ExtensionInstall, ExtensionRegistry, extension + +from deerflow_extension_example.plugin import ( + ExampleMiddlewareContributor, + ExampleService, + ExampleSystemObserver, + ExampleTaskLifecycle, + build_router, +) + +__all__ = ["install"] + + +@extension(api="0.1.2", name="example") +def install(registry: ExtensionRegistry, config: Mapping[str, Any]) -> None: + """Register one example of each supported contribution kind.""" + if config.get("enabled", True) is False: + return + + service = ExampleService() + registry.middlewares(ExampleMiddlewareContributor()) + registry.task_lifecycle(ExampleTaskLifecycle()) + registry.system_model_observer(ExampleSystemObserver()) + registry.service(service) + registry.routers((build_router(service),)) + + +_entry_point: ExtensionInstall = install diff --git a/examples/deerflow-extension-example/deerflow_extension_example/plugin.py b/examples/deerflow-extension-example/deerflow_extension_example/plugin.py new file mode 100644 index 000000000..fb301c6d7 --- /dev/null +++ b/examples/deerflow-extension-example/deerflow_extension_example/plugin.py @@ -0,0 +1,176 @@ +"""The example's five deliberately small contribution implementations.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass, field +from threading import Lock +from typing import Any + +from deerflow_extension_api import ( + AgentBuildContext, + AgentScope, + ExtensionData, + ExtensionRuntimeDeps, + MiddlewarePlacement, + Placement, + SystemModelRequest, + SystemModelResult, + SystemOperationKind, + TaskInfo, + TaskOutcome, + task_store_from_runtime, +) +from fastapi import APIRouter, Depends, HTTPException +from langchain.agents.middleware import AgentMiddleware +from langgraph.prebuilt.tool_node import ToolCallRequest + + +@dataclass +class ExampleStats: + """Small extension-owned value used in both app and task stores.""" + + tool_calls: int = 0 + tasks: dict[str, int] = field(default_factory=dict) + system_model_calls: dict[str, dict[str, int]] = field(default_factory=dict) + _lock: Lock = field(default_factory=Lock, repr=False, compare=False) + + def note_tool_call(self) -> None: + with self._lock: + self.tool_calls += 1 + + def task_tool_calls(self) -> int: + with self._lock: + return self.tool_calls + + def absorb_task(self, tool_calls: int, outcome: TaskOutcome) -> None: + with self._lock: + self.tool_calls += tool_calls + key = outcome.value + self.tasks[key] = self.tasks.get(key, 0) + 1 + + def note_system_call(self, kind: SystemOperationKind, *, failed: bool) -> None: + with self._lock: + entry = self.system_model_calls.setdefault( + kind.value, + {"calls": 0, "errors": 0}, + ) + entry["calls"] += 1 + if failed: + entry["errors"] += 1 + + def snapshot(self) -> dict[str, Any]: + with self._lock: + return { + "tasks": dict(self.tasks), + "tool_calls": self.tool_calls, + "system_model_calls": {kind: dict(counts) for kind, counts in self.system_model_calls.items()}, + } + + +def _stats(store: ExtensionData) -> ExampleStats: + return store.get_or_init(ExampleStats, ExampleStats) + + +class ExampleMiddleware(AgentMiddleware): + async def awrap_tool_call( + self, + request: ToolCallRequest, + handler: Callable[[ToolCallRequest], Awaitable[Any]], + ) -> Any: + task_store = task_store_from_runtime(getattr(request, "runtime", None)) + task_stats = task_store.get(ExampleStats) if task_store is not None else None + if task_stats is not None: + task_stats.note_tool_call() + return await handler(request) + + +class ExampleMiddlewareContributor: + def contribute_middlewares( + self, + app_store: ExtensionData, + ctx: AgentBuildContext, + ) -> Sequence[MiddlewarePlacement]: + return ( + MiddlewarePlacement( + ExampleMiddleware(), + Placement.TOOL_VISIBLE, + AgentScope.BOTH, + ), + ) + + +class ExampleTaskLifecycle: + async def on_task_start( + self, + app_store: ExtensionData, + task_store: ExtensionData, + info: TaskInfo, + ) -> None: + task_store.set(ExampleStats()) + + async def on_task_stop( + self, + app_store: ExtensionData, + task_store: ExtensionData, + info: TaskInfo, + outcome: TaskOutcome, + ) -> None: + task_stats = task_store.remove(ExampleStats) + _stats(app_store).absorb_task( + task_stats.task_tool_calls() if task_stats is not None else 0, + outcome, + ) + + +class ExampleSystemObserver: + async def on_system_model_call( + self, + app_store: ExtensionData, + task_store: ExtensionData, + kind: SystemOperationKind, + request: SystemModelRequest, + result: SystemModelResult, + ) -> None: + _stats(app_store).note_system_call(kind, failed=result.error is not None) + + +class ExampleService: + def __init__(self) -> None: + self._deps: ExtensionRuntimeDeps | None = None + + async def start(self, deps: ExtensionRuntimeDeps) -> None: + self._deps = deps + + async def stop(self) -> None: + self._deps = None + + async def require_deps(self) -> ExtensionRuntimeDeps: + deps = self._deps + if deps is None or deps.app_store is None: + raise HTTPException( + status_code=503, + detail="extension-example is not running", + ) + return deps + + +def build_router(service: ExampleService) -> APIRouter: + """Build paths during registration, before runtime dependencies exist.""" + router = APIRouter(prefix="/api/extension-example", tags=["extension-example"]) + + @router.get("/stats") + async def read_stats( + deps: ExtensionRuntimeDeps = Depends(service.require_deps), + ) -> dict[str, Any]: + assert deps.app_store is not None + return { + "scope_id": deps.app_store.scope_id, + "session_factory_available": deps.session_factory is not None, + "host_policy": { + "max_subagents_per_run": deps.policy.max_subagents_per_run, + }, + **_stats(deps.app_store).snapshot(), + } + + return router diff --git a/examples/deerflow-extension-example/pyproject.toml b/examples/deerflow-extension-example/pyproject.toml new file mode 100644 index 000000000..0d9821664 --- /dev/null +++ b/examples/deerflow-extension-example/pyproject.toml @@ -0,0 +1,43 @@ +[project] +name = "deerflow-extension-example" +version = "0.1.0" +description = "A compact standalone example covering every DeerFlow extension contribution kind" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "deerflow-extension-api>=0.1.2,<0.2", + "fastapi>=0.115.0,<1", + "langchain>=1.3,<2", + "langgraph>=1.2.9,<1.3", +] + +[project.entry-points."deerflow.extensions"] +example = "deerflow_extension_example:install" + +[project.optional-dependencies] +dev = [ + "httpx>=0.28,<1", + "pytest>=9,<10", + "ruff>=0.14,<1", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["deerflow_extension_example"] + +[tool.ruff] +line-length = 240 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP"] + +[tool.ruff.lint.isort] +known-first-party = ["deerflow_extension_example"] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" diff --git a/examples/deerflow-extension-example/tests/test_entry_point.py b/examples/deerflow-extension-example/tests/test_entry_point.py new file mode 100644 index 000000000..6497a4e45 --- /dev/null +++ b/examples/deerflow-extension-example/tests/test_entry_point.py @@ -0,0 +1,11 @@ +from importlib.metadata import distribution + + +def test_installed_distribution_exposes_deerflow_extension_entry_point() -> None: + entry_points = [entry_point for entry_point in distribution("deerflow-extension-example").entry_points if entry_point.group == "deerflow.extensions"] + + assert [(entry_point.name, entry_point.value) for entry_point in entry_points] == [("example", "deerflow_extension_example:install")] + + install = entry_points[0].load() + assert install.__deerflow_api__ == "0.1.2" + assert install.__deerflow_name__ == "example" diff --git a/examples/deerflow-extension-example/tests/test_plugin.py b/examples/deerflow-extension-example/tests/test_plugin.py new file mode 100644 index 000000000..555c7858f --- /dev/null +++ b/examples/deerflow-extension-example/tests/test_plugin.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import Any + +from deerflow_extension_api import ( + EXTENSION_TASK_STORE_KEY, + AgentBuildContext, + AgentScope, + ExtensionData, + ExtensionRegistry, + ExtensionRuntimeDeps, + HostPolicySnapshot, + SystemModelRequest, + SystemModelResult, + SystemOperationKind, + TaskInfo, + TaskOutcome, +) +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient + +from deerflow_extension_example import install + + +class FakeRegistry: + def __init__(self) -> None: + self.middleware_contributors: list[Any] = [] + self.task_lifecycle_contributors: list[Any] = [] + self.system_model_observers: list[Any] = [] + self.services: list[Any] = [] + self.contributed_routers: list[Any] = [] + + def middlewares(self, contributor: Any) -> None: + self.middleware_contributors.append(contributor) + + def task_lifecycle(self, contributor: Any) -> None: + self.task_lifecycle_contributors.append(contributor) + + def system_model_observer(self, observer: Any) -> None: + self.system_model_observers.append(observer) + + def service(self, service: Any) -> None: + self.services.append(service) + + def routers(self, routers: Any) -> None: + self.contributed_routers.extend(routers) + + +@dataclass +class FakeRuntime: + context: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class FakeToolRequest: + runtime: FakeRuntime + + +def test_install_registers_all_five_contribution_kinds() -> None: + registry = FakeRegistry() + + install(registry, {}) + + assert isinstance(registry, ExtensionRegistry) + assert len(registry.middleware_contributors) == 1 + assert len(registry.task_lifecycle_contributors) == 1 + assert len(registry.system_model_observers) == 1 + assert len(registry.services) == 1 + assert len(registry.contributed_routers) == 1 + assert [route.path for route in registry.contributed_routers[0].routes] == ["/api/extension-example/stats"] + assert install.__deerflow_api__ == "0.1.2" + assert install.__deerflow_name__ == "example" + + +def test_disabled_extension_registers_nothing() -> None: + registry = FakeRegistry() + + install(registry, {"enabled": False}) + + assert registry.middleware_contributors == [] + assert registry.task_lifecycle_contributors == [] + assert registry.system_model_observers == [] + assert registry.services == [] + assert registry.contributed_routers == [] + + +def test_registered_contributions_publish_one_shared_stats_snapshot() -> None: + registry = FakeRegistry() + install(registry, {}) + app_store = ExtensionData("app") + task_store = ExtensionData("task-1") + task = TaskInfo( + task_id="task-1", + run_id="run-1", + thread_id="thread-1", + kind="lead", + ) + + async def exercise_contributions() -> tuple[int, int, dict[str, Any], int]: + lifecycle = registry.task_lifecycle_contributors[0] + await lifecycle.on_task_start(app_store, task_store, task) + placement = registry.middleware_contributors[0].contribute_middlewares( + app_store, + AgentBuildContext(scope=AgentScope.LEAD), + )[0] + + async def tool_handler(_request: object) -> str: + return "tool-result" + + request = FakeToolRequest(runtime=FakeRuntime(context={EXTENSION_TASK_STORE_KEY: task_store})) + assert await placement.middleware.awrap_tool_call(request, tool_handler) == "tool-result" + + await registry.system_model_observers[0].on_system_model_call( + app_store, + task_store, + SystemOperationKind.TITLE, + SystemModelRequest(messages="title prompt"), + SystemModelResult(error=RuntimeError("provider unavailable")), + ) + await lifecycle.on_task_stop( + app_store, + task_store, + task, + TaskOutcome.COMPLETED, + ) + + app = FastAPI() + app.include_router(registry.contributed_routers[0]) + async with AsyncClient( + transport=ASGITransport(app=app), + base_url="http://test", + ) as client: + before_start = await client.get("/api/extension-example/stats") + await registry.services[0].start( + ExtensionRuntimeDeps( + app_store=app_store, + policy=HostPolicySnapshot(max_subagents_per_run=6), + session_factory=object(), + ) + ) + response = await client.get("/api/extension-example/stats") + await registry.services[0].stop() + after_stop = await client.get("/api/extension-example/stats") + return ( + before_start.status_code, + response.status_code, + response.json(), + after_stop.status_code, + ) + + before_start, status_code, body, after_stop = asyncio.run(exercise_contributions()) + + assert before_start == 503 + assert status_code == 200 + assert after_stop == 503 + assert body == { + "scope_id": "app", + "session_factory_available": True, + "host_policy": {"max_subagents_per_run": 6}, + "tasks": {"completed": 1}, + "tool_calls": 1, + "system_model_calls": {"title": {"calls": 1, "errors": 1}}, + } diff --git a/scripts/serve.sh b/scripts/serve.sh index bbd86274a..82b419d3f 100755 --- a/scripts/serve.sh +++ b/scripts/serve.sh @@ -390,7 +390,7 @@ if ! $SKIP_INSTALL; then # `--all-packages` propagates extras into workspace members (deerflow-harness # in particular). Required for postgres extras — see PR #2584. # Intentionally unquoted to splat multiple `--extra X` pairs. - (cd backend && uv sync --quiet --all-packages $UV_EXTRAS_FLAGS) || { echo "✗ Backend dependency install failed"; exit 1; } + (cd backend && uv sync --locked --quiet --all-packages $UV_EXTRAS_FLAGS) || { echo "✗ Backend dependency install failed"; exit 1; } (cd frontend && "$DEERFLOW_PNPM_PYTHON" "$DEERFLOW_PNPM_RUNNER" install --silent) || { echo "✗ Frontend dependency install failed"; exit 1; } echo "✓ Dependencies synced" else @@ -464,7 +464,7 @@ mkdir -p temp/client_body_temp temp/proxy_temp temp/fastcgi_temp temp/uwsgi_temp # 1. Gateway API run_service "Gateway" \ - "cd backend && PYTHONPATH=. uv run uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 $GATEWAY_EXTRA_FLAGS > ../logs/gateway.log 2>&1" \ + "cd backend && PYTHONPATH=. uv run --no-sync uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 $GATEWAY_EXTRA_FLAGS > ../logs/gateway.log 2>&1" \ 8001 30 # 2. Frontend