From f17ca3777a1c14b40374f8c496f3f0cd8a54d487 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Sat, 12 Sep 2026 15:59:41 +0800 Subject: [PATCH] feat(extensions): add in-place upgrade that keeps private config (#5347) * feat(extensions): add in-place upgrade that keeps private config Replace a managed local snapshot or re-pin an already-installed requirement without going through remove, which dropped plugins[].config. * fix(extensions): keep snapshot, enabled, and git re-pin on upgrade Rollback keys off staging_root so a failed snapshot rename cannot rmtree the live tree. Upgrade preserves plugins[].enabled. Re-pin identification uses tool.uv.sources so git upgrades adopt the existing plugin record instead of failing closed after uv already switched the revision. * test(extensions): cover requirement re-pin identification on upgrade Re-pinning deerflow-extension-demo==2.0.0 to ==3.0.0 leaves added_names empty, so identification must take the added_specs fallback. Assert private config/required/enabled survive and the lock records 3.0.0. * fix(extensions): reject upgrade of an uninstalled git source Bare git+ URLs are not named Requirements, so the pre-uv-add installed check was skipped and upgrade acted as install. Resolve them against [tool.uv.sources] in the extensions group before uv add. --- AGENTS.md | 3 +- Makefile | 8 +- README.md | 3 +- backend/AGENTS.md | 1 + .../harness/deerflow/extensions/AGENTS.md | 20 +- .../harness/deerflow/extensions/cli.py | 26 ++ .../harness/deerflow/extensions/manager.py | 169 +++++++- .../tests/test_extension_dependency_sync.py | 19 +- backend/tests/test_extension_manager.py | 400 +++++++++++++++++- 9 files changed, 608 insertions(+), 41 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 10258920a..42fa3a779 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,7 +77,7 @@ Third-party extensions are loaded from a top-level `plugins:` list in `config.ya kept out of the API-writable `extensions_config.json`). Packaged extensions can contribute middleware, task lifecycle, system-model observers, Gateway services, and FastAPI HTTP routers; the [reference extension](examples/deerflow-extension-example/) demonstrates all -five. Manage them with `deerflow extensions install/list/enable/disable/remove` or the root +five. Manage them with `deerflow extensions install/upgrade/list/enable/disable/remove` or the root `make extension-*` wrappers. Every mutation requires a Gateway restart, and both build hooks and extension code execute with Gateway privileges, so only trusted operator sources belong in this path. The manager transaction, accepted source forms, lock discipline, and @@ -125,6 +125,7 @@ make config # Generate local config files from the examples make check # Check that required tools are installed make install # Install all dependencies (frontend + backend + pre-commit hooks) make extension-install SOURCE=... # Install and enable a trusted Python extension +make extension-upgrade SOURCE=... # Replace an installed extension and keep its config make extension-list # List configured Python extensions make extension-enable NAME=... # Enable an installed extension (restart required) make extension-disable NAME=... # Disable without uninstalling (restart required) diff --git a/Makefile b/Makefile index 1f1bd5cbe..df76b54c6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # DeerFlow - Unified Development Environment -.PHONY: help config config-upgrade check check-agent-guidance install extension-install extension-list extension-enable extension-disable extension-remove setup doctor support-bundle detect-thread-boundaries detect-blocking-io dev dev-daemon start start-daemon nginx stop up down clean docker-init docker-start docker-stop docker-logs docker-logs-frontend docker-logs-gateway docker-logs-redis setup-sandbox +.PHONY: help config config-upgrade check check-agent-guidance install extension-install extension-upgrade extension-list extension-enable extension-disable extension-remove setup doctor support-bundle detect-thread-boundaries detect-blocking-io dev dev-daemon start start-daemon nginx stop up down clean docker-init docker-start docker-stop docker-logs docker-logs-frontend docker-logs-gateway docker-logs-redis setup-sandbox BASH ?= bash BACKEND_UV_RUN = cd backend && uv run @@ -34,6 +34,7 @@ help: @echo " make detect-blocking-io - Inventory blocking IO that may block the backend event loop" @echo " make install - Install all dependencies (frontend + backend + pre-commit hooks)" @echo " make extension-install SOURCE=... - Install and enable a trusted Python extension" + @echo " make extension-upgrade SOURCE=... - Replace an installed extension and keep its config" @echo " make extension-list - List configured Python extensions" @echo " make extension-enable NAME=... - Enable an installed extension" @echo " make extension-disable NAME=... - Disable an extension without uninstalling it" @@ -113,6 +114,11 @@ extension-install: $(if $(and $(filter command line,$(origin SOURCE)),$(strip $(value SOURCE))),,$(error usage: make extension-install SOURCE=)) @cd backend && uv run --frozen --no-group extensions deerflow extensions install --source-env __deerflow_extension_source__ +extension-upgrade: export DEER_FLOW_EXTENSION_SOURCE := $(value SOURCE) +extension-upgrade: + $(if $(and $(filter command line,$(origin SOURCE)),$(strip $(value SOURCE))),,$(error usage: make extension-upgrade SOURCE=)) + @cd backend && uv run --frozen --no-group extensions deerflow extensions upgrade --source-env __deerflow_extension_source__ + extension-list: @cd backend && uv run --frozen --no-group extensions deerflow extensions list diff --git a/README.md b/README.md index 4e87a6393..e848ea910 100644 --- a/README.md +++ b/README.md @@ -1047,6 +1047,7 @@ make extension-install \ make extension-install SOURCE="$PWD/examples/deerflow-extension-example" make extension-list +make extension-upgrade SOURCE="$PWD/examples/deerflow-extension-example" make extension-disable NAME=acme make extension-enable NAME=acme make extension-remove NAME=acme @@ -1058,7 +1059,7 @@ source, automation can acknowledge that boundary explicitly with `cd backend && uv run --frozen --no-group extensions deerflow extensions install --yes`. The manager requires uv 0.8.0 or newer; the provided Docker images pin uv 0.11.1. The other direct -commands are `deerflow extensions list`, `enable NAME`, `disable NAME`, and `remove NAME`; +commands are `deerflow extensions upgrade SOURCE`, `list`, `enable NAME`, `disable NAME`, and `remove NAME`; `NAME` may be the extension name, Python distribution, or `module:install` value. Do not put credentials in a source URL — a URL carrying embedded userinfo or a credential-looking query parameter is rejected before uv runs. Remote Git sources must use public HTTPS; SSH diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 7a5b19849..7e4d91639 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -155,6 +155,7 @@ uv run pytest tests/test_bench_concurrency.py tests/test_bench_worker.py -q make check # Check system requirements make install # Install all dependencies (frontend + backend) make extension-install SOURCE=... # Install and enable a trusted Python extension +make extension-upgrade SOURCE=... # Replace an installed extension and keep its config make extension-list # List configured Python extensions make extension-enable NAME=... # Enable an installed extension make extension-disable NAME=... # Disable an extension without uninstalling it diff --git a/backend/packages/harness/deerflow/extensions/AGENTS.md b/backend/packages/harness/deerflow/extensions/AGENTS.md index 60fc2eaed..a5b34bbd7 100644 --- a/backend/packages/harness/deerflow/extensions/AGENTS.md +++ b/backend/packages/harness/deerflow/extensions/AGENTS.md @@ -11,8 +11,8 @@ Packaged extensions use one PEP 621 entry point in the `deerflow.extensions` group, for example `example = "deerflow_extension_example:install"`. The operator CLI is dispatched from the existing `deerflow` console script to `extensions/cli.py` and exposes only these -surfaces: `install SOURCE [--yes]`, `list`, `enable NAME`, `disable NAME`, and -`remove NAME`. `NAME` resolves against the entry-point name, distribution name, or +surfaces: `install SOURCE [--yes]`, `upgrade SOURCE [--yes]`, `list`, `enable NAME`, +`disable NAME`, and `remove NAME`. `NAME` resolves against the entry-point name, distribution name, or `module:install` value. The root `make extension-*` targets are convenience wrappers; because they execute from `backend/`, documentation should use absolute local source paths with `SOURCE=` unless backend-relative behavior is intentional. @@ -53,11 +53,15 @@ and `UV_INSECURE_HOST`, which would remove the TLS validation the HTTPS-only sou depends on; index, proxy, cache, and credential-provider settings remain available. The `--no-workspace` boundary requires uv 0.8.0 or newer. The stock Docker paths pin uv 0.11.1, and the manager fails before mutation when the host uv is older. -All install/remove/enable/disable mutations for a checkout hold the cross-process +All install/upgrade/remove/enable/disable mutations for a checkout hold the cross-process `.deer-flow/extension-manager.lock`; remove deactivates config before changing the package declaration, and rollback preserves a concurrent external config edit instead of replacing -it. The MVP has no in-place upgrade: operators retain private config, remove the old -package, install the new source pin, and restore that config. +it. Upgrade replaces a managed local snapshot (or re-pins a package requirement that is already +in the `extensions` group) and adopts the existing `plugins:` record so private `config`, +`required`, and `enabled` stay put. It fails closed if that local snapshot, requirement, or Git source is not +already installed; a plain `install` still refuses an already-snapshotted local directory. +Failed upgrades restore the previous snapshot even when a concurrent dependency-file edit +blocks lock/pyproject rollback, then leave that operator edit in place. Local-directory installs are snapshots, not editable links. The manager validates the source, derives the destination from the normalized distribution name, and copies it to @@ -126,9 +130,9 @@ newer uv can bump `uv.lock`'s `revision` (or make `uv lock --check` disagree wit generated elsewhere) while CI stays green, and the pinned uv in the production image then fails on the committed lock. `backend/tests/test_ci_uv_version_pin.py` keeps the four locations in step, which makes a uv upgrade one deliberate, reviewable change. -Rebuild the Gateway image after changing the managed set. Every install, enable, disable, -remove, or config mutation also requires a Gateway restart because plugin loading is -startup-only. +Rebuild the Gateway image after changing the managed set. Every install, upgrade, enable, +disable, remove, or config mutation also requires a Gateway restart because plugin loading +is startup-only. The root management wrappers bootstrap the checkout environment without the extension group via `uv run --frozen --no-group extensions`, so a broken or disappeared extension source cannot trigger project validation before the operator can list, disable, or remove it, while a diff --git a/backend/packages/harness/deerflow/extensions/cli.py b/backend/packages/harness/deerflow/extensions/cli.py index 3200ce56f..d349dda54 100644 --- a/backend/packages/harness/deerflow/extensions/cli.py +++ b/backend/packages/harness/deerflow/extensions/cli.py @@ -34,6 +34,17 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="abort Gateway startup when this extension fails to load (default: report and skip)", ) + upgrade = commands.add_parser( + "upgrade", + help="replace an installed extension source and keep its private config", + ) + upgrade.add_argument("--source-env", action="store_true", help=argparse.SUPPRESS) + upgrade.add_argument("source", help="local directory, Python package requirement, or Git URL") + upgrade.add_argument( + "--yes", + action="store_true", + help="acknowledge that upgrading an extension executes trusted third-party code", + ) disable = commands.add_parser("disable", help="disable an extension without uninstalling it") disable.add_argument("--name-env", action="store_true", help=argparse.SUPPRESS) disable.add_argument("name", help="extension name, distribution, or module:install entry point") @@ -68,6 +79,21 @@ def main(argv: Sequence[str] | None = None) -> int: installed = manager.install(source, yes=trusted, required=args.required) print(f"Installed and enabled {installed.name} ({installed.distribution}). Restart DeerFlow to load it.") return 0 + if args.command == "upgrade": + source = _source_argument(args) + trusted = args.yes + if not trusted: + print("Warning: a Python extension executes code with Gateway privileges.") + try: + trusted = input("Upgrade this trusted source? [y/N] ").strip().lower() in {"y", "yes"} + except EOFError: + trusted = False + if not trusted: + print("Extension upgrade cancelled.", file=sys.stderr) + return 2 + installed = manager.upgrade(source, yes=trusted) + print(f"Upgraded {installed.name} ({installed.distribution}). Restart DeerFlow to load it.") + return 0 if args.command == "disable": name = manager.set_enabled(_name_argument(args), enabled=False) print(f"Disabled {name}. Restart DeerFlow to apply the change.") diff --git a/backend/packages/harness/deerflow/extensions/manager.py b/backend/packages/harness/deerflow/extensions/manager.py index 533d306c5..854f4235d 100644 --- a/backend/packages/harness/deerflow/extensions/manager.py +++ b/backend/packages/harness/deerflow/extensions/manager.py @@ -131,14 +131,19 @@ class ExtensionManager: selected_config = root_config if root_config.is_file() or not legacy_config.is_file() else legacy_config self.config_path = selected_config.resolve() - def install(self, source: str, *, yes: bool = False, required: bool = False) -> InstalledExtension: + def install(self, source: str, *, yes: bool = False, required: bool = False, replace: bool = False) -> InstalledExtension: """Install an extension source and enable its packaging entry point.""" with _manager_lock(self.project_root): - return self._install(source, yes=yes, required=required) + return self._install(source, yes=yes, required=required, replace=replace) - def _install(self, source: str, *, yes: bool, required: bool) -> InstalledExtension: + def upgrade(self, source: str, *, yes: bool = False) -> InstalledExtension: + """Replace an installed extension source without dropping its private config or enabled state.""" + return self.install(source, yes=yes, replace=True) + + def _install(self, source: str, *, yes: bool, required: bool, replace: bool) -> InstalledExtension: if not yes: - raise PermissionError("installing an extension executes trusted third-party code; pass yes=True to continue") + action = "upgrading" if replace else "installing" + raise PermissionError(f"{action} an extension executes trusted third-party code; pass yes=True to continue") source_argument = Path(source).expanduser() if _is_link_like(source_argument): @@ -156,29 +161,55 @@ class ExtensionManager: managed_source = (managed_root / normalized_distribution).resolve() if not managed_source.is_relative_to(managed_root): raise ValueError(f"invalid extension distribution name: {distribution!r}") - if managed_source.exists(): + if managed_source.exists() and not replace: raise FileExistsError(f"extension source is already installed: {managed_source}") + if replace and not managed_source.exists(): + raise ValueError(f"extension source is not installed: {managed_source}; use install") uv_source = str(managed_source.relative_to(self.backend_dir)) else: if source_argument.exists(): raise ValueError("local extension sources must be directories so they can be snapshotted for deployment") _validate_remote_source(source) + if replace: + try: + remote_distribution = _normalize_distribution(Requirement(source).name) + except InvalidRequirement: + remote_distribution = None + if remote_distribution is not None: + if remote_distribution not in _extension_dependency_names(self.pyproject_path): + raise ValueError(f"extension {remote_distribution!r} is not installed; use install") + elif _installed_git_distribution(source, self.pyproject_path) is None: + # Bare git+ URLs are not named Requirements; resolve them + # against the already-installed extensions group instead. + raise ValueError("extension source is not installed; use install") # uv add/sync execute the package's build backend. A config this manager # could never write to must fail before that code runs, not afterwards # through rollback. self._read_plugins() _require_supported_uv(self.backend_dir) - dependencies_before = _extension_dependency_names(self.pyproject_path) + specs_before = _extension_dependencies(self.pyproject_path) + sources_before = _uv_sources(self.pyproject_path) dependency_snapshots = ( _FileSnapshot.capture(self.pyproject_path), _FileSnapshot.capture(self.backend_dir / "uv.lock"), ) managed_dependency_contents: tuple[bytes | None, ...] | None = None uv_attempted = False + staging_root: Path | None = None + staged_source: Path | None = None try: if managed_source is not None: managed_source.parent.mkdir(parents=True, exist_ok=True) + if replace and managed_source.exists(): + staging_root = Path( + tempfile.mkdtemp( + prefix=f".{managed_source.name}.upgrade-", + dir=managed_source.parent, + ) + ) + staged_source = staging_root / "source" + managed_source.rename(staged_source) shutil.copytree( source_path, managed_source, @@ -211,10 +242,12 @@ class ExtensionManager: _validate_locked_local_sources(self.backend_dir / "uv.lock", self.backend_dir) _sync_environment(self.project_root, self.backend_dir, self.config_path) if metadata is None: - added = _extension_dependency_names(self.pyproject_path) - dependencies_before - if len(added) != 1: - raise RuntimeError("could not identify the distribution added by uv") - distribution = next(iter(added)) + distribution = _identify_uv_added_distribution( + self.pyproject_path, + specs_before=specs_before, + sources_before=sources_before, + replace=replace, + ) name, use = _discover_installed_entry_point(self.backend_dir, distribution) metadata = (distribution, name, use) else: @@ -230,8 +263,11 @@ class ExtensionManager: "enabled": True, "required": required, "config": {}, - } + }, + preserve_enabled=replace, ) + if staging_root is not None: + shutil.rmtree(staging_root, ignore_errors=True) except BaseException as operation_error: # _enable_plugin performs the only config mutation as the final, # atomic step. A failure before it must not roll back an operator @@ -245,12 +281,21 @@ class ExtensionManager: strict=True, ) ) + if staging_root is not None: + # Key restore off staging, not a flag set after rename: a failed + # rename must leave the live snapshot in place and only remove + # the empty .*.upgrade-* directory. + if staged_source is not None and staged_source.exists(): + if managed_source is not None: + shutil.rmtree(managed_source, ignore_errors=True) + staged_source.rename(managed_source) + shutil.rmtree(staging_root, ignore_errors=True) + elif managed_source is not None and not dependency_recovery_conflict: + shutil.rmtree(managed_source, ignore_errors=True) if dependency_recovery_conflict: raise RuntimeError("extension installation recovery preserved a concurrent dependency-file edit") from operation_error for snapshot in dependency_snapshots: snapshot.restore() - if managed_source is not None: - shutil.rmtree(managed_source, ignore_errors=True) # The recovery sync itself may rewrite the dependency files, so the # second restore has to run even when that sync fails. try: @@ -406,7 +451,7 @@ class ExtensionManager: ) return tuple(configured) - def _enable_plugin(self, plugin: dict[str, Any]) -> None: + def _enable_plugin(self, plugin: dict[str, Any], *, preserve_enabled: bool = False) -> None: original, plugins = self._read_plugins() exact_use_matches = [item for item in plugins if isinstance(item, dict) and item.get("use") == plugin["use"]] identity_conflicts = [item for item in plugins if isinstance(item, dict) and item not in exact_use_matches and (item.get("name") == plugin["name"] or _same_distribution(item.get("package"), plugin["package"]))] @@ -420,7 +465,8 @@ class ExtensionManager: existing["name"] = plugin["name"] existing["package"] = plugin["package"] existing["use"] = plugin["use"] - existing["enabled"] = True + if not preserve_enabled: + existing["enabled"] = True existing.setdefault("required", plugin["required"]) existing.setdefault("config", {}) _write_plugins_block(self.config_path, original, plugins) @@ -615,6 +661,22 @@ def _strip_git_prefix(reference: str) -> str: return reference[4:] if reference.lower().startswith("git+") else reference +def _git_repository_identity(reference: str) -> tuple[str, int | None, str] | None: + """Host, port, and path that identify a Git repo, ignoring ref and fragment.""" + parsed = urllib.parse.urlsplit(_strip_git_prefix(reference.strip())) + if parsed.scheme.lower() not in {"http", "https"}: + return None + host = parsed.hostname + if host is None: + return None + path = parsed.path.rsplit("@", 1)[0].rstrip("/") + if path.endswith(".git"): + path = path[:-4] + if not path: + return None + return (host.lower(), parsed.port, path) + + def _is_scp_like_reference(source: str) -> bool: # The bare shorthand is checked directly; a PEP 508 direct reference keeps # it behind the requirement name, which packaging strips off the URL. @@ -639,20 +701,83 @@ def _normalize_query_key(key: str) -> str: return re.sub(r"[^A-Za-z0-9]+", "-", camel_case_split).strip("-").lower() -def _extension_dependency_names(pyproject: Path) -> set[str]: +def _extension_dependencies(pyproject: Path) -> tuple[str, ...]: with pyproject.open("rb") as stream: document = tomllib.load(stream) dependencies = document.get("dependency-groups", {}).get("extensions", []) + return tuple(dependency for dependency in dependencies if isinstance(dependency, str)) + + +def _uv_sources(pyproject: Path) -> dict[str, Any]: + with pyproject.open("rb") as stream: + document = tomllib.load(stream) + sources = document.get("tool", {}).get("uv", {}).get("sources", {}) + return sources if isinstance(sources, dict) else {} + + +def _distribution_name_from_spec(spec: str) -> str | None: + match = re.match(r"\s*([A-Za-z0-9][A-Za-z0-9._-]*)", spec) + if match is None: + return None + return _normalize_distribution(match.group(1)) + + +def _identify_uv_added_distribution( + pyproject: Path, + *, + specs_before: tuple[str, ...], + sources_before: dict[str, Any], + replace: bool, +) -> str: + names_before: set[str] = set() + for spec in specs_before: + name = _distribution_name_from_spec(spec) + if name is not None: + names_before.add(name) + added_names = _extension_dependency_names(pyproject) - names_before + if len(added_names) == 1: + return next(iter(added_names)) + added_specs = [spec for spec in _extension_dependencies(pyproject) if spec not in specs_before] + if len(added_specs) == 1: + name = _distribution_name_from_spec(added_specs[0]) + if name is not None: + return name + if replace: + changed_sources = [name for name, source in _uv_sources(pyproject).items() if isinstance(name, str) and sources_before.get(name) != source] + if len(changed_sources) == 1: + return _normalize_distribution(changed_sources[0]) + raise RuntimeError("could not identify the distribution added by uv") + + +def _extension_dependency_names(pyproject: Path) -> set[str]: names: set[str] = set() - for dependency in dependencies: - if not isinstance(dependency, str): - continue - match = re.match(r"\s*([A-Za-z0-9][A-Za-z0-9._-]*)", dependency) - if match: - names.add(_normalize_distribution(match.group(1))) + for dependency in _extension_dependencies(pyproject): + name = _distribution_name_from_spec(dependency) + if name is not None: + names.add(name) return names +def _installed_git_distribution(source: str, pyproject: Path) -> str | None: + """Return the extensions-group distribution already pinned to this Git repo.""" + requested = _git_repository_identity(source) + if requested is None: + return None + installed = _extension_dependency_names(pyproject) + for name, declared in _uv_sources(pyproject).items(): + if not isinstance(name, str) or not isinstance(declared, dict): + continue + git_url = declared.get("git") + if not isinstance(git_url, str): + continue + if _git_repository_identity(git_url) != requested: + continue + normalized = _normalize_distribution(name) + if normalized in installed: + return normalized + return None + + _LOCK_LOCAL_PATH_KEYS = frozenset({"path", "directory", "editable", "virtual"}) _LOCK_LOCAL_URL_KEYS = frozenset({"registry", "url", "git"}) _LOCK_LOCAL_SOURCE_VIOLATION = "uv.lock contains a local dependency source outside the backend Docker build context" diff --git a/backend/tests/test_extension_dependency_sync.py b/backend/tests/test_extension_dependency_sync.py index 8444d44bc..22f7194b2 100644 --- a/backend/tests/test_extension_dependency_sync.py +++ b/backend/tests/test_extension_dependency_sync.py @@ -86,6 +86,14 @@ def test_root_makefile_exposes_extension_management_commands() -> None: assert "uv run --frozen --no-group extensions" in install assert "--yes" not in install + upgrade = _make_recipe(makefile, "extension-upgrade") + assert "deerflow extensions upgrade" in upgrade + assert "--source-env __deerflow_extension_source__" in upgrade + assert "DEER_FLOW_EXTENSION_SOURCE" not in upgrade + assert "$(SOURCE)" not in upgrade + assert "uv run --frozen --no-group extensions" in upgrade + assert "--yes" not in upgrade + for target, command in ( ("extension-list", "deerflow extensions list"), ("extension-enable", "deerflow extensions enable"), @@ -102,6 +110,7 @@ def test_extension_management_bootstrap_does_not_resolve_a_broken_extension_sour for target in ( "extension-install", + "extension-upgrade", "extension-list", "extension-enable", "extension-disable", @@ -180,6 +189,7 @@ def test_root_extension_shortcuts_are_cross_platform_and_keep_trust_confirmation for target in ( "extension-install", + "extension-upgrade", "extension-enable", "extension-disable", "extension-remove", @@ -189,12 +199,17 @@ def test_root_extension_shortcuts_are_cross_platform_and_keep_trust_confirmation assert "usage: make" in recipe, target assert "--yes" not in _make_recipe(makefile, "extension-install") + assert "--yes" not in _make_recipe(makefile, "extension-upgrade") def test_root_extension_shortcuts_reject_ambient_environment_arguments() -> None: environment = os.environ.copy() - for target, variable in (("extension-install", "SOURCE"), ("extension-enable", "NAME")): + for target, variable in ( + ("extension-install", "SOURCE"), + ("extension-upgrade", "SOURCE"), + ("extension-enable", "NAME"), + ): environment[variable] = "ambient-value" result = subprocess.run( ["make", "--no-print-directory", "-n", target], @@ -212,6 +227,7 @@ def test_root_extension_shortcuts_reject_ambient_environment_arguments() -> None ("target", "variable", "env_option"), [ ("extension-install", "SOURCE", "--source-env __deerflow_extension_source__"), + ("extension-upgrade", "SOURCE", "--source-env __deerflow_extension_source__"), ("extension-enable", "NAME", "--name-env __deerflow_extension_name__"), ("extension-disable", "NAME", "--name-env __deerflow_extension_name__"), ("extension-remove", "NAME", "--name-env __deerflow_extension_name__"), @@ -244,6 +260,7 @@ def test_root_extension_shortcuts_keep_command_line_arguments_out_of_the_shell_r ("target", "variable", "env_option"), [ ("extension-install", "SOURCE", "--source-env __deerflow_extension_source__"), + ("extension-upgrade", "SOURCE", "--source-env __deerflow_extension_source__"), ("extension-enable", "NAME", "--name-env __deerflow_extension_name__"), ("extension-disable", "NAME", "--name-env __deerflow_extension_name__"), ("extension-remove", "NAME", "--name-env __deerflow_extension_name__"), diff --git a/backend/tests/test_extension_manager.py b/backend/tests/test_extension_manager.py index 7dcd0c692..e76b09575 100644 --- a/backend/tests/test_extension_manager.py +++ b/backend/tests/test_extension_manager.py @@ -166,13 +166,16 @@ def _assert_demo_entry_point_loads(backend: Path) -> None: assert completed.returncode == 0, completed.stderr -def _write_demo_wheel(directory: Path) -> Path: - directory.mkdir() - wheel = directory / "deerflow_extension_demo-1.0.0-py3-none-any.whl" - dist_info = "deerflow_extension_demo-1.0.0.dist-info" +def _write_demo_wheel(directory: Path, *, version: str = "1.0.0", marker: str | None = None) -> Path: + directory.mkdir(parents=True, exist_ok=True) + wheel = directory / f"deerflow_extension_demo-{version}-py3-none-any.whl" + dist_info = f"deerflow_extension_demo-{version}.dist-info" + init = "def install(registry, config):\n return None\n" + if marker is not None: + init = f"MARKER = {marker!r}\n{init}" records = { - "demo_extension/__init__.py": "def install(registry, config):\n return None\n", - f"{dist_info}/METADATA": ("Metadata-Version: 2.1\nName: deerflow-extension-demo\nVersion: 1.0.0\nRequires-Python: >=3.12\n"), + "demo_extension/__init__.py": init, + f"{dist_info}/METADATA": (f"Metadata-Version: 2.1\nName: deerflow-extension-demo\nVersion: {version}\nRequires-Python: >=3.12\n"), f"{dist_info}/WHEEL": ("Wheel-Version: 1.0\nGenerator: deerflow-extension-test\nRoot-Is-Purelib: true\nTag: py3-none-any\n"), f"{dist_info}/entry_points.txt": ("[deerflow.extensions]\ndemo = demo_extension:install\n"), } @@ -219,6 +222,389 @@ def test_install_local_directory_makes_it_deployable_and_enabled(tmp_path: Path) _assert_demo_entry_point_loads(root / "backend") +def test_install_rejects_an_already_snapshotted_local_directory(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + manager = ExtensionManager(root) + manager.install(str(source), yes=True) + + with pytest.raises(FileExistsError, match="already installed"): + manager.install(str(source), yes=True) + + +def test_upgrade_replaces_local_snapshot_and_preserves_private_config(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + manager = ExtensionManager(root) + manager.install(str(source), yes=True) + config_path = root / "config.yaml" + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + config["plugins"][0]["required"] = True + config["plugins"][0]["config"] = {"label": "keep-this"} + config["plugins"][0]["enabled"] = False + config_path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + + (source / "demo_extension" / "__init__.py").write_text( + "MARKER = 'v2'\ndef install(registry, config):\n return None\n", + encoding="utf-8", + ) + + result = manager.upgrade(str(source), yes=True) + + assert result.name == "demo" + managed = root / "backend" / "extensions" / "sources" / "deerflow-extension-demo" + assert "MARKER = 'v2'" in (managed / "demo_extension" / "__init__.py").read_text(encoding="utf-8") + plugins = yaml.safe_load(config_path.read_text(encoding="utf-8"))["plugins"] + assert plugins == [ + { + "name": "demo", + "package": "deerflow-extension-demo", + "use": "demo_extension:install", + "enabled": False, + "required": True, + "config": {"label": "keep-this"}, + } + ] + _assert_demo_entry_point_loads(root / "backend") + + +def test_failed_upgrade_restores_the_previous_snapshot_and_config(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + manager = ExtensionManager(root) + manager.install(str(source), yes=True) + config_path = root / "config.yaml" + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + config["plugins"][0]["config"] = {"label": "keep-this"} + original_config = yaml.safe_dump(config, sort_keys=False) + config_path.write_text(original_config, encoding="utf-8") + original_init = (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo" / "demo_extension" / "__init__.py").read_text(encoding="utf-8") + original_pyproject = (root / "backend" / "pyproject.toml").read_bytes() + + broken = tmp_path / "broken-source" + broken.mkdir() + _write_local_extension(broken, entry_target="missing_demo_extension:install") + + with pytest.raises(ValueError, match="could not be loaded"): + manager.upgrade(str(broken), yes=True) + + managed = root / "backend" / "extensions" / "sources" / "deerflow-extension-demo" + assert (managed / "demo_extension" / "__init__.py").read_text(encoding="utf-8") == original_init + assert (root / "backend" / "pyproject.toml").read_bytes() == original_pyproject + assert yaml.safe_load(config_path.read_text(encoding="utf-8"))["plugins"][0]["config"] == {"label": "keep-this"} + leftover = list((root / "backend" / "extensions" / "sources").glob(".*.upgrade-*")) + assert leftover == [] + + +def test_deerflow_extensions_upgrade_exposes_the_local_replace_flow( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(root)) + assert deerflow_main(["extensions", "install", str(source), "--yes"]) == 0 + capsys.readouterr() + + (source / "demo_extension" / "__init__.py").write_text( + "MARKER = 'v2'\ndef install(registry, config):\n return None\n", + encoding="utf-8", + ) + exit_code = deerflow_main(["extensions", "upgrade", str(source), "--yes"]) + + assert exit_code == 0 + assert "Upgraded demo" in capsys.readouterr().out + managed = root / "backend" / "extensions" / "sources" / "deerflow-extension-demo" + assert "MARKER = 'v2'" in (managed / "demo_extension" / "__init__.py").read_text(encoding="utf-8") + + +def test_upgrade_rejects_a_local_source_that_is_not_installed(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + + with pytest.raises(ValueError, match="not installed"): + ExtensionManager(root).upgrade(str(source), yes=True) + + assert not (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo").exists() + assert yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8")).get("plugins") is None + + +def test_upgrade_rejects_a_requirement_that_is_not_installed(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + + with pytest.raises(ValueError, match="not installed"): + ExtensionManager(root).upgrade("deerflow-extension-demo==2.0.0", yes=True) + + +def test_upgrade_rejects_a_git_source_that_is_not_installed(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + root.mkdir() + _write_host_project(root) + pyproject = root / "backend" / "pyproject.toml" + before = pyproject.read_text(encoding="utf-8") + + with pytest.raises(ValueError, match="not installed"): + ExtensionManager(root).upgrade( + "git+https://github.com/acme/deerflow-extension-demo.git@main", + yes=True, + ) + + assert pyproject.read_text(encoding="utf-8") == before + assert yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8")).get("plugins") is None + + +def test_upgrade_repins_an_installed_git_source_and_preserves_private_config(tmp_path: Path) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-git-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + (source / "demo_extension" / "__init__.py").write_text( + "MARKER = 'v1'\ndef install(registry, config):\n return None\n", + encoding="utf-8", + ) + first_revision = _commit_local_extension(source) + (source / "demo_extension" / "__init__.py").write_text( + "MARKER = 'v2'\ndef install(registry, config):\n return None\n", + encoding="utf-8", + ) + test_hooks = source / ".git" / "test-hooks" + subprocess.run(["git", "add", "."], cwd=source, check=True) + subprocess.run( + ["git", "-c", f"core.hooksPath={test_hooks}", "commit", "-qm", "upgrade pin"], + cwd=source, + check=True, + ) + second_revision = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=source, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + bare_repository = tmp_path / "demo.git" + subprocess.run(["git", "clone", "-q", "--bare", str(source), str(bare_repository)], check=True) + subprocess.run(["git", "--git-dir", str(bare_repository), "update-server-info"], check=True) + + with _serve_directory(tmp_path) as base_url: + manager = ExtensionManager(root) + manager.install(f"git+{base_url}/demo.git@{first_revision}", yes=True) + config_path = root / "config.yaml" + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + config["plugins"][0]["required"] = True + config["plugins"][0]["config"] = {"label": "keep-this"} + config["plugins"][0]["enabled"] = False + config_path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + + result = manager.upgrade(f"git+{base_url}/demo.git@{second_revision}", yes=True) + + _assert_demo_entry_point_loads(root / "backend") + marker = subprocess.run( + [ + str(root / "backend" / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python")), + "-c", + "import demo_extension; print(demo_extension.MARKER)", + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + assert result.name == "demo" + assert marker == "v2" + assert second_revision in (root / "backend" / "uv.lock").read_text(encoding="utf-8") + assert not (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo").exists() + plugins = yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8"))["plugins"] + assert plugins == [ + { + "name": "demo", + "package": "deerflow-extension-demo", + "use": "demo_extension:install", + "enabled": False, + "required": True, + "config": {"label": "keep-this"}, + } + ] + + +def test_upgrade_repins_an_installed_requirement_and_preserves_private_config( + tmp_path: Path, + monkeypatch, +) -> None: + """Re-pinning ==2.0.0 to ==3.0.0 keeps the same distribution name. + + added_names is empty; identification must take the added_specs fallback so + private config/required/enabled survive the lock re-pin. + """ + root = tmp_path / "deer-flow" + simple_root = tmp_path / "simple" + package_dir = simple_root / "deerflow-extension-demo" + root.mkdir() + _write_host_project(root) + _write_demo_wheel(package_dir, version="2.0.0", marker="v2") + _write_demo_wheel(package_dir, version="3.0.0", marker="v3") + (package_dir / "index.html").write_text( + """\ + + +deerflow_extension_demo-2.0.0-py3-none-any.whl +deerflow_extension_demo-3.0.0-py3-none-any.whl + +""", + encoding="utf-8", + ) + + with _serve_directory(simple_root) as index_url: + monkeypatch.setenv("UV_DEFAULT_INDEX", index_url) + manager = ExtensionManager(root) + manager.install("deerflow-extension-demo==2.0.0", yes=True) + config_path = root / "config.yaml" + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + config["plugins"][0]["required"] = True + config["plugins"][0]["config"] = {"label": "keep-this"} + config["plugins"][0]["enabled"] = False + config_path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + + result = manager.upgrade("deerflow-extension-demo==3.0.0", yes=True) + + _assert_demo_entry_point_loads(root / "backend") + marker = subprocess.run( + [ + str(root / "backend" / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python")), + "-c", + "import demo_extension; print(demo_extension.MARKER)", + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + pyproject = (root / "backend" / "pyproject.toml").read_text(encoding="utf-8") + lock = (root / "backend" / "uv.lock").read_text(encoding="utf-8") + assert result.name == "demo" + assert marker == "v3" + assert "deerflow-extension-demo==3.0.0" in pyproject + assert "deerflow-extension-demo==2.0.0" not in pyproject + assert re.search(r'name = "deerflow-extension-demo"\s+version = "3.0.0"', lock) is not None + assert not (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo").exists() + plugins = yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8"))["plugins"] + assert plugins == [ + { + "name": "demo", + "package": "deerflow-extension-demo", + "use": "demo_extension:install", + "enabled": False, + "required": True, + "config": {"label": "keep-this"}, + } + ] + + +def test_failed_upgrade_leaves_snapshot_when_staging_rename_fails(tmp_path: Path, monkeypatch) -> None: + """A snapshot that cannot be moved must stay the live tree. + + Path.rename can fail after mkdtemp (file held open, Windows AV). Treating + that like a failed install would rmtree the original snapshot that was + never replaced. + """ + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + manager = ExtensionManager(root) + manager.install(str(source), yes=True) + config_path = root / "config.yaml" + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + config["plugins"][0]["config"] = {"label": "keep-this"} + config_path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + managed = root / "backend" / "extensions" / "sources" / "deerflow-extension-demo" + original_init = (managed / "demo_extension" / "__init__.py").read_text(encoding="utf-8") + + (source / "demo_extension" / "__init__.py").write_text( + "MARKER = 'v2'\ndef install(registry, config):\n return None\n", + encoding="utf-8", + ) + original_rename = Path.rename + + def _rename(self, target): + if self.resolve() == managed.resolve(): + raise OSError("snapshot file in use") + return original_rename(self, target) + + monkeypatch.setattr("deerflow.extensions.manager.Path.rename", _rename) + + with pytest.raises(OSError, match="snapshot file in use"): + manager.upgrade(str(source), yes=True) + + assert (managed / "demo_extension" / "__init__.py").read_text(encoding="utf-8") == original_init + assert yaml.safe_load(config_path.read_text(encoding="utf-8"))["plugins"][0]["config"] == {"label": "keep-this"} + assert list((root / "backend" / "extensions" / "sources").glob(".*.upgrade-*")) == [] + + +def test_failed_upgrade_restores_snapshot_when_a_concurrent_dependency_edit_blocks_lock_rollback( + tmp_path: Path, + monkeypatch, +) -> None: + root = tmp_path / "deer-flow" + source = tmp_path / "demo-source" + root.mkdir() + source.mkdir() + _write_host_project(root) + _write_local_extension(source) + manager = ExtensionManager(root) + manager.install(str(source), yes=True) + original_init = (root / "backend" / "extensions" / "sources" / "deerflow-extension-demo" / "demo_extension" / "__init__.py").read_text(encoding="utf-8") + pyproject_path = root / "backend" / "pyproject.toml" + + (source / "demo_extension" / "__init__.py").write_text( + "MARKER = 'v2'\ndef install(registry, config):\n return None\n", + encoding="utf-8", + ) + + def _fail_after_operator_edit(*_args, **_kwargs): + pyproject_path.write_text( + pyproject_path.read_text(encoding="utf-8") + "\n# operator edit during upgrade\n", + encoding="utf-8", + ) + raise RuntimeError("simulated dependency sync failure") + + monkeypatch.setattr("deerflow.extensions.manager._sync_environment", _fail_after_operator_edit) + + with pytest.raises(RuntimeError, match="recovery.*dependency"): + manager.upgrade(str(source), yes=True) + + managed = root / "backend" / "extensions" / "sources" / "deerflow-extension-demo" + assert (managed / "demo_extension" / "__init__.py").read_text(encoding="utf-8") == original_init + assert "# operator edit during upgrade" in pyproject_path.read_text(encoding="utf-8") + assert list((root / "backend" / "extensions" / "sources").glob(".*.upgrade-*")) == [] + + def test_install_defaults_to_a_fail_open_plugin_record(tmp_path: Path) -> None: """A managed install must not silently choose the fail-closed side: with `required: true`, a later broken extension aborts Gateway startup entirely, @@ -292,7 +678,7 @@ def test_mutating_operations_are_serialized_for_one_checkout(tmp_path: Path, mon release_first = threading.Event() second_entered = threading.Event() - def _fake_install(self, source: str, *, yes: bool, required: bool): + def _fake_install(self, source: str, *, yes: bool, required: bool, replace: bool = False): if source == "first": first_entered.set() assert release_first.wait(timeout=5)