From 0b780518405e280e57631b07ee44a83f6cdd5018 Mon Sep 17 00:00:00 2001 From: Wenchao An Date: Mon, 12 Oct 2026 22:51:20 +0800 Subject: [PATCH] feat(extensions): add example guide and packaging checks (#6755) * feat(extensions): add example guide and packaging checks * fix(extensions): align packaging checks with installer Reject Windows drive-qualified wheel paths before extraction and reuse the installer entry-point validator. Cover unsafe paths, dotted attributes, and valid dotted modules in regression tests. * fix(extensions): skip unused wheel extraction Only extract wheels when browser manifests need validation, with a backend-only regression test. Link the three remaining example READMEs in the developer guide. * docs(frontend): trim guidance intro below size budget --------- Co-authored-by: Willem Jiang --- backend/tests/test_extension_check_script.py | 143 ++++++++++++++ examples/README.md | 58 ++++++ .../deerflow-extension-bookmarks/README.md | 3 + examples/deerflow-extension-example/README.md | 3 + .../en/harness/extensions/quick-start.mdx | 16 ++ .../zh/harness/extensions/quick-start.mdx | 14 ++ scripts/AGENTS.md | 12 +- scripts/check_extension.py | 176 ++++++++++++++++++ 8 files changed, 420 insertions(+), 5 deletions(-) create mode 100644 backend/tests/test_extension_check_script.py create mode 100644 examples/README.md create mode 100644 scripts/check_extension.py diff --git a/backend/tests/test_extension_check_script.py b/backend/tests/test_extension_check_script.py new file mode 100644 index 0000000000..e51cab8ae8 --- /dev/null +++ b/backend/tests/test_extension_check_script.py @@ -0,0 +1,143 @@ +"""Packaging checks catch defects before a trusted extension is installed.""" + +import importlib.util +import json +from pathlib import Path +from zipfile import ZipFile + +import pytest + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "check_extension.py" +spec = importlib.util.spec_from_file_location("extension_check_script", SCRIPT) +checker = importlib.util.module_from_spec(spec) +spec.loader.exec_module(checker) + + +def wheel( + tmp_path, + *, + entry="example = example:install", + requirement="deerflow-extension-api>=0.2,<0.3", + files=None, +): + path = tmp_path / "example-0.1.0-py3-none-any.whl" + with ZipFile(path, "w") as archive: + archive.writestr( + "example-0.1.0.dist-info/METADATA", + f"Metadata-Version: 2.3\nName: example\nVersion: 0.1.0\nRequires-Dist: {requirement}\n", + ) + archive.writestr( + "example-0.1.0.dist-info/entry_points.txt", + f"[deerflow.extensions]\n{entry}\n", + ) + archive.writestr( + "example/__init__.py", + "raise RuntimeError('The checker must not import plugin code')\n", + ) + for name, content in (files or {}).items(): + archive.writestr(name, content) + return path + + +def test_packaged_entry_is_checked_without_importing_plugin(tmp_path): + checker.check_wheel(tmp_path, wheel(tmp_path)) + + +def test_backend_only_wheel_skips_extraction(tmp_path, monkeypatch): + def fail_if_extracted(*args, **kwargs): + pytest.fail("Backend-only wheels must not be extracted") + + monkeypatch.setattr(ZipFile, "extractall", fail_if_extracted) + checker.check_wheel(tmp_path, wheel(tmp_path)) + + +@pytest.mark.parametrize( + ("entry", "message"), + [ + ("example = missing:install", "entry module"), + ("example = example", "entry point target"), + ("example = example:factory.install", "entry point target"), + ("example = example:install\nsecond = example:install", "exactly one"), + ], +) +def test_invalid_packaged_entry_is_rejected(tmp_path, entry, message): + with pytest.raises(ValueError, match=message): + checker.check_wheel(tmp_path, wheel(tmp_path, entry=entry)) + + +def test_dotted_entry_module_is_accepted_without_importing_plugin(tmp_path): + checker.check_wheel( + tmp_path, + wheel(tmp_path, entry="example = example.nested:install", files={"example/nested.py": "raise RuntimeError('The checker must not import plugin code')\n"}), + ) + + +@pytest.mark.parametrize( + "requirement", + [ + "unrelated>=1", + "deerflow-extension-api>=99", + 'deerflow-extension-api>=0.2; extra == "optional"', + ], +) +def test_missing_or_incompatible_api_dependency_is_rejected(tmp_path, requirement): + with pytest.raises(ValueError, match="extension-api"): + checker.check_wheel(tmp_path, wheel(tmp_path, requirement=requirement)) + + +def test_dropped_manifest_is_detected_from_src_layout(tmp_path): + package = tmp_path / "src" / "example" + package.mkdir(parents=True) + (package / "ui_manifest.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError, match="manifest.*not packaged"): + checker.check_wheel(tmp_path, wheel(tmp_path)) + + +def test_packaged_assets_use_host_manifest_validation(tmp_path): + manifest = json.dumps({"schema_version": 1, "entry": "main.mjs", "files": ["main.mjs", "styles.css"]}) + files = { + "example/ui_manifest.json": manifest, + "example/main.mjs": "export default {};", + "example/styles.css": "body {}", + } + checker.check_wheel(tmp_path, wheel(tmp_path, files=files)) + del files["example/styles.css"] + with pytest.raises(FileNotFoundError): + checker.check_wheel(tmp_path, wheel(tmp_path, files=files)) + + +def test_custom_manifest_can_be_checked_explicitly(tmp_path): + path = "example/resources.json" + manifest = json.dumps({"schema_version": 1, "entry": "main.mjs", "files": ["main.mjs"]}) + checker.check_wheel( + tmp_path, + wheel(tmp_path, files={path: manifest, "example/main.mjs": "export default {};"}), + manifests=[path], + ) + + +def test_wheel_root_manifest_is_also_validated(tmp_path): + with pytest.raises(ValueError, match="Unsupported browser asset manifest"): + checker.check_wheel(tmp_path, wheel(tmp_path, files={"ui_manifest.json": "{}"})) + + +def test_single_module_source_cannot_drop_its_manifest(tmp_path): + (tmp_path / "example.py").write_text("", encoding="utf-8") + (tmp_path / "ui_manifest.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError, match="manifest.*not packaged"): + checker.check_wheel(tmp_path, wheel(tmp_path)) + + +@pytest.mark.parametrize("name", ["../escaped", "/escaped", r"example\escaped", "C:/ui_manifest.json", "C:ui_manifest.json", "C:/example/ui_manifest.json", "C:example/ui_manifest.json"]) +def test_wheel_paths_cannot_escape_temporary_directory(tmp_path, monkeypatch, name): + def fail_if_extracted(*args, **kwargs): + pytest.fail("Unsafe wheel paths reached extraction") + + monkeypatch.setattr(ZipFile, "extractall", fail_if_extracted) + with pytest.raises(ValueError, match="wheel path"): + checker.check_wheel(tmp_path, wheel(tmp_path, files={name: "no"})) + + +def test_invalid_packaged_python_is_rejected(tmp_path): + with pytest.raises(SyntaxError): + checker.check_wheel(tmp_path, wheel(tmp_path, files={"example/broken.py": "def broken(:\n"})) diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000000..0f31fa25af --- /dev/null +++ b/examples/README.md @@ -0,0 +1,58 @@ +# Develop an extension from an existing example + +Start with the smallest example that covers your extension. Each directory is an +independent Python package using the public `deerflow-extension-api` contract. + +| Start here | What it demonstrates | +| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| [Extension example](deerflow-extension-example/README.md) | Middleware, task lifecycle, model observations, services, and HTTP routes. Start here for backend extensions. | +| [Conversation bookmarks](deerflow-extension-bookmarks/README.md) | A browser page, conversation action, backend actions, packaged resources, and a model tool. Start here for full-stack plugins. | +| [Agent teams](deerflow-extension-agent-teams/README.md) | Full Agent orchestration. Use this after the basic contribution and lifecycle patterns are familiar. | + +Additional examples cover [text-list classification](deerflow-extension-jev-classify/README.md), +[context pruning](deerflow-extension-jev-context/README.md), and +[advisory screening of fetched content](deerflow-extension-jev-screening/README.md). + +## Edit, test, check, then install + +1. Copy the selected example outside the DeerFlow checkout. Rename its Python + package, distribution and `deerflow.extensions` entry point. For a full-stack + plugin, also choose its own namespace and browser module name. +2. Follow the example's README to configure its dependencies and run its tests. + Keep package behavior tests alongside the extension. +3. From the DeerFlow checkout root, prepare the backend environment with + `make -C backend install`, then check the package: + + ```bash + uv run --project backend --no-sync python scripts/check_extension.py /absolute/path/to/your-extension + ``` + + To try the checker on an unchanged example: + + ```bash + uv run --project backend --no-sync python scripts/check_extension.py examples/deerflow-extension-bookmarks + ``` + +4. Follow that example's deployment and try-it instructions. Restart Gateway, + refresh the browser when applicable, and exercise the installed feature. + Use `make extension-upgrade SOURCE=/absolute/path/to/your-extension` to apply + later source edits: local installs are snapshots. + +## What the checker covers + +The script builds a wheel in a temporary directory, checks exactly one extension +entry point and its packaged Python module, parses packaged Python syntax, and +checks that the API dependency range accepts this checkout's contract version. +It validates conventional `ui_manifest.json` resources with the same validator +used by Gateway, including detecting manifests left out of the wheel in flat or +`src/` layouts. A custom manifest filename needs an explicit wheel-relative path: + +```bash +uv run --project backend --no-sync python scripts/check_extension.py /absolute/path/to/your-extension --manifest your_package/resources.json +``` + +The build executes the package's build backend, so check trusted sources. The +checker does not import the extension, call `install()`, install the extension, or +change deployment configuration. Registration/configuration behavior and the +`@extension` version marker are verified when loading the plugin; continue to run +the package tests and the installed feature after packaging checks pass. diff --git a/examples/deerflow-extension-bookmarks/README.md b/examples/deerflow-extension-bookmarks/README.md index b834613e3c..eed952753f 100644 --- a/examples/deerflow-extension-bookmarks/README.md +++ b/examples/deerflow-extension-bookmarks/README.md @@ -1,5 +1,8 @@ # Conversation bookmarks — independent plugin example +For the shared edit → test → packaging check → install workflow, see the +[existing-example guide](../README.md). + A Pi-inspired example of a complete feature in one package: save the last visible assistant answer from a conversation menu, then search, rename, read or delete bookmarks in its own **My bookmarks** page, reached from the workspace sidebar. diff --git a/examples/deerflow-extension-example/README.md b/examples/deerflow-extension-example/README.md index 59fbbd7593..cde734b95e 100644 --- a/examples/deerflow-extension-example/README.md +++ b/examples/deerflow-extension-example/README.md @@ -1,5 +1,8 @@ # DeerFlow extension example +For the shared edit → test → packaging check → install workflow, see the +[existing-example guide](../README.md). + This directory is a compact, standalone Python package showing five of the eight DeerFlow extension contribution kinds: middleware, task lifecycle, system-model observer, service, and router. It depends on the public diff --git a/frontend/src/content/en/harness/extensions/quick-start.mdx b/frontend/src/content/en/harness/extensions/quick-start.mdx index f2c7c0bf80..e562c6ba15 100644 --- a/frontend/src/content/en/harness/extensions/quick-start.mdx +++ b/frontend/src/content/en/harness/extensions/quick-start.mdx @@ -9,6 +9,22 @@ import { Callout, Steps } from "nextra/components"; This chapter walks through a complete extension named `hello`. It contributes one middleware that times every tool call and logs a warning when a call is slower than a configured threshold. By the end you will have packaged it, unit-tested it without DeerFlow, installed it into a checkout, and seen it run. +## Start from an existing example + +To adapt working code, start with `examples/deerflow-extension-example` for backend +hooks or `examples/deerflow-extension-bookmarks` for a browser feature with backend +actions and a model tool. The [example guide](https://github.com/bytedance/deer-flow/blob/main/examples/README.md) +links their tests and installation steps. After preparing the backend environment, +run the packaging check from the checkout root: + +```bash +uv run --project backend --no-sync python scripts/check_extension.py /absolute/path/to/your-extension +``` + +It builds and checks a temporary wheel without importing the extension or changing +deployment configuration. Continue with the example's tests and installed-feature +verification. To build a minimal middleware from scratch, follow the tutorial below. + ## Prerequisites - A DeerFlow checkout that runs with `make dev`. See [Quick Start](/docs/application/quick-start). diff --git a/frontend/src/content/zh/harness/extensions/quick-start.mdx b/frontend/src/content/zh/harness/extensions/quick-start.mdx index dc050e44be..f5981472a8 100644 --- a/frontend/src/content/zh/harness/extensions/quick-start.mdx +++ b/frontend/src/content/zh/harness/extensions/quick-start.mdx @@ -9,6 +9,20 @@ import { Callout, Steps } from "nextra/components"; 本章完整走一遍一个名为 `hello` 的扩展。它贡献一个中间件，为每次工具调用计时，调用慢于配置阈值时记录一条警告。读完你会把它打包、在不依赖 DeerFlow 的情况下做单元测试、安装进一个 checkout，并看到它运行。 +## 从现有示例开始 + +如果要修改已有代码，后端扩展从 `examples/deerflow-extension-example` 开始； +带页面、后端 action 和模型工具的插件从 `examples/deerflow-extension-bookmarks` 开始。 +[示例上手路径](https://github.com/bytedance/deer-flow/blob/main/examples/README.md) +列出了对应的测试和安装步骤。准备好后端环境后，在 checkout 根目录运行打包检查： + +```bash +uv run --project backend --no-sync python scripts/check_extension.py /absolute/path/to/your-extension +``` + +脚本会构建并检查临时 wheel，不导入扩展，也不修改部署配置。 +随后继续运行示例测试，并验证安装后的功能。若要从零写一个最小中间件，接着阅读下面的教程。 + ## 前提 - 一个能用 `make dev` 运行的 DeerFlow checkout。见[快速上手](/docs/application/quick-start)。 diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 57fb015b81..706b7ed8db 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -25,11 +25,13 @@ path variables and compare storage defaults with the launcher's actual shell blo ## Dependency Check Diagnostics -`check.py` captures tool output as UTF-8 with replacement for malformed bytes, -independently of the host locale. Its Python pnpm runner inherits the environment -with `PYTHONIOENCODING=utf-8:backslashreplace`, matching the capture encoding. -Keep Unicode failure diagnostics and exit status available to `make check`. -Real subprocess regressions live in `backend/tests/test_check_script.py`. +`check_extension.py` checks wheels with Gateway's asset validator. +Tests: `backend/tests/test_extension_check_script.py`. + +`check.py` decodes captured output as UTF-8 with replacement and runs pnpm with +`PYTHONIOENCODING=utf-8:backslashreplace`. Preserve Unicode diagnostics and exit +status in `make check`, independently of the host locale. +Tests: `backend/tests/test_check_script.py`. ## Manual Claude OAuth Export diff --git a/scripts/check_extension.py b/scripts/check_extension.py new file mode 100644 index 0000000000..2dfb46f7d7 --- /dev/null +++ b/scripts/check_extension.py @@ -0,0 +1,176 @@ +"""Build and inspect a trusted extension wheel without installing its code. + +Run from the checkout root with the backend environment: +uv run --project backend --no-sync python scripts/check_extension.py PACKAGE +""" + +from __future__ import annotations + +import argparse +import ast +import configparser +import subprocess +import sys +import tempfile +from collections.abc import Sequence +from email.parser import Parser +from pathlib import Path, PurePosixPath, PureWindowsPath +from zipfile import BadZipFile, ZipFile + +from deerflow_extension_api import API_VERSION, BrowserAssets +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +from deerflow.extensions.browser_assets import load_browser_assets +from deerflow.extensions.manager import _validate_entry_point + + +def check_wheel(source: Path, wheel: Path, *, manifests: Sequence[str] = ()) -> None: + with ( + ZipFile(wheel) as archive, + tempfile.TemporaryDirectory(prefix="deerflow-extension-wheel-") as temporary, + ): + names = archive.namelist() + if len(names) != len(set(names)): + raise ValueError("Duplicate wheel paths") + for name in names: + path = PurePosixPath(name) + if ( + path.is_absolute() + or PureWindowsPath(name).drive + or ".." in path.parts + or "\\" in name + ): + raise ValueError("Invalid wheel path") + + metadata_paths = [ + name for name in names if name.endswith(".dist-info/METADATA") + ] + if len(metadata_paths) != 1: + raise ValueError("Expected one wheel distribution") + metadata = Parser().parsestr(archive.read(metadata_paths[0]).decode("utf-8")) + requirements = [ + Requirement(value) for value in metadata.get_all("Requires-Dist", []) + ] + api = [ + requirement + for requirement in requirements + if canonicalize_name(requirement.name) == "deerflow-extension-api" + and ( + requirement.marker is None or requirement.marker.evaluate({"extra": ""}) + ) + ] + if not api or any( + API_VERSION not in requirement.specifier for requirement in api + ): + raise ValueError( + f"Declare a deerflow-extension-api dependency compatible with this checkout ({API_VERSION})" + ) + + entry_path = str(PurePosixPath(metadata_paths[0]).parent / "entry_points.txt") + entries = configparser.ConfigParser(interpolation=None) + entries.optionxform = str + if entry_path in names: + entries.read_string(archive.read(entry_path).decode("utf-8")) + if ( + not entries.has_section("deerflow.extensions") + or len(entries["deerflow.extensions"]) != 1 + ): + raise ValueError("Declare exactly one deerflow.extensions entry point") + entry_name, target = next(iter(entries["deerflow.extensions"].items())) + _validate_entry_point(entry_name, target) + module, _ = target.rsplit(":", 1) + module_path = module.replace(".", "/") + if not any( + name in names + for name in (f"{module_path}.py", f"{module_path}/__init__.py") + ): + raise ValueError("Extension entry module is not packaged") + for name in names: + if name.endswith(".py"): + ast.parse(archive.read(name), filename=name) + + # Existing examples use flat or src-layout packages and ui_manifest.json. + # Custom manifest names are explicit rather than guessing from plugin code. + manifest_paths = set(manifests) + top_module = module.split(".")[0] + for import_root in (source, source / "src"): + package = import_root / top_module + if package.is_dir(): + manifest_paths.update( + path.relative_to(import_root).as_posix() + for path in package.rglob("ui_manifest.json") + ) + elif ( + package.with_suffix(".py").is_file() + and (import_root / "ui_manifest.json").is_file() + ): + manifest_paths.add("ui_manifest.json") + manifest_paths.update( + name for name in names if PurePosixPath(name).name == "ui_manifest.json" + ) + for manifest in manifest_paths: + if manifest not in names: + raise ValueError(f"Browser manifest is not packaged: {manifest}") + if manifest_paths: + archive.extractall(temporary) + for manifest in sorted(manifest_paths): + path = Path(temporary) / manifest + load_browser_assets(BrowserAssets("check", path.parent, manifest=path.name)) + + print(f"Entry point: {entry_name} = {target}") + print(f"API dependency: compatible with {API_VERSION}") + print(f"Browser manifests: {len(manifest_paths)} checked") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Build and check a trusted Python extension package using the current DeerFlow contract." + ) + parser.add_argument( + "package", type=Path, help="directory containing pyproject.toml" + ) + parser.add_argument( + "--manifest", + action="append", + default=[], + help="custom browser manifest path inside the wheel (repeatable)", + ) + args = parser.parse_args() + try: + source = args.package.resolve() + if not (source / "pyproject.toml").is_file(): + raise ValueError("Package directory must contain pyproject.toml") + with tempfile.TemporaryDirectory( + prefix="deerflow-extension-check-" + ) as temporary: + result = subprocess.run( + ["uv", "build", str(source), "--wheel", "--out-dir", temporary], + capture_output=True, + encoding="utf-8", + errors="replace", + timeout=120, + check=False, + ) + if result.returncode: + raise ValueError(f"Wheel build failed:\n{result.stderr}") + wheels = list(Path(temporary).glob("*.whl")) + if len(wheels) != 1: + raise ValueError("Build must produce one wheel") + check_wheel(source, wheels[0], manifests=args.manifest) + print(f"Packaging checks passed: {wheels[0].name}") + return 0 + except ( + OSError, + ValueError, + SyntaxError, + BadZipFile, + configparser.Error, + subprocess.TimeoutExpired, + ) as exc: + print(f"Extension check failed: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main())