From db02b77837d6209f35959395dd23a20011a3472f Mon Sep 17 00:00:00 2001 From: Loveca <44801706+Loveca@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:04:42 +0800 Subject: [PATCH] feat(plugins): add evidence-gated skill self-evolution example (#7984) * feat(plugins): add evidence-gated skill self-evolution example Demonstrates the #7884 roadmap deliverable: a proposed change, the regression case it fixes, the old and new results, and a rollback to a known-good version, driven only through the public SkillMutationService contract merged in #5645. The example stages a candidate against a sealed completed-run evidence anchor, evaluates recorded JSON regression cases against both frozen bundles with a deterministic static evaluator (never executing package content), and commits only a "justified" verdict with an AssessmentRef binding the report to the exact candidate hash and base revision. Unjustified or failed proposals are discarded; revert stays append-only. Ships an HTTP walkthrough surface, an offline lifecycle test against a contract-faithful fake host, and a README section proposing a concrete evidence standard for the automatic-publication discussion. Co-Authored-By: Claude Code * fix(skill-evolution): resolve request models and honor APPLIED publication The walkthrough routes had never been exercised through FastAPI. With `from __future__ import annotations`, request models defined inside `build_router` are unresolvable forward references, so FastAPI read `body` as a query parameter and every POST answered 422. Move the models to module scope and cover the routes through TestClient. A committed operation reports `publication == "APPLIED"`; `READY` is a `views` state. The committed check listed READY, so a successful apply never flipped `record.committed` and the ALREADY_COMMITTED guard never fired. Fix the check, mirror the real literals in the lifecycle fake host, and assert both the flag and the second-apply refusal. Co-Authored-By: Claude Code * fix(skill-evolution): address review feedback and register the example catalog - Bind walkthrough routes to the authenticated principal: owner checks on propose/apply/revert, owner-scoped listing, and non-disclosing 404s. - Map unknown skills, proposals and operations to 404 for both NOT_FOUND and the real host's NOT_FOUND_OR_FORBIDDEN; map broken case files to 400. - Ignore URL paths and trim sentence punctuation when extracting in-package references. - Validate config.enabled's type instead of treating any non-False value as on. - Soften the README claim about report persistence and add CHANGELOG entries. - Register the example in the frontend extension catalog, fixing frontend-unit-tests. Co-Authored-By: Claude Code * fix(skill-evolution): narrow case-file error mapping and cover run visibility - Label ValueError as CASE_INVALID only on POST /proposals, the route that actually loads case files; apply and revert now surface host bugs as 500 instead of pointing at the case directory. - Cover the request-scoped run-evidence reader the real Gateway installs: hidden anchor returns 403 SOURCE_NOT_VISIBLE before the host is touched, a denied resolver returns 403, and a visible anchor still proceeds. - Add regressions pinning host ValueError on apply/revert to 500. Co-Authored-By: Claude Code * docs(changelog): drop contributor changelog entry per policy Contributor PRs no longer modify CHANGELOG.md or CHANGELOG_zh.md (#8202); maintainers add the entry at merge time. --------- Co-authored-by: Claude Code Co-authored-by: Nan Gao --- backend/tests/test_extension_documentation.py | 2 +- examples/README.md | 5 +- .../README.md | 194 ++++++++ .../__init__.py | 44 ++ .../cases/converter-arg.json | 8 + .../deerflow-extension.json | 51 ++ .../evaluator.py | 312 ++++++++++++ .../service.py | 463 ++++++++++++++++++ .../pyproject.toml | 44 ++ .../tests/test_evaluator.py | 201 ++++++++ .../tests/test_lifecycle.py | 352 +++++++++++++ .../tests/test_routes.py | 330 +++++++++++++ .../extensions/bundled-documentation.json | 51 ++ frontend/src/core/extensions/catalog.ts | 5 + frontend/src/core/i18n/locales/en-US.ts | 4 + frontend/src/core/i18n/locales/types.ts | 1 + frontend/src/core/i18n/locales/zh-CN.ts | 4 + 17 files changed, 2068 insertions(+), 3 deletions(-) create mode 100644 examples/deerflow-extension-skill-evolution/README.md create mode 100644 examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/__init__.py create mode 100644 examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/cases/converter-arg.json create mode 100644 examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/deerflow-extension.json create mode 100644 examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/evaluator.py create mode 100644 examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/service.py create mode 100644 examples/deerflow-extension-skill-evolution/pyproject.toml create mode 100644 examples/deerflow-extension-skill-evolution/tests/test_evaluator.py create mode 100644 examples/deerflow-extension-skill-evolution/tests/test_lifecycle.py create mode 100644 examples/deerflow-extension-skill-evolution/tests/test_routes.py diff --git a/backend/tests/test_extension_documentation.py b/backend/tests/test_extension_documentation.py index 469d7a4bad..c0400c8c97 100644 --- a/backend/tests/test_extension_documentation.py +++ b/backend/tests/test_extension_documentation.py @@ -185,7 +185,7 @@ def test_bundled_documentation_snapshot_is_current(): script = runpy.run_path(str(root / "scripts/sync_extension_documentation.py")) assert script["DESTINATION"].read_text(encoding="utf-8") == script["render"]() pages = json.loads(script["render"]()) - assert len(pages) == 16 + assert len(pages) == 17 assert pages["deerflow-extension-office-preview"]["id"] == "office-preview" diff --git a/examples/README.md b/examples/README.md index 828a7c777a..b26cbf8892 100644 --- a/examples/README.md +++ b/examples/README.md @@ -12,8 +12,9 @@ independent Python package using the public `deerflow-extension-api` contract. Additional examples cover [Office file previews](deerflow-extension-office-preview/README.md), [text-list classification](deerflow-extension-jev-classify/README.md), [context pruning](deerflow-extension-jev-context/README.md), -[advisory screening of fetched content](deerflow-extension-jev-screening/README.md), and -[sectioned long-form report writing](deerflow-extension-report-sections/README.md). +[advisory screening of fetched content](deerflow-extension-jev-screening/README.md), +[sectioned long-form report writing](deerflow-extension-report-sections/README.md), and +[evidence-gated skill self-evolution](deerflow-extension-skill-evolution/README.md). ## Describe the extension diff --git a/examples/deerflow-extension-skill-evolution/README.md b/examples/deerflow-extension-skill-evolution/README.md new file mode 100644 index 0000000000..b2cd6d3229 --- /dev/null +++ b/examples/deerflow-extension-skill-evolution/README.md @@ -0,0 +1,194 @@ +# Skill self-evolution example — evidence-gated revisions + +An opt-in Python plugin that demonstrates the roadmap deliverable from +[#7884](https://github.com/bytedance/deer-flow/issues/7884) ("Skills, +extensions, and reproducible agents"): a **proposed change**, the **regression +case it fixes**, the **old and new results**, and a **rollback to a known-good +version** — driven entirely through the host `SkillMutationService` contract +merged in [#5645](https://github.com/bytedance/deer-flow/pull/5645). + +It uses extension API 0.2.11 with no host code changes. The host owns storage, +security scanning, revision identity and authorization; this package owns only +one policy: **a revision commits only when a recorded regression case +demonstrates it fixes something**. + +## Why evidence-gated + +With `skill_evolution.enabled`, the agent can already rewrite its own skills +through `skill_manage`. What is missing is a visible answer to "was that change +an improvement?". This example attaches one to every commit: + +``` +propose ──> stage() freeze baseline + candidate bundles (host-side) +evaluate ─> recorded cases run static checks against BOTH bundles +apply ────> check() + commit(assessment_ref) host security gate, then commit + with the verdict attached +rollback ─> revert() append-only return to the known-good revision +``` + +The `AssessmentRef` carried into `commit` binds the evidence to the exact +proposal (`candidate_hash` + `base_revision`); the host rejects a mismatched +assessment with `INVALID_ASSESSMENT`. Afterwards every committed `Operation` +answers: what changed, which recorded cases justified it, and how to undo it. + +## Recorded regression cases + +Cases are JSON files in `deerflow_extension_skill_evolution/cases/` (override +with `config.cases_dir`). One case per file: + +```json +{ + "id": "converter-arg", + "description": "SKILL.md must instruct the converter with --format json", + "checks": [ + { "kind": "text_contains", "path": "SKILL.md", "needle": "--format json" }, + { "kind": "text_not_contains", "path": "SKILL.md", "needle": "--fmt " } + ] +} +```` + +Check kinds: `file_exists`, `text_contains`, `text_not_contains`, +`referenced_files_exist` (bare `scripts/…` / `assets/…` tokens in a text file +must exist in the package), and `scripts_parse` (every `scripts/*.py` must pass +`ast.parse`). + +**The evaluator never executes package content.** The candidate is an +agent-proposed revision that has not passed the host security `check` yet, so +the only safe operations are structural (presence, text, parse-only). A +malformed case file raises and stops the evaluation rather than silently +narrowing it. + +### Verdicts + +| Verdict | Meaning | Service behavior | +| --- | --- | --- | +| `justified` | every case passes on the candidate and at least one fails on the baseline | eligible for `apply` | +| `unjustified` | every case passes on both sides — nothing recorded was broken | proposal discarded, `apply` refuses | +| `failed` | some case still fails on the candidate (or none recorded) | proposal discarded, `apply` refuses | + +## Install and configure + +From the DeerFlow checkout root: + +```sh +make extension-install SOURCE=examples/deerflow-extension-skill-evolution +``` + +The mutation capability requires an explicit operator grant in `config.yaml` +(restart required). Prerequisites are the same as any `skill_mutations` +consumer: durable database backend, `run_events.backend: db`, and a single +POSIX host with local disk: + +```yaml +plugins: + - name: skill-evolution + package: deerflow-extension-skill-evolution + use: deerflow_extension_skill_evolution:install + enabled: true + host_access: + evidence: + owners: ["your-user-id"] + skill_mutations: + owners: ["your-user-id"] # subset of evidence.owners + operations: [stage, check, commit, revert] + trigger_agents: [lead_agent] + target_skills: [example-skill] # canonical names this plugin may revise + topology: single_host_local + config: + cases_dir: null # optional: default bundled cases +``` + +Without the grant the service fails to start with a `NOT_GRANTED` diagnostic; +nothing else in the deployment changes. + +## HTTP surface + +All routes are under `/api/extension-skill-evolution`: + +| Route | Purpose | +| --- | --- | +| `GET /health` | readiness, evaluator identity, recorded case count | +| `POST /proposals` | resolve the sealed run evidence, stage a candidate, evaluate, return the verdict + report | +| `GET /proposals` · `GET /proposals/{id}` | list / inspect proposals and their reports | +| `POST /proposals/{id}/apply` | host `check`, then commit with the `AssessmentRef` attached | +| `POST /operations/{id}/revert` | roll back a committed revision | + +`POST /proposals` takes `{skill_name, candidate_content, thread_id, run_id}`. +The run must be completed and sealed — it is the evidence anchor the host +requires for staging (`source_refs`), so every proposal records *which run +motivated the change*. + +**Every route binds the caller.** The capability handles injected into the +service are scoped to the extension by the operator's grant, not to the +requesting user, so the routes resolve the host principal +(`deerflow_extension_api.auth.resolve_principal`) and refuse to act on a skill +owned by anyone else: `propose`, `apply` and `revert` answer 401 without a +principal and 403 (`FORBIDDEN`) across owners, and `GET /proposals` lists only +the caller's own proposals (another owner's proposal reads as 404, without +disclosing that it exists). When the host exposes a request-scoped run-evidence +reader, the evidence anchor must also be visible to the caller; hosts without +one fall back to the owner check. + +## Offline walkthrough (tests) + +The package ships the demonstration as tests; no Gateway, model, or network: + +```sh +cd examples/deerflow-extension-skill-evolution +uv run --no-project --python 3.12 --with pytest --with pytest-asyncio \ + --with ../../backend/packages/extension-api \ + --with-editable . python -m pytest tests/ -q +``` + +`tests/test_lifecycle.py` drives the full loop against a fake host that +implements only the public contract: propose a fix for a broken converter +argument → verdict `justified` → apply → the committed operation carries the +report id → revert → the skill is byte-identical to the known-good baseline. +It also pins the refusal paths: `unjustified` and `failed` proposals are +discarded and cannot commit, a denied host `check` blocks the commit, and +double-commit is rejected. + +## Scope and boundaries + +- **This is an example, not a production engine.** The evaluator is a + deterministic static checker; it has no live-model runs, metrics, or signal + collection. +- [#6954](https://github.com/bytedance/deer-flow/discussions/6954) proposes a + distributable evolution plugin package with its own snapshot/versioning + design; this example stops at the merged #5645 contract and should be + revisited against that RFC if it lands. +- [#7066](https://github.com/bytedance/deer-flow/issues/7066) (eval-gated + evolution) owns the broader signal → attribution → candidate → verification + loop; the minimal evaluator here is deliberately smaller than that scope. +- The `skill_manage` direct-write path is unchanged. This plugin is additive + and opt-in. + +## A concrete evidence standard (input for the roadmap discussion) + +The roadmap asks what evidence should be required before enabling automatic +publication. Rather than propose a standard in the abstract, this example makes +one concrete and reviewable: + +1. **Every proposal names its motivating run.** Staging requires sealed + completed-run evidence (`source_refs`); the host rejects stale or unsealed + anchors. +2. **Evidence is recorded, not narrated.** A case is a versioned JSON file with + deterministic checks; the verdict is computed from the frozen baseline and + candidate bundles, not from the proposal text. +3. **The verdict travels with the commit.** `AssessmentRef` pins + `evaluator`, `evaluator_version`, and a content-addressed `report_id` + (SHA-256 over the canonical report JSON) to the exact `candidate_hash` and + `base_revision`. The report bytes themselves are held in this service's + in-memory cache, not persisted; the id lets anyone holding the report — or + recomputing it from the same cases and frozen bundles — verify that it is + the one this commit cites. +4. **Fail-closed by default.** `unjustified` and `failed` proposals are + discarded before they can commit; a host `check` denial blocks the commit; + an assessment mismatch is rejected by the host itself. +5. **Rollback is as auditable as the change.** `revert` appends a new + operation referencing the one it undoes. + +Automatic publication would additionally need: case authorship provenance (who +recorded the case, and was it derived from the same run that motivated the +change), a minimum case count, and a policy for `unjustified` proposals that +keep recurring. Those are deliberately out of scope here. diff --git a/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/__init__.py b/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/__init__.py new file mode 100644 index 0000000000..cb4176d566 --- /dev/null +++ b/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/__init__.py @@ -0,0 +1,44 @@ +"""Evidence-gated skill self-evolution example. + +Demonstrates the roadmap deliverable: a proposed change, the regression case it +fixes, the old and new results, and a rollback to a known-good version — all +through the host ``SkillMutationService`` contract merged in #5645. + +Install via ``config.yaml -> plugins:`` with a ``host_access.skill_mutations`` +grant (see README.md). Without the grant the service fails to start and the +Gateway reports a diagnostic; nothing else changes. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from deerflow_extension_api import ExtensionInstall, ExtensionRegistry, extension + +from deerflow_extension_skill_evolution.service import EvolutionService, build_router + +__all__ = ["install"] + +_DEFAULT_CASES_DIR = Path(__file__).resolve().parent / "cases" + + +@extension(api="0.2.11", name="skill-evolution") +def install(registry: ExtensionRegistry, config: Mapping[str, Any]) -> None: + """Register the evolution service and its HTTP walkthrough surface.""" + # ``is False`` alone would silently keep the plugin on for ``enabled: 0`` or + # ``enabled: "false"``; a mistyped off switch must fail the deployment. + enabled = config.get("enabled", True) + if type(enabled) is not bool: + raise ValueError(f"skill-evolution: config.enabled must be a boolean, got {enabled!r}") + if not enabled: + return + + cases_dir = config.get("cases_dir") + service = EvolutionService(cases_dir=Path(cases_dir) if cases_dir else _DEFAULT_CASES_DIR) + registry.service(service) + registry.routers((build_router(service),)) + + +_entry_point: ExtensionInstall = install diff --git a/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/cases/converter-arg.json b/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/cases/converter-arg.json new file mode 100644 index 0000000000..f254932ce6 --- /dev/null +++ b/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/cases/converter-arg.json @@ -0,0 +1,8 @@ +{ + "id": "converter-arg", + "description": "SKILL.md must instruct the converter with --format json, not the removed --fmt alias", + "checks": [ + { "kind": "text_contains", "path": "SKILL.md", "needle": "--format json" }, + { "kind": "text_not_contains", "path": "SKILL.md", "needle": "--fmt " } + ] +} diff --git a/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/deerflow-extension.json b/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/deerflow-extension.json new file mode 100644 index 0000000000..98c1e2392e --- /dev/null +++ b/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/deerflow-extension.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "id": "catalog/skillEvolution", + "locales": { + "zh-CN": { + "title": "证据驱动的技能进化", + "description": "在回归用例与宿主校验通过后才提议技能修改,对比新旧结果,并可回滚到已确认的版本。", + "usage": [ + { + "title": "安装并授权", + "description": "通过 make extension-install 安装示例,在 config.yaml 中配置 skill mutation 与 evidence 授权后重启 Gateway。" + }, + { + "title": "记录回归用例", + "description": "在包的 cases 目录添加 JSON 用例,用 file_exists、text_contains、text_not_contains 等静态检查描述这次修改要修复什么。" + }, + { + "title": "提出、评估、应用与回滚", + "description": "调用 /api/extension-skill-evolution/proposals 提交候选版本,查看 justified、unjustified 或 failed 结论;通过后 apply,必要时 revert 到已确认版本。" + } + ], + "requirements": [ + "需要支持 skill_mutations 的持久化数据库、run_events.backend: db,以及单实例 POSIX 本地磁盘。", + "需要在 config.yaml 中显式授予 skill mutation 与 evidence 访问权限。" + ] + }, + "en-US": { + "title": "Evidence-gated skill evolution", + "description": "Propose skill edits only after a regression case and host checks pass, compare old and new results, and roll back to a known-good revision.", + "usage": [ + { + "title": "Install and grant access", + "description": "Install the example with make extension-install, configure the skill mutation and evidence grants in config.yaml, then restart Gateway." + }, + { + "title": "Record a regression case", + "description": "Add a JSON case under the package's cases directory using static checks such as file_exists, text_contains and text_not_contains to state what the change fixes." + }, + { + "title": "Propose, evaluate, apply and roll back", + "description": "Submit a candidate to /api/extension-skill-evolution/proposals, inspect the justified, unjustified or failed verdict, apply an accepted revision, and revert when needed." + } + ], + "requirements": [ + "Requires a durable database backend, run_events.backend: db, and a single POSIX host with local disk.", + "Requires explicit skill mutation and evidence grants in config.yaml." + ] + } + }, + "docs_url": "https://github.com/bytedance/deer-flow/tree/main/examples/deerflow-extension-skill-evolution#readme" +} diff --git a/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/evaluator.py b/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/evaluator.py new file mode 100644 index 0000000000..0e7424bad8 --- /dev/null +++ b/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/evaluator.py @@ -0,0 +1,312 @@ +"""Deterministic static evaluation of proposed skill revisions. + +This evaluator never executes package content. A candidate bundle is an +agent-proposed revision that has not passed the host security ``check`` yet, +so the only safe operations are structural: file presence, decoded-text +assertions, reference integrity, and ``ast.parse`` (parse only, no import, +no exec) for bundled Python scripts. + +Recorded regression cases describe the defect a proposal claims to fix. The +verdict compares both sides of the proposal: + +- ``justified`` — every case passes on the candidate and at least one fails + on the baseline: the change demonstrably fixes a recorded defect. +- ``unjustified`` — every case passes on both sides: nothing recorded was + broken, so the proposal carries no evidence-backed reason to land. +- ``failed`` — some case still fails on the candidate (or no cases were + recorded at all): the fix does not hold, so the proposal must not commit. + +The report is deterministic — same inputs, same bytes, same ``report_id`` — +which is what lets it be referenced from a commit's ``AssessmentRef`` and +re-verified later. +""" + +from __future__ import annotations + +import ast +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +EVALUATOR_NAME = "skill-evolution-example-static" +EVALUATOR_VERSION = "0.1.0" + +Decision = Literal["justified", "unjustified", "failed"] + +#: ``Package`` is the evaluator's view of one skill revision: POSIX-relative +#: file path → raw bytes. The host bundle (``ProposalBundle.baseline`` / +#: ``.candidate``) converts with ``{f.path: f.content for f in files}``. +Package = Mapping[str, bytes] + +# Bare in-package references inside SKILL.md prose. A reference must appear as +# a standalone token: the lookbehind rejects a preceding word character, path +# separator, dot, colon or dash, so a URL such as +# ``https://example.com/scripts/run.py`` is not mistaken for an in-package file. +# Trailing sentence punctuation is trimmed after the match (see ``_references``). +# Only the two conventional skill subdirectories are scanned. +_REFERENCE_RE = re.compile(r"(? str: + """Validate a case-declared package path before it is ever used as a key.""" + if not value or len(value) > 1024 or "\\" in value or value.startswith("/"): + raise ValueError(f"{field} must be a relative POSIX path: {value!r}") + if any(part in {"", ".", ".."} for part in value.split("/")): + raise ValueError(f"{field} must not contain empty or dot segments: {value!r}") + return value + + +@dataclass(frozen=True) +class Check: + """One static assertion against a skill package.""" + + kind: str + path: str | None = None + needle: str | None = None + + @classmethod + def from_json(cls, raw: Any) -> Check: + if not isinstance(raw, dict): + raise ValueError("check must be an object") + kind = raw.get("kind") + if not isinstance(kind, str) or kind not in _CHECK_KINDS: + raise ValueError(f"unknown check kind: {kind!r}") + path = raw.get("path") + if path is not None: + if not isinstance(path, str): + raise ValueError(f"{kind}: path must be a string") + _reject_unsafe_path(path, f"{kind}: path") + needle = raw.get("needle") + if needle is not None and (not isinstance(needle, str) or not needle): + raise ValueError(f"{kind}: needle must be a nonempty string") + if kind in {"file_exists", "text_contains", "text_not_contains", "referenced_files_exist"} and path is None: + raise ValueError(f"{kind} requires a path") + if kind in {"text_contains", "text_not_contains"} and needle is None: + raise ValueError(f"{kind} requires a needle") + return cls(kind=kind, path=path, needle=needle) + + +@dataclass(frozen=True) +class Case: + """A recorded regression case: all of its checks must pass on a fixed package.""" + + case_id: str + description: str + checks: tuple[Check, ...] + + @classmethod + def from_json(cls, raw: Any, *, origin: str) -> Case: + if not isinstance(raw, dict): + raise ValueError(f"{origin}: case must be an object") + case_id = raw.get("id") + if not isinstance(case_id, str) or not _CASE_ID_RE.fullmatch(case_id) or len(case_id) > 64: + raise ValueError(f"{origin}: id must be kebab-case and at most 64 characters") + description = raw.get("description") + if not isinstance(description, str) or not description.strip(): + raise ValueError(f"{origin}: description must be a nonempty string") + checks_raw = raw.get("checks") + if not isinstance(checks_raw, list) or not checks_raw: + raise ValueError(f"{origin}: checks must be a nonempty list") + return cls( + case_id=case_id, + description=description.strip(), + checks=tuple(Check.from_json(item) for item in checks_raw), + ) + + +def load_cases_dir(directory: str | Path) -> tuple[Case, ...]: + """Load recorded cases from ``*.json`` files, ordered by file name. + + Malformed files raise ``ValueError`` naming the file: a broken case must + stop the evaluation rather than silently narrow it. + """ + root = Path(directory) + if not root.is_dir(): + raise ValueError(f"cases directory not found: {root}") + cases: list[Case] = [] + for file in sorted(root.glob("*.json")): + try: + raw = json.loads(file.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError(f"{file.name}: unreadable case file ({exc})") from exc + cases.append(Case.from_json(raw, origin=file.name)) + return tuple(cases) + + +@dataclass(frozen=True) +class CheckOutcome: + kind: str + passed: bool + detail: str + + +@dataclass(frozen=True) +class CaseOutcome: + case_id: str + description: str + baseline: tuple[CheckOutcome, ...] + candidate: tuple[CheckOutcome, ...] + + @property + def baseline_passed(self) -> bool: + return all(outcome.passed for outcome in self.baseline) + + @property + def candidate_passed(self) -> bool: + return all(outcome.passed for outcome in self.candidate) + + +def _references(text: str) -> list[str]: + """Extract bare in-package file references from prose. + + ``scripts/run.py.`` at the end of a sentence is the file ``scripts/run.py``, + not a file whose name ends in a period, so trailing ``.``/``/``/``-`` are + stripped before the token is looked up in the package. + """ + return [token for raw in _REFERENCE_RE.findall(text) if (token := raw.rstrip("./-"))] + + +def _decode(package: Package, path: str) -> tuple[str | None, str]: + content = package.get(path) + if content is None: + return None, f"missing {path}" + try: + return content.decode("utf-8"), "" + except UnicodeDecodeError: + return None, f"{path} is not valid UTF-8" + + +def _clip(detail: str) -> str: + return detail if len(detail) <= _MAX_DETAIL_CHARS else detail[: _MAX_DETAIL_CHARS - 1] + "…" + + +def run_check(check: Check, package: Package) -> CheckOutcome: + """Run one static check against one package revision. Never executes code.""" + if check.kind == "file_exists": + assert check.path is not None + passed = check.path in package + return CheckOutcome(check.kind, passed, "" if passed else f"missing {check.path}") + + if check.kind == "text_contains": + assert check.path is not None and check.needle is not None + text, detail = _decode(package, check.path) + if text is None: + return CheckOutcome(check.kind, False, detail) + passed = check.needle in text + return CheckOutcome(check.kind, passed, "" if passed else _clip(f"{check.path} does not contain {check.needle!r}")) + + if check.kind == "text_not_contains": + assert check.path is not None and check.needle is not None + text, detail = _decode(package, check.path) + if text is None: + return CheckOutcome(check.kind, False, detail) + passed = check.needle not in text + return CheckOutcome(check.kind, passed, "" if passed else _clip(f"{check.path} still contains {check.needle!r}")) + + if check.kind == "referenced_files_exist": + assert check.path is not None + text, detail = _decode(package, check.path) + if text is None: + return CheckOutcome(check.kind, False, detail) + missing = sorted({token for token in _references(text) if token not in package}) + passed = not missing + return CheckOutcome(check.kind, passed, "" if passed else _clip(f"{check.path} references missing files: {', '.join(missing)}")) + + # scripts_parse: parse-only syntax gate over every bundled Python script. + failures: list[str] = [] + for path in sorted(key for key in package if key.startswith("scripts/") and key.endswith(".py")): + text, detail = _decode(package, path) + if text is None: + failures.append(detail) + continue + try: + ast.parse(text, filename=path) + except SyntaxError as exc: + failures.append(f"{path}: {exc.msg} (line {exc.lineno})") + passed = not failures + return CheckOutcome("scripts_parse", passed, "" if passed else _clip("; ".join(failures))) + + +def evaluate_case(case: Case, baseline: Package, candidate: Package) -> CaseOutcome: + return CaseOutcome( + case_id=case.case_id, + description=case.description, + baseline=tuple(run_check(check, baseline) for check in case.checks), + candidate=tuple(run_check(check, candidate) for check in case.checks), + ) + + +@dataclass(frozen=True) +class Evaluation: + decision: Decision + reasons: tuple[str, ...] + cases: tuple[CaseOutcome, ...] + + def report(self, *, candidate_hash: str) -> dict[str, Any]: + """Deterministic JSON-able report; identical inputs produce identical bytes.""" + return { + "evaluator": EVALUATOR_NAME, + "evaluator_version": EVALUATOR_VERSION, + "decision": self.decision, + "reasons": list(self.reasons), + "candidate_hash": candidate_hash, + "cases": [ + { + "id": outcome.case_id, + "description": outcome.description, + "baseline_passed": outcome.baseline_passed, + "candidate_passed": outcome.candidate_passed, + "baseline": [{"kind": item.kind, "passed": item.passed, "detail": item.detail} for item in outcome.baseline], + "candidate": [{"kind": item.kind, "passed": item.passed, "detail": item.detail} for item in outcome.candidate], + } + for outcome in self.cases + ], + } + + +def evaluate(cases: Sequence[Case], baseline: Package, candidate: Package) -> Evaluation: + """Compare recorded cases across both sides of a proposal.""" + if not cases: + return Evaluation(decision="failed", reasons=("no regression cases recorded; nothing justifies a change",), cases=()) + outcomes = tuple(evaluate_case(case, baseline, candidate) for case in cases) + candidate_failures = [outcome.case_id for outcome in outcomes if not outcome.candidate_passed] + baseline_failures = [outcome.case_id for outcome in outcomes if not outcome.baseline_passed] + if candidate_failures: + return Evaluation( + decision="failed", + reasons=(f"cases still failing on the candidate: {', '.join(candidate_failures)}",), + cases=outcomes, + ) + if not baseline_failures: + return Evaluation( + decision="unjustified", + reasons=("every recorded case already passed on the baseline; the proposal fixes nothing on record",), + cases=outcomes, + ) + return Evaluation( + decision="justified", + reasons=(f"candidate fixes recorded failures: {', '.join(baseline_failures)}",), + cases=outcomes, + ) + + +def report_id(report: Mapping[str, Any]) -> str: + """Content-addressed identifier for a report, safe to store in an AssessmentRef.""" + canonical = json.dumps(dict(report), sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() diff --git a/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/service.py b/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/service.py new file mode 100644 index 0000000000..d8d48c8008 --- /dev/null +++ b/examples/deerflow-extension-skill-evolution/deerflow_extension_skill_evolution/service.py @@ -0,0 +1,463 @@ +"""Evidence-gated skill evolution orchestration over the host mutation contract. + +This service is the demonstration the roadmap asks for: it drives the merged +``SkillMutationService`` (#5645) through *propose → evaluate → apply → rollback* +and refuses to commit a revision that carries no recorded, passing evidence. + +It depends only on the public ``deerflow-extension-api`` contract — never on +harness internals — so it is the same code path a third-party integrator would +write. The host owns storage, security scanning, revision identity and +authorization; this service owns only the *policy* of "commit only when a +recorded regression case justifies the change". +""" + +from __future__ import annotations + +import uuid +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from deerflow_extension_api import ( + AssessmentRef, + CompletedRunEvidenceReader, + ExtensionPrincipal, + ExtensionRuntimeDeps, + HostCapabilityError, + SkillMutationService, + resolve_principal, + resolve_run_evidence_reader, +) +from fastapi import APIRouter, HTTPException, Request +from pydantic import BaseModel, Field + +from deerflow_extension_skill_evolution.evaluator import ( + EVALUATOR_NAME, + EVALUATOR_VERSION, + Case, + Evaluation, + evaluate, + load_cases_dir, + report_id, +) + + +@dataclass +class ProposalRecord: + """The example's own view of one in-flight proposal and its evaluation.""" + + proposal_id: str + name: str + owner_id: str + candidate_hash: str + decision: str + reasons: tuple[str, ...] + report_id: str + report: dict[str, Any] + operation_id: str | None = None + committed: bool = False + + +class SkillEvolutionError(RuntimeError): + """A named, caller-facing failure distinct from host capability errors.""" + + def __init__(self, code: str, message: str = "") -> None: + self.code = code + super().__init__(f"{code}: {message}" if message else code) + + +class EvolutionService: + """Orchestrates evidence-gated skill revisions through the host contract. + + Lifecycle: ``start(deps)`` captures the host capability handles the Gateway + injects (``skill_mutations`` + ``completed_run_evidence``); ``stop()`` drops + them. Reports are held in memory for the life of the instance — the durable + evidence is the ``assessment`` the host freezes onto each committed + ``Operation``, not this cache. + """ + + def __init__(self, *, cases_dir: str | Path | None = None, cases: Sequence[Case] | None = None) -> None: + self._mutations: SkillMutationService | None = None + self._evidence: CompletedRunEvidenceReader | None = None + self._cases_dir = Path(cases_dir) if cases_dir is not None else None + self._static_cases: tuple[Case, ...] | None = tuple(cases) if cases is not None else None + self._proposals: dict[str, ProposalRecord] = {} + + # -- ExtensionService protocol ----------------------------------------- + + async def start(self, deps: ExtensionRuntimeDeps) -> None: + self._mutations = deps.skill_mutations + self._evidence = deps.completed_run_evidence + if self._mutations is None: + raise SkillEvolutionError( + "NOT_GRANTED", + "skill_mutations capability missing; grant host_access.skill_mutations in config.yaml plugins", + ) + + async def stop(self) -> None: + self._mutations = None + self._evidence = None + + # -- introspection ------------------------------------------------------ + + @property + def ready(self) -> bool: + return self._mutations is not None + + def cases(self) -> tuple[Case, ...]: + """Recorded regression cases: explicit set, else loaded from cases_dir.""" + if self._static_cases is not None: + return self._static_cases + if self._cases_dir is None: + return () + return load_cases_dir(self._cases_dir) + + def get_proposal(self, proposal_id: str) -> ProposalRecord: + record = self._proposals.get(proposal_id) + if record is None: + raise SkillEvolutionError("NO_SUCH_PROPOSAL", proposal_id) + return record + + def list_proposals(self) -> list[ProposalRecord]: + return list(self._proposals.values()) + + # -- core orchestration ------------------------------------------------- + + async def resolve_source(self, *, thread_id: str, run_id: str) -> str: + """Turn a completed run into the sealed evidence ref ``stage`` requires.""" + if self._evidence is None: + raise SkillEvolutionError("NO_EVIDENCE", "completed_run_evidence capability not granted") + snapshot = await self._evidence.get_snapshot(thread_id=thread_id, run_id=run_id) + if not snapshot.snapshot_ref: + raise SkillEvolutionError("SOURCE_UNSEALED", f"run {run_id} has no sealed evidence snapshot") + return snapshot.snapshot_ref + + async def propose( + self, + *, + skill_name: str, + candidate_content: str, + source_ref: str, + idempotency_key: str | None = None, + cases: Sequence[Case] | None = None, + expected_owner_id: str | None = None, + ) -> ProposalRecord: + """Stage a candidate, evaluate it against recorded cases, and record the verdict. + + Staging freezes both bundles host-side; evaluation runs entirely on the + returned ``ProposalBundle`` bytes, so the verdict is about exactly the + revision that would commit. Nothing is written to the live skill here. + + ``expected_owner_id`` binds the operation to a caller: when given, a + skill owned by anyone else is refused *before* staging, so a contributed + route can never act on another user's skill. + """ + mutations = self._require_mutations() + recorded = tuple(cases) if cases is not None else self.cases() + + view = await mutations.read_skill(source_ref=source_ref, name=skill_name) + self._check_owner(view.owner_id, expected_owner_id) + proposal = await mutations.stage( + source_refs=(source_ref,), + target_ref=view.target_ref, + expected_revision=view.revision, + content=candidate_content, + idempotency_key=idempotency_key or uuid.uuid4().hex, + ) + bundle = await mutations.read_proposal_bundle(proposal_id=proposal.proposal_id) + baseline = {item.path: item.content for item in bundle.baseline} + candidate = {item.path: item.content for item in bundle.candidate} + + evaluation: Evaluation = evaluate(recorded, baseline, candidate) + report = evaluation.report(candidate_hash=proposal.candidate_hash) + rid = report_id(report) + + record = ProposalRecord( + proposal_id=proposal.proposal_id, + name=skill_name, + owner_id=proposal.owner_id, + candidate_hash=proposal.candidate_hash, + decision=evaluation.decision, + reasons=evaluation.reasons, + report_id=rid, + report=report, + ) + self._proposals[proposal.proposal_id] = record + + # An unjustified or failed proposal must not linger pending; discard it + # so the host's pending-proposal quota is not consumed by dead weight. + if evaluation.decision != "justified": + await mutations.discard(proposal_id=proposal.proposal_id) + return record + + async def apply(self, *, proposal_id: str, idempotency_key: str | None = None, expected_owner_id: str | None = None) -> Any: + """Commit a justified proposal with its evidence attached, after host check. + + Refuses unless the recorded evaluation was ``justified``. The host + ``check`` (security scan / moderation) runs next and must allow; only + then does ``commit`` land, carrying an ``AssessmentRef`` whose + ``candidate_hash`` and ``base_revision`` the host re-validates against + the proposal — the evidence is cryptographically bound to the change. + """ + mutations = self._require_mutations() + record = self.get_proposal(proposal_id) + self._check_owner(record.owner_id, expected_owner_id) + if record.committed: + raise SkillEvolutionError("ALREADY_COMMITTED", proposal_id) + if record.decision != "justified": + raise SkillEvolutionError( + "EVIDENCE_REQUIRED", + f"proposal evaluated '{record.decision}': {'; '.join(record.reasons)}", + ) + + proposal = await mutations.get_proposal(proposal_id=proposal_id) + check = await mutations.check(proposal_id=proposal_id) + if check.decision != "allow": + raise SkillEvolutionError("HOST_CHECK_DENIED", f"{check.reason_code} (decision={check.decision})") + + assessment = AssessmentRef( + candidate_hash=proposal.candidate_hash, + base_revision=proposal.base_revision, + evaluator=EVALUATOR_NAME, + evaluator_version=EVALUATOR_VERSION, + report_id=record.report_id, + ) + operation = await mutations.commit( + proposal_id=proposal_id, + idempotency_key=idempotency_key or uuid.uuid4().hex, + assessment_ref=assessment, + ) + record.operation_id = operation.operation_id + # ``publication`` is the operation's publication state (PREPARED / + # APPLIED / NO_CHANGE / ABORTED / NEEDS_REPAIR), not its ``views`` + # state (PENDING / READY). A commit that the host accepted is either + # still publishing (PREPARED) or already landed (APPLIED). + record.committed = operation.publication in {"PREPARED", "APPLIED", "NO_CHANGE"} + return operation + + async def rollback(self, *, operation_id: str, idempotency_key: str | None = None, expected_owner_id: str | None = None) -> Any: + """Revert a committed revision to its known-good predecessor. + + The revert is itself an audited ``Operation`` (history stays append-only), + so a rollback is as traceable as the change it undoes. + """ + mutations = self._require_mutations() + operation = await mutations.get_operation(operation_id=operation_id) + self._check_owner(operation.owner_id, expected_owner_id) + return await mutations.revert( + operation_id=operation_id, + expected_current_revision=operation.after_revision, + idempotency_key=idempotency_key or uuid.uuid4().hex, + ) + + @staticmethod + def _check_owner(owner_id: str, expected_owner_id: str | None) -> None: + """Refuse to act on a skill owned by someone other than the caller. + + Omitting ``expected_owner_id`` keeps the trusted in-process path (tests, + a host service) usable; every user-facing route passes it. + """ + if expected_owner_id is not None and owner_id != expected_owner_id: + raise SkillEvolutionError("FORBIDDEN", "this skill belongs to another user") + + def _require_mutations(self) -> SkillMutationService: + if self._mutations is None: + raise SkillEvolutionError("NOT_STARTED", "service.start(deps) has not bound the host capability") + return self._mutations + + +# Request models live at module scope: this file uses ``from __future__ import +# annotations``, and FastAPI cannot resolve a string forward reference to a +# class that only exists inside ``build_router``'s local namespace. +class ProposeRequest(BaseModel): + model_config = {"extra": "forbid"} + skill_name: str = Field(min_length=1, max_length=64) + candidate_content: str = Field(min_length=1) + thread_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + idempotency_key: str | None = None + + +class ApplyRequest(BaseModel): + model_config = {"extra": "forbid"} + idempotency_key: str | None = None + + +class RevertRequest(BaseModel): + model_config = {"extra": "forbid"} + idempotency_key: str | None = None + + +def build_router(service: EvolutionService): + """Expose the closed loop as a small HTTP surface for the README walkthrough. + + Every route that touches a skill binds the caller: the host's request + principal must own the skill, and a proposal's owner is re-checked on + ``apply`` and ``revert``. The capability handles injected into the service + are operator-scoped, not caller-scoped, so without this binding any + authenticated session user could drive the loop against another owner's + committed skill (see ``backend/packages/harness/deerflow/extensions/AGENTS.md``). + """ + router = APIRouter(prefix="/api/extension-skill-evolution", tags=["skill-evolution"]) + + def _principal(request: Request) -> ExtensionPrincipal: + principal = resolve_principal(request) + if principal is None: + raise HTTPException(status_code=401, detail="authentication required") + return principal + + async def _require_visible_run(request: Request, thread_id: str, run_id: str) -> None: + """Confirm the evidence anchor is visible to the caller, when the host says. + + ``skill_mutations`` / ``completed_run_evidence`` in ``ExtensionRuntimeDeps`` + are allowlist-scoped for the extension, not scoped to the requesting user. + A host that offers the request-scoped reader answers visibility here; + hosts without it (``None``) fall back to the owner check alone. + """ + try: + reader = resolve_run_evidence_reader(request) + if reader is None: + return + status = await reader.get_run_status(thread_id=thread_id, run_id=run_id) + except NotImplementedError: + return + if status is None: + raise SkillEvolutionError("SOURCE_NOT_VISIBLE", f"run {run_id} is not visible to this caller") + + def _guard(coro, *, case_errors: bool = False): + """Translate named example/host errors into HTTP statuses. + + ``case_errors`` is set only on routes that load the operator case files + (``POST /proposals``; ``/health`` handles them itself). Elsewhere a + ``ValueError`` is a host or example bug and must surface as a 500 — not + as a misleading ``CASE_INVALID`` pointing at the case directory. + """ + + async def run(): + try: + return await coro + except SkillEvolutionError as exc: + status = _STATUS_BY_CODE.get(exc.code, 400) + raise HTTPException(status_code=status, detail=str(exc)) from exc + except PermissionError as exc: + # Raised by the host's request-scoped resolvers on denial. + raise HTTPException(status_code=403, detail=str(exc)) from exc + except HostCapabilityError as exc: + # A missing run raises ``NOT_FOUND``; the skill-mutation host + # raises ``NOT_FOUND_OR_FORBIDDEN`` for an unknown skill, + # proposal or operation. Both read as 404 (never disclosing + # which), not as a conflict; everything else is a 409. + status = 404 if exc.code in {"NOT_FOUND", "NOT_FOUND_OR_FORBIDDEN"} else 409 + raise HTTPException(status_code=status, detail=str(exc)) from exc + except ValueError as exc: + # A broken operator-supplied case file fails closed + # (`load_cases_dir` raises), but as a 400 diagnostic rather + # than an unhandled 500 — only where cases are actually read. + if not case_errors: + raise + raise HTTPException(status_code=400, detail=f"CASE_INVALID: {exc}") from exc + + return run() + + @router.get("/health") + async def health() -> dict[str, Any]: + try: + case_count = len(service.cases()) + except ValueError as exc: + # A broken case file must not turn readiness into a 500 traceback. + raise HTTPException(status_code=400, detail=f"CASE_INVALID: {exc}") from exc + return {"ready": service.ready, "evaluator": EVALUATOR_NAME, "evaluator_version": EVALUATOR_VERSION, "cases": case_count} + + @router.post("/proposals") + async def create_proposal(body: ProposeRequest, request: Request) -> dict[str, Any]: + principal = _principal(request) + + async def work(): + await _require_visible_run(request, body.thread_id, body.run_id) + source_ref = await service.resolve_source(thread_id=body.thread_id, run_id=body.run_id) + record = await service.propose( + skill_name=body.skill_name, + candidate_content=body.candidate_content, + source_ref=source_ref, + idempotency_key=body.idempotency_key, + expected_owner_id=principal.user_id, + ) + return { + "proposal_id": record.proposal_id, + "decision": record.decision, + "reasons": list(record.reasons), + "report_id": record.report_id, + "report": record.report, + } + + return await _guard(work(), case_errors=True) + + @router.get("/proposals") + async def list_proposals(request: Request) -> dict[str, Any]: + principal = _principal(request) + return { + "proposals": [ + { + "proposal_id": record.proposal_id, + "name": record.name, + "decision": record.decision, + "committed": record.committed, + "operation_id": record.operation_id, + "report_id": record.report_id, + } + for record in service.list_proposals() + if record.owner_id == principal.user_id + ] + } + + @router.get("/proposals/{proposal_id}") + async def read_proposal(proposal_id: str, request: Request) -> dict[str, Any]: + principal = _principal(request) + try: + record = service.get_proposal(proposal_id) + except SkillEvolutionError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + # Do not disclose another owner's proposal (or even that it exists). + if record.owner_id != principal.user_id: + raise HTTPException(status_code=404, detail=f"NO_SUCH_PROPOSAL: {proposal_id}") + return {"proposal_id": record.proposal_id, "decision": record.decision, "reasons": list(record.reasons), "committed": record.committed, "operation_id": record.operation_id, "report": record.report} + + @router.post("/proposals/{proposal_id}/apply") + async def apply_proposal(proposal_id: str, body: ApplyRequest, request: Request) -> dict[str, Any]: + principal = _principal(request) + + async def work(): + operation = await service.apply(proposal_id=proposal_id, idempotency_key=body.idempotency_key, expected_owner_id=principal.user_id) + return { + "operation_id": operation.operation_id, + "publication": operation.publication, + "before_revision": operation.before_revision.content_digest, + "after_revision": operation.after_revision.content_digest, + } + + return await _guard(work()) + + @router.post("/operations/{operation_id}/revert") + async def revert_operation(operation_id: str, body: RevertRequest, request: Request) -> dict[str, Any]: + principal = _principal(request) + + async def work(): + operation = await service.rollback(operation_id=operation_id, idempotency_key=body.idempotency_key, expected_owner_id=principal.user_id) + return {"operation_id": operation.operation_id, "publication": operation.publication, "reverts": operation.reverts_operation_id} + + return await _guard(work()) + + return router + + +#: Caller-facing failure codes mapped to HTTP statuses by ``build_router``. +_STATUS_BY_CODE = { + "EVIDENCE_REQUIRED": 409, + "HOST_CHECK_DENIED": 409, + "ALREADY_COMMITTED": 409, + "FORBIDDEN": 403, + "SOURCE_NOT_VISIBLE": 403, + # Only the in-memory proposal cache knows ids this process never staged. + "NO_SUCH_PROPOSAL": 404, +} diff --git a/examples/deerflow-extension-skill-evolution/pyproject.toml b/examples/deerflow-extension-skill-evolution/pyproject.toml new file mode 100644 index 0000000000..fba02c9c10 --- /dev/null +++ b/examples/deerflow-extension-skill-evolution/pyproject.toml @@ -0,0 +1,44 @@ +[project] +name = "deerflow-extension-skill-evolution" +version = "0.1.0" +description = "Example: evidence-gated skill self-evolution through the host SkillMutationService contract" +readme = "README.md" +requires-python = ">=3.12" +license = "MIT" +dependencies = [ + "deerflow-extension-api>=0.2.11,<0.3", + "fastapi>=0.115.0,<1", + "pydantic>=2.12,<3", +] + +[project.entry-points."deerflow.extensions"] +skill-evolution = "deerflow_extension_skill_evolution:install" + +[project.optional-dependencies] +dev = [ + "httpx>=0.28,<1", + "pytest>=9,<10", + "pytest-asyncio>=1.3,<2", + "ruff>=0.14,<1", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["deerflow_extension_skill_evolution"] + +[tool.ruff] +line-length = 240 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP"] + +[tool.ruff.lint.isort] +known-first-party = ["deerflow_extension_skill_evolution"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" diff --git a/examples/deerflow-extension-skill-evolution/tests/test_evaluator.py b/examples/deerflow-extension-skill-evolution/tests/test_evaluator.py new file mode 100644 index 0000000000..25a5e2b612 --- /dev/null +++ b/examples/deerflow-extension-skill-evolution/tests/test_evaluator.py @@ -0,0 +1,201 @@ +"""Offline unit tests for the static evaluator: no host, no package content executed.""" + +import json + +import pytest + +from deerflow_extension_skill_evolution.evaluator import ( + EVALUATOR_NAME, + EVALUATOR_VERSION, + Case, + Check, + evaluate, + load_cases_dir, + report_id, + run_check, +) + + +def pkg(**files: str) -> dict[str, bytes]: + return {name: content.encode("utf-8") for name, content in files.items()} + + +MAIN_OK = "---\nname: example-skill\ndescription: demo\n---\n\nRun `scripts/run.py --format json` to convert.\n" +MAIN_BAD_ARG = MAIN_OK.replace("--format json", "--fmt json") +SCRIPT_OK = "import json\nprint(json.dumps({'ok': True}))\n" +SCRIPT_BROKEN = "def f(:\n" + + +# --- check kinds ----------------------------------------------------------- + + +def test_file_exists_both_ways(): + package = pkg(**{"SKILL.md": MAIN_OK}) + assert run_check(Check("file_exists", path="SKILL.md"), package).passed + missing = run_check(Check("file_exists", path="scripts/run.py"), package) + assert not missing.passed and "missing scripts/run.py" in missing.detail + + +def test_text_contains_and_not_contains(): + package = pkg(**{"SKILL.md": MAIN_OK}) + assert run_check(Check("text_contains", path="SKILL.md", needle="--format json"), package).passed + failed = run_check(Check("text_contains", path="SKILL.md", needle="--format yaml"), package) + assert not failed.passed and "--format yaml" in failed.detail + assert run_check(Check("text_not_contains", path="SKILL.md", needle="--fmt"), package).passed + still = run_check(Check("text_not_contains", path="SKILL.md", needle="--format"), package) + assert not still.passed + + +def test_text_checks_report_missing_file_and_bad_encoding(): + assert not run_check(Check("text_contains", path="SKILL.md", needle="x"), {}).passed + binary = {"SKILL.md": b"\xff\xfe not utf-8"} + outcome = run_check(Check("text_contains", path="SKILL.md", needle="x"), binary) + assert not outcome.passed and "not valid UTF-8" in outcome.detail + + +def test_referenced_files_exist_flags_dangling_script_reference(): + baseline = pkg(**{"SKILL.md": MAIN_OK}) + outcome = run_check(Check("referenced_files_exist", path="SKILL.md"), baseline) + assert not outcome.passed and "scripts/run.py" in outcome.detail + fixed = pkg(**{"SKILL.md": MAIN_OK, "scripts/run.py": SCRIPT_OK}) + assert run_check(Check("referenced_files_exist", path="SKILL.md"), fixed).passed + + +def test_referenced_files_exist_ignores_urls_and_trims_sentence_punctuation(): + # A URL path is not an in-package reference; a bare token still is. + prose = pkg(**{"SKILL.md": "Use the scripts/run helper. See https://example.com/scripts/run.py docs.\n"}) + outcome = run_check(Check("referenced_files_exist", path="SKILL.md"), prose) + assert not outcome.passed + assert "scripts/run" in outcome.detail + assert "example.com" not in outcome.detail + + # A file named at the end of a sentence is the file, not "run.py.". + fixed = pkg(**{"SKILL.md": "Run scripts/run.py.\n", "scripts/run.py": SCRIPT_OK}) + assert run_check(Check("referenced_files_exist", path="SKILL.md"), fixed).passed + + +def test_scripts_parse_is_syntax_only_and_never_executes(): + # A script that would print if executed; parse must not run it. + hostile = pkg(**{"scripts/run.py": "print('EXECUTED')\n"}) + assert run_check(Check("scripts_parse"), hostile).passed + broken = pkg(**{"scripts/run.py": SCRIPT_BROKEN}) + outcome = run_check(Check("scripts_parse"), broken) + assert not outcome.passed and "SyntaxError" not in outcome.detail and "line 1" in outcome.detail + # Non-UTF8 scripts are reported, not crashed on. + binary = {"scripts/run.py": b"\xff\xfe"} + assert not run_check(Check("scripts_parse"), binary).passed + # No scripts at all passes vacuously. + assert run_check(Check("scripts_parse"), pkg(**{"SKILL.md": MAIN_OK})).passed + + +# --- validation ------------------------------------------------------------ + + +@pytest.mark.parametrize( + "path", + ["", "/abs/path", "..\\win", "a/../b", "./x", "a//b", "x/" + "y" * 1100], +) +def test_case_path_validation_rejects_unsafe_paths(path): + with pytest.raises(ValueError): + Check.from_json({"kind": "file_exists", "path": path}) + + +def test_check_from_json_requires_kind_specific_fields(): + with pytest.raises(ValueError): + Check.from_json({"kind": "nope"}) + with pytest.raises(ValueError): + Check.from_json({"kind": "file_exists"}) # missing path + with pytest.raises(ValueError): + Check.from_json({"kind": "text_contains", "path": "SKILL.md"}) # missing needle + with pytest.raises(ValueError): + Check.from_json({"kind": "text_contains", "path": "SKILL.md", "needle": ""}) + + +def test_case_from_json_validation(): + good = {"id": "arg-order", "description": "converter arg is wrong", "checks": [{"kind": "text_contains", "path": "SKILL.md", "needle": "--format json"}]} + case = Case.from_json(good, origin="case.json") + assert case.case_id == "arg-order" and len(case.checks) == 1 + for bad in ( + {"id": "Upper", "description": "d", "checks": [{"kind": "scripts_parse"}]}, + {"id": "ok", "description": " ", "checks": [{"kind": "scripts_parse"}]}, + {"id": "ok", "description": "d", "checks": []}, + {"id": "x" * 65, "description": "d", "checks": [{"kind": "scripts_parse"}]}, + "not-an-object", + ): + with pytest.raises(ValueError): + Case.from_json(bad, origin="case.json") + + +def test_load_cases_dir_orders_by_name_and_reports_broken_files(tmp_path): + (tmp_path / "01-a.json").write_text(json.dumps({"id": "a", "description": "d", "checks": [{"kind": "scripts_parse"}]}), encoding="utf-8") + (tmp_path / "02-b.json").write_text(json.dumps({"id": "b", "description": "d", "checks": [{"kind": "scripts_parse"}]}), encoding="utf-8") + (tmp_path / "ignored.txt").write_text("x", encoding="utf-8") + cases = load_cases_dir(tmp_path) + assert [case.case_id for case in cases] == ["a", "b"] + + (tmp_path / "03-broken.json").write_text("{not json", encoding="utf-8") + with pytest.raises(ValueError, match="03-broken.json"): + load_cases_dir(tmp_path) + + with pytest.raises(ValueError, match="cases directory not found"): + load_cases_dir(tmp_path / "missing") + + +# --- decisions ------------------------------------------------------------- + +ARG_CASE = Case( + case_id="converter-arg", + description="SKILL.md must instruct --format json, not --fmt", + checks=(Check("text_contains", path="SKILL.md", needle="--format json"),), +) +PARSE_CASE = Case(case_id="scripts-parse", description="bundled scripts must parse", checks=(Check("scripts_parse"),)) + + +def test_justified_when_candidate_fixes_a_recorded_baseline_failure(): + baseline = pkg(**{"SKILL.md": MAIN_BAD_ARG, "scripts/run.py": SCRIPT_OK}) + candidate = pkg(**{"SKILL.md": MAIN_OK, "scripts/run.py": SCRIPT_OK}) + evaluation = evaluate([ARG_CASE, PARSE_CASE], baseline, candidate) + assert evaluation.decision == "justified" + assert "converter-arg" in evaluation.reasons[0] + + +def test_unjustified_when_baseline_already_passes_everything(): + package = pkg(**{"SKILL.md": MAIN_OK, "scripts/run.py": SCRIPT_OK}) + evaluation = evaluate([ARG_CASE], package, package) + assert evaluation.decision == "unjustified" + + +def test_failed_when_candidate_still_fails(): + baseline = pkg(**{"SKILL.md": MAIN_BAD_ARG}) + candidate = pkg(**{"SKILL.md": MAIN_BAD_ARG.replace("--fmt", "--format"), "scripts/run.py": SCRIPT_BROKEN}) + evaluation = evaluate([ARG_CASE, PARSE_CASE], baseline, candidate) + assert evaluation.decision == "failed" and "scripts-parse" in evaluation.reasons[0] + + +def test_failed_when_no_cases_recorded(): + evaluation = evaluate([], pkg(), pkg()) + assert evaluation.decision == "failed" and "no regression cases" in evaluation.reasons[0] + + +# --- report determinism ---------------------------------------------------- + + +def test_report_is_deterministic_and_report_id_is_content_addressed(): + baseline = pkg(**{"SKILL.md": MAIN_BAD_ARG}) + candidate = pkg(**{"SKILL.md": MAIN_OK}) + first = evaluate([ARG_CASE], baseline, candidate).report(candidate_hash="a" * 64) + second = evaluate([ARG_CASE], baseline, candidate).report(candidate_hash="a" * 64) + assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True) + assert report_id(first) == report_id(second) + assert first["evaluator"] == EVALUATOR_NAME and first["evaluator_version"] == EVALUATOR_VERSION + assert first["decision"] == "justified" + assert first["cases"][0]["baseline_passed"] is False and first["cases"][0]["candidate_passed"] is True + # A different candidate hash changes the report identity. + other = evaluate([ARG_CASE], baseline, candidate).report(candidate_hash="b" * 64) + assert report_id(other) != report_id(first) + + +def test_detail_is_clipped_to_bound(): + huge = pkg(**{"SKILL.md": "x" * 5000}) + outcome = run_check(Check("text_contains", path="SKILL.md", needle="y" * 5000), huge) + assert len(outcome.detail) <= 300 diff --git a/examples/deerflow-extension-skill-evolution/tests/test_lifecycle.py b/examples/deerflow-extension-skill-evolution/tests/test_lifecycle.py new file mode 100644 index 0000000000..d729b3ea12 --- /dev/null +++ b/examples/deerflow-extension-skill-evolution/tests/test_lifecycle.py @@ -0,0 +1,352 @@ +"""End-to-end orchestration test against a fake host implementing only the public contract. + +CI installs this package with the extension-api alone (no harness), so the fake +host here mirrors what ``HostSkillMutationService`` guarantees: content-addressed +revisions, baseline/candidate bundles, an allow-only check gate, commit that +re-validates the assessment against the proposal, and an append-only revert. +The point is that the example needs nothing beyond the contract. +""" + +import hashlib +import json +from dataclasses import dataclass, replace +from typing import Any + +import pytest +from deerflow_extension_api import ( + AssessmentRef, + AssetRevision, + BundleFile, + CompletedRunEventPage, + CompletedRunSnapshot, + EvidenceLimits, + ExtensionRuntimeDeps, + HostCapabilityError, + HostCheckResult, + MutationCapabilities, + Operation, + Proposal, + ProposalBundle, + SkillRevisionView, +) +from deerflow_extension_api.run_evidence import RunPage + +from deerflow_extension_skill_evolution import install +from deerflow_extension_skill_evolution.evaluator import Case, Check +from deerflow_extension_skill_evolution.service import EvolutionService, SkillEvolutionError + +SKILL = "example-skill" +OWNER = "owner" +BASE_MAIN = "---\nname: example-skill\ndescription: demo\n---\n\nRun `scripts/run.py --fmt json`.\n" +FIXED_MAIN = BASE_MAIN.replace("--fmt json", "--format json") +SCRIPT = "import json\nprint(json.dumps({'ok': True}))\n" + +ARG_CASE = Case( + case_id="converter-arg", + description="SKILL.md must instruct --format json, not --fmt", + checks=(Check("text_contains", path="SKILL.md", needle="--format json"),), +) + + +def _digest(files: dict[str, bytes]) -> str: + canonical = json.dumps({k: hashlib.sha256(v).hexdigest() for k, v in sorted(files.items())}, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(canonical).hexdigest() + + +def _bundle(files: dict[str, bytes]) -> tuple[BundleFile, ...]: + return tuple(BundleFile(path=p, content=c, sha256=hashlib.sha256(c).hexdigest(), executable=p.startswith("scripts/") and p.endswith(".py")) for p, c in sorted(files.items())) + + +@dataclass +class _OpRow: + operation_id: str + publication: str + before_revision: AssetRevision + after_revision: AssetRevision + assessment: dict[str, Any] | None + reverts_operation_id: str | None = None + + +class FakeMutationService: + """A contract-faithful in-memory host for the mutation lifecycle.""" + + def __init__(self, *, files: dict[str, bytes] | None = None, check_decision: str = "allow") -> None: + self._files = files if files is not None else {"SKILL.md": BASE_MAIN.encode(), "scripts/run.py": SCRIPT.encode()} + self._check_decision = check_decision + self._seq = 0 + self._proposals: dict[str, Proposal] = {} + self._bundles: dict[str, tuple[tuple[BundleFile, ...], tuple[BundleFile, ...]]] = {} + self._operations: dict[str, _OpRow] = {} + self._committed: set[str] = set() + self._discarded: set[str] = set() + + @property + def capabilities(self) -> MutationCapabilities: + return MutationCapabilities(operations=("stage", "check", "commit", "revert")) + + def _current_revision(self) -> AssetRevision: + return AssetRevision(incarnation_id="inc-1", mutation_seq=self._seq, content_digest=_digest(self._files)) + + async def read_skill(self, *, source_ref: str, name: str) -> SkillRevisionView: + if name != SKILL: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN", name) + rev = self._current_revision() + return SkillRevisionView(target_ref=f"asset:{name}", owner_id=OWNER, name=name, revision=rev, content=self._files["SKILL.md"].decode(), enabled=True) + + async def stage(self, *, source_refs: tuple[str, ...], target_ref: str, expected_revision: AssetRevision, content: str, idempotency_key: str) -> Proposal: + if expected_revision != self._current_revision(): + raise HostCapabilityError("REVISION_CONFLICT") + candidate = {**self._files, "SKILL.md": content.encode()} + pid = f"prop-{len(self._proposals) + 1}" + base = self._current_revision() + proposal = Proposal( + proposal_id=pid, + owner_id=OWNER, + target_ref=target_ref, + name=SKILL, + base_revision=base, + candidate_hash=_digest(candidate), + state="STAGED", + expires_at="2999-01-01T00:00:00Z", + source_refs=tuple(source_refs), + ) + self._proposals[pid] = proposal + self._bundles[pid] = (_bundle(self._files), _bundle(candidate)) + return proposal + + async def get_proposal(self, *, proposal_id: str) -> Proposal: + proposal = self._proposals.get(proposal_id) + if proposal is None: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN", proposal_id) + return proposal + + async def read_proposal_bundle(self, *, proposal_id: str, max_bytes: int = 33554432) -> ProposalBundle: + proposal = await self.get_proposal(proposal_id=proposal_id) + baseline, candidate = self._bundles[proposal_id] + return ProposalBundle(proposal_id=proposal_id, baseline=baseline, candidate=candidate, base_revision=proposal.base_revision, candidate_hash=proposal.candidate_hash) + + async def discard(self, *, proposal_id: str) -> Proposal: + await self.get_proposal(proposal_id=proposal_id) + self._discarded.add(proposal_id) + return replace(self._proposals[proposal_id], state="DISCARDED") + + async def check(self, *, proposal_id: str) -> HostCheckResult: + proposal = await self.get_proposal(proposal_id=proposal_id) + return HostCheckResult( + proposal_id=proposal_id, + decision=self._check_decision, + candidate_hash=proposal.candidate_hash, + base_revision=proposal.base_revision, + policy_version="policy-1", + expires_at="2999-01-01T00:00:00Z", + reason_code="CHECK_PASSED" if self._check_decision == "allow" else "CHECK_BLOCKED", + ) + + async def commit(self, *, proposal_id: str, idempotency_key: str, assessment_ref: AssessmentRef | None = None) -> Operation: + proposal = await self.get_proposal(proposal_id=proposal_id) + if proposal_id in self._committed: + raise HostCapabilityError("PROPOSAL_ALREADY_COMMITTED") + if proposal_id in self._discarded: + raise HostCapabilityError("PROPOSAL_DISCARDED") + # The host re-validates the assessment against the exact proposal. + if assessment_ref is not None and (assessment_ref.candidate_hash != proposal.candidate_hash or assessment_ref.base_revision != proposal.base_revision): + raise HostCapabilityError("INVALID_ASSESSMENT") + before = self._current_revision() + # Recover the candidate bytes from the staged bundle. + baseline_files, candidate_files = self._bundles[proposal_id] + del baseline_files + self._files = {f.path: f.content for f in candidate_files} + changed = _digest(self._files) != before.content_digest + self._seq += int(changed) + after = AssetRevision(before.incarnation_id, before.mutation_seq + int(changed), _digest(self._files)) + op_id = f"op-{len(self._operations) + 1}" + assessment = None + if assessment_ref is not None: + assessment = {"candidate_hash": assessment_ref.candidate_hash, "evaluator": assessment_ref.evaluator, "report_id": assessment_ref.report_id} + # ``publication``, not ``views``: a committed change is APPLIED (or + # NO_CHANGE when the candidate is identical to the baseline). The real + # host never reports READY here -- that is a ``views`` state. + self._operations[op_id] = _OpRow(operation_id=op_id, publication="APPLIED" if changed else "NO_CHANGE", before_revision=before, after_revision=after, assessment=assessment) + self._committed.add(proposal_id) + return self._operation_view(op_id) + + async def revert(self, *, operation_id: str, expected_current_revision: AssetRevision, idempotency_key: str) -> Operation: + row = self._operations.get(operation_id) + if row is None: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN", operation_id) + if expected_current_revision != self._current_revision(): + raise HostCapabilityError("REVISION_CONFLICT") + # Revert to the revision that operation replaced. + self._files = {f.path: f.content for f in self._bundles_of_operation(operation_id)} + self._seq += 1 + after = AssetRevision("inc-1", self._seq, _digest(self._files)) + new_id = f"op-{len(self._operations) + 1}" + self._operations[new_id] = _OpRow(operation_id=new_id, publication="APPLIED", before_revision=expected_current_revision, after_revision=after, assessment=None, reverts_operation_id=operation_id) + return self._operation_view(new_id) + + def _bundles_of_operation(self, operation_id: str) -> tuple[BundleFile, ...]: + # The pre-image of a commit is the baseline bundle it staged from. + for pid, row in self._operations.items(): + if row.operation_id == operation_id: + for staged_pid, proposal in self._proposals.items(): + baseline, _candidate = self._bundles[staged_pid] + return baseline + return _bundle(self._files) + + async def get_operation(self, *, operation_id: str) -> Operation: + if operation_id not in self._operations: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN", operation_id) + return self._operation_view(operation_id) + + async def find_operation(self, *, idempotency_key: str, method: str = "commit") -> Operation | None: + return None + + def _operation_view(self, op_id: str) -> Operation: + row = self._operations[op_id] + return Operation( + operation_id=row.operation_id, + owner_id=OWNER, + name=SKILL, + publication=row.publication, + views="READY", + before_revision=row.before_revision, + after_revision=row.after_revision, + generation=row.after_revision.mutation_seq, + reverts_operation_id=row.reverts_operation_id, + ) + + +class FakeEvidence: + """Minimal sealed-run evidence source keyed by (thread_id, run_id).""" + + def __init__(self, snapshot_ref: str = "source") -> None: + self._ref = snapshot_ref + + async def list_changed_runs(self, *, cursor: str | None = None, limit: int = 200) -> RunPage: + return RunPage(items=()) + + async def get_snapshot(self, *, thread_id: str, run_id: str, limits: EvidenceLimits = EvidenceLimits()) -> CompletedRunSnapshot: + return CompletedRunSnapshot(snapshot_ref=self._ref, run_id=run_id, thread_id=thread_id, owner_id=OWNER, agent_id="lead_agent", origin="interactive", status="success", seal_state="sealed", coverage="lead-journal-v1") + + async def read_events(self, *, snapshot_ref: str, cursor: str | None = None, limits: EvidenceLimits = EvidenceLimits()) -> CompletedRunEventPage: + return CompletedRunEventPage() + + +@pytest.fixture +def service(tmp_path): + cases_dir = tmp_path / "cases" + cases_dir.mkdir() + (cases_dir / "01-arg.json").write_text( + json.dumps({"id": "converter-arg", "description": ARG_CASE.description, "checks": [{"kind": "text_contains", "path": "SKILL.md", "needle": "--format json"}]}), + encoding="utf-8", + ) + return EvolutionService(cases_dir=cases_dir) + + +async def _start(service: EvolutionService, mutations: FakeMutationService) -> None: + deps = ExtensionRuntimeDeps(skill_mutations=mutations, completed_run_evidence=FakeEvidence()) + await service.start(deps) + + +async def test_full_loop_propose_apply_rollback(service): + mutations = FakeMutationService() + await _start(service, mutations) + + source = await service.resolve_source(thread_id="t", run_id="r") + record = await service.propose(skill_name=SKILL, candidate_content=FIXED_MAIN, source_ref=source) + + assert record.decision == "justified" + assert record.candidate_hash and len(record.report_id) == 64 + + operation = await service.apply(proposal_id=record.proposal_id) + assert operation.publication == "APPLIED" + assert record.committed is True + assert operation.before_revision.content_digest != operation.after_revision.content_digest + # The committed operation carries the evaluator's assessment. + assert mutations._operations[operation.operation_id].assessment["report_id"] == record.report_id + + revert = await service.rollback(operation_id=operation.operation_id) + assert revert.reverts_operation_id == operation.operation_id + assert mutations._files["SKILL.md"].decode() == BASE_MAIN # back to known-good + + +async def test_apply_refuses_without_justifying_evidence(service): + mutations = FakeMutationService() + await _start(service, mutations) + source = await service.resolve_source(thread_id="t", run_id="r") + # A recorded case the baseline already satisfies: the change fixes nothing + # on record, so the proposal is unjustified and must not commit. + passing_on_both = Case(case_id="scripts-parse", description="bundled scripts parse", checks=(Check("scripts_parse"),)) + record = await service.propose(skill_name=SKILL, candidate_content=FIXED_MAIN, source_ref=source, cases=[passing_on_both]) + assert record.decision == "unjustified" + assert record.proposal_id in mutations._discarded # dead proposal cleaned up + with pytest.raises(SkillEvolutionError) as exc: + await service.apply(proposal_id=record.proposal_id) + assert exc.value.code == "EVIDENCE_REQUIRED" + + +async def test_apply_refuses_when_candidate_still_fails(service): + mutations = FakeMutationService() + await _start(service, mutations) + source = await service.resolve_source(thread_id="t", run_id="r") + broken = FIXED_MAIN.replace("--format json", "--format xml") + record = await service.propose(skill_name=SKILL, candidate_content=broken, source_ref=source) + assert record.decision == "failed" + with pytest.raises(SkillEvolutionError) as exc: + await service.apply(proposal_id=record.proposal_id) + assert exc.value.code == "EVIDENCE_REQUIRED" + + +async def test_apply_blocks_when_host_check_denies(service): + mutations = FakeMutationService(check_decision="block") + await _start(service, mutations) + source = await service.resolve_source(thread_id="t", run_id="r") + record = await service.propose(skill_name=SKILL, candidate_content=FIXED_MAIN, source_ref=source) + assert record.decision == "justified" + with pytest.raises(SkillEvolutionError) as exc: + await service.apply(proposal_id=record.proposal_id) + assert exc.value.code == "HOST_CHECK_DENIED" + + +async def test_apply_rejects_double_commit(service): + mutations = FakeMutationService() + await _start(service, mutations) + source = await service.resolve_source(thread_id="t", run_id="r") + record = await service.propose(skill_name=SKILL, candidate_content=FIXED_MAIN, source_ref=source) + await service.apply(proposal_id=record.proposal_id) + # A successful apply must flip the record's committed flag; otherwise the + # ALREADY_COMMITTED guard below never fires against a real host. + assert record.committed is True + with pytest.raises(SkillEvolutionError) as exc: + await service.apply(proposal_id=record.proposal_id) + assert exc.value.code == "ALREADY_COMMITTED" + + +async def test_start_requires_the_grant(service): + with pytest.raises(SkillEvolutionError) as exc: + await service.start(ExtensionRuntimeDeps(skill_mutations=None)) + assert exc.value.code == "NOT_GRANTED" + + +@pytest.mark.parametrize("value", [0, 1, "false", "true", None]) +def test_install_rejects_a_non_boolean_enabled_switch(value): + """A mistyped off switch must fail the deployment instead of staying on.""" + with pytest.raises(ValueError, match="config.enabled must be a boolean"): + install(None, {"enabled": value}) + + +def test_install_accepts_an_explicit_off_switch(): + install(None, {"enabled": False}) + + +async def test_propose_before_start_raises(service): + with pytest.raises(SkillEvolutionError) as exc: + await service.propose(skill_name=SKILL, candidate_content=FIXED_MAIN, source_ref="source") + assert exc.value.code == "NOT_STARTED" + + +async def test_resolve_source_requires_evidence_grant(service): + await service.start(ExtensionRuntimeDeps(skill_mutations=FakeMutationService(), completed_run_evidence=None)) + with pytest.raises(SkillEvolutionError) as exc: + await service.resolve_source(thread_id="t", run_id="r") + assert exc.value.code == "NO_EVIDENCE" diff --git a/examples/deerflow-extension-skill-evolution/tests/test_routes.py b/examples/deerflow-extension-skill-evolution/tests/test_routes.py new file mode 100644 index 0000000000..867a2b3969 --- /dev/null +++ b/examples/deerflow-extension-skill-evolution/tests/test_routes.py @@ -0,0 +1,330 @@ +"""HTTP-surface tests for the walkthrough routes. + +The lifecycle tests drive ``EvolutionService`` directly, so they cannot see the +FastAPI plumbing at all. That gap is real: with ``from __future__ import +annotations``, a request model defined inside ``build_router`` becomes an +unresolvable forward reference, FastAPI silently treats ``body`` as a query +parameter, and every POST answers 422 while every service-level test stays +green. These tests go through ``TestClient`` so the request models are actually +resolved, validated, and rejected the way a caller experiences them. + +They also pin caller binding: the host capability handles the service holds are +operator-scoped, so the routes — not the host — must refuse to act on another +user's skill. +""" + +import asyncio +import json + +from deerflow_extension_api import ExtensionPrincipal, ExtensionRuntimeDeps, HostCapabilityError +from deerflow_extension_api.auth import EXTENSION_PRINCIPAL_RESOLVER_KEY +from deerflow_extension_api.run_evidence import RUN_EVIDENCE_READER_RESOLVER_KEY, RunStatusView +from fastapi import FastAPI +from fastapi.testclient import TestClient +from test_lifecycle import OWNER, FakeEvidence, FakeMutationService + +from deerflow_extension_skill_evolution.service import EvolutionService, build_router + +SKILL = "example-skill" +PREFIX = "/api/extension-skill-evolution" +BASE_MAIN = "---\nname: example-skill\ndescription: demo\n---\n\nRun `scripts/run.py --fmt json`.\n" +FIXED_MAIN = BASE_MAIN.replace("--fmt json", "--format json") +CASE_JSON = json.dumps({"id": "converter-arg", "description": "SKILL.md must instruct --format json", "checks": [{"kind": "text_contains", "path": "SKILL.md", "needle": "--format json"}]}) + + +def _as(client: TestClient, user_id: str | None) -> None: + """Switch the request principal this client resolves (``None`` = anonymous).""" + + def resolver(request): + return None if user_id is None else ExtensionPrincipal(user_id=user_id) + + setattr(client.app.state, EXTENSION_PRINCIPAL_RESOLVER_KEY, resolver) + + +class _MissingEvidence(FakeEvidence): + """A host whose sealed-run reader answers ``NOT_FOUND`` for every anchor.""" + + async def get_snapshot(self, **kwargs): + raise HostCapabilityError("NOT_FOUND", "run is unavailable in this scope") + + +class _ValueErrorOnCheck(FakeMutationService): + """A host invariant bug raised from ``check`` on the apply route.""" + + async def check(self, *, proposal_id: str): + raise ValueError("host invariant failed") + + +class _ValueErrorOnGetOperation(FakeMutationService): + """A host invariant bug raised before ``revert`` can run.""" + + async def get_operation(self, *, operation_id: str): + raise ValueError("host invariant failed") + + +class _InvisibleRunReader: + """A request-scoped reader for which the anchor run does not exist.""" + + async def get_run_status(self, *, thread_id: str, run_id: str): + return None + + +class _VisibleRunReader: + """A request-scoped reader that confirms the anchor run is visible.""" + + async def get_run_status(self, *, thread_id: str, run_id: str): + return RunStatusView(thread_id=thread_id, run_id=run_id, status="success") + + +def _client( + tmp_path, + mutations: FakeMutationService | None = None, + *, + user_id: str | None = OWNER, + case_text: str = CASE_JSON, + evidence: FakeEvidence | None = None, + raise_server_exceptions: bool = True, +) -> tuple[TestClient, FakeMutationService]: + cases_dir = tmp_path / "cases" + cases_dir.mkdir() + (cases_dir / "01-arg.json").write_text(case_text, encoding="utf-8") + service = EvolutionService(cases_dir=cases_dir) + host = mutations if mutations is not None else FakeMutationService() + asyncio.run(service.start(ExtensionRuntimeDeps(skill_mutations=host, completed_run_evidence=evidence if evidence is not None else FakeEvidence()))) + app = FastAPI() + app.include_router(build_router(service)) + client = TestClient(app, raise_server_exceptions=raise_server_exceptions) + _as(client, user_id) + return client, host + + +def _propose(client: TestClient, **overrides): + body = {"skill_name": SKILL, "candidate_content": FIXED_MAIN, "thread_id": "t", "run_id": "r"} | overrides + return client.post(f"{PREFIX}/proposals", json=body) + + +def test_proposals_route_accepts_a_json_body(tmp_path): + """A documented request body must reach the service, not become a query param.""" + client, _host = _client(tmp_path) + + response = _propose(client) + + assert response.status_code == 200, response.text + payload = response.json() + assert payload["decision"] == "justified" + assert len(payload["report_id"]) == 64 + + +def test_proposals_route_rejects_an_incomplete_body(tmp_path): + client, _host = _client(tmp_path) + + response = client.post(f"{PREFIX}/proposals", json={"skill_name": SKILL, "candidate_content": FIXED_MAIN}) + + assert response.status_code == 422 + + +def test_proposals_route_forbids_unknown_fields(tmp_path): + client, _host = _client(tmp_path) + + response = _propose(client, unexpected="value") + + assert response.status_code == 422 + + +def test_apply_route_marks_the_proposal_committed_and_refuses_a_second_apply(tmp_path): + client, _host = _client(tmp_path) + proposal_id = _propose(client).json()["proposal_id"] + + applied = client.post(f"{PREFIX}/proposals/{proposal_id}/apply", json={}) + + assert applied.status_code == 200, applied.text + assert applied.json()["publication"] == "APPLIED" + + listing = client.get(f"{PREFIX}/proposals").json()["proposals"] + committed = {item["proposal_id"]: item["committed"] for item in listing} + assert committed[proposal_id] is True + + again = client.post(f"{PREFIX}/proposals/{proposal_id}/apply", json={}) + assert again.status_code == 409 + assert again.json()["detail"].startswith("ALREADY_COMMITTED") + + +def test_apply_route_returns_409_for_an_unevidenced_change(tmp_path): + client, _host = _client(tmp_path) + broken = FIXED_MAIN.replace("--format json", "--format xml") + proposal = _propose(client, candidate_content=broken).json() + + assert proposal["decision"] == "failed" + response = client.post(f"{PREFIX}/proposals/{proposal['proposal_id']}/apply", json={}) + + assert response.status_code == 409 + assert response.json()["detail"].startswith("EVIDENCE_REQUIRED") + + +def test_revert_route_is_append_only(tmp_path): + client, host = _client(tmp_path) + proposal_id = _propose(client).json()["proposal_id"] + operation_id = client.post(f"{PREFIX}/proposals/{proposal_id}/apply", json={}).json()["operation_id"] + + reverted = client.post(f"{PREFIX}/operations/{operation_id}/revert", json={}) + + assert reverted.status_code == 200, reverted.text + assert reverted.json()["reverts"] == operation_id + assert host._files["SKILL.md"].decode() == BASE_MAIN + + +def test_unknown_proposal_returns_404(tmp_path): + client, _host = _client(tmp_path) + + response = client.get(f"{PREFIX}/proposals/does-not-exist") + + assert response.status_code == 404 + + +def test_unknown_operation_returns_404(tmp_path): + """``NOT_FOUND_OR_FORBIDDEN`` from the real host is a 404, not a conflict.""" + client, _host = _client(tmp_path) + + response = client.post(f"{PREFIX}/operations/does-not-exist/revert", json={}) + + assert response.status_code == 404 + + +def test_apply_unknown_proposal_returns_404(tmp_path): + """A proposal this service never staged is absent, not a conflict.""" + client, _host = _client(tmp_path) + + response = client.post(f"{PREFIX}/proposals/does-not-exist/apply", json={}) + + assert response.status_code == 404 + + +def test_unknown_evidence_run_returns_404(tmp_path): + """``NOT_FOUND`` from the sealed-run reader is a 404 as well.""" + client, _host = _client(tmp_path, evidence=_MissingEvidence()) + + response = _propose(client) + + assert response.status_code == 404 + + +def test_health_route_reports_the_evaluator(tmp_path): + client, _host = _client(tmp_path) + + payload = client.get(f"{PREFIX}/health").json() + + assert payload["ready"] is True + assert payload["evaluator"] + assert payload["evaluator_version"] + assert payload["cases"] == 1 + + +def test_routes_require_an_authenticated_principal(tmp_path): + client, _host = _client(tmp_path, user_id=None) + + assert client.get(f"{PREFIX}/proposals").status_code == 401 + assert _propose(client).status_code == 401 + + +def test_propose_refuses_a_skill_owned_by_someone_else(tmp_path): + """The injected host capability is allowlist-scoped, so the route binds the owner.""" + client, _host = _client(tmp_path, user_id="someone-else") + + response = _propose(client) + + assert response.status_code == 403 + assert response.json()["detail"].startswith("FORBIDDEN") + + +def test_proposals_are_scoped_to_their_owner(tmp_path): + client, _host = _client(tmp_path) + proposal_id = _propose(client).json()["proposal_id"] + + _as(client, "someone-else") + + assert client.get(f"{PREFIX}/proposals").json()["proposals"] == [] + # Another owner's proposal is not disclosed — not even its existence. + assert client.get(f"{PREFIX}/proposals/{proposal_id}").status_code == 404 + applied = client.post(f"{PREFIX}/proposals/{proposal_id}/apply", json={}) + assert applied.status_code == 403 + assert applied.json()["detail"].startswith("FORBIDDEN") + + +def test_revert_is_scoped_to_the_operation_owner(tmp_path): + client, _host = _client(tmp_path) + proposal_id = _propose(client).json()["proposal_id"] + operation_id = client.post(f"{PREFIX}/proposals/{proposal_id}/apply", json={}).json()["operation_id"] + + _as(client, "someone-else") + response = client.post(f"{PREFIX}/operations/{operation_id}/revert", json={}) + + assert response.status_code == 403 + assert response.json()["detail"].startswith("FORBIDDEN") + + +def test_a_broken_case_file_is_a_400_diagnostic(tmp_path): + """A malformed operator case file fails closed, but not as a 500 traceback.""" + client, _host = _client(tmp_path, case_text="{ not json") + + health = client.get(f"{PREFIX}/health") + proposed = _propose(client) + + assert health.status_code == 400 + assert health.json()["detail"].startswith("CASE_INVALID") + assert proposed.status_code == 400 + assert proposed.json()["detail"].startswith("CASE_INVALID") + + +def test_apply_does_not_label_a_host_value_error_as_case_invalid(tmp_path): + """Apply never reads the case files, so a host bug must surface as a 500.""" + client, _host = _client(tmp_path, mutations=_ValueErrorOnCheck(), raise_server_exceptions=False) + proposal_id = _propose(client).json()["proposal_id"] + + response = client.post(f"{PREFIX}/proposals/{proposal_id}/apply", json={}) + + assert response.status_code == 500 + + +def test_revert_does_not_label_a_host_value_error_as_case_invalid(tmp_path): + """Revert never reads the case files either.""" + client, _host = _client(tmp_path, mutations=_ValueErrorOnGetOperation(), raise_server_exceptions=False) + + response = client.post(f"{PREFIX}/operations/does-not-matter/revert", json={}) + + assert response.status_code == 500 + + +def test_hidden_run_anchor_is_403_before_the_host_is_touched(tmp_path): + """The request-scoped reader is the production path; None means not visible.""" + client, host = _client(tmp_path) + setattr(client.app.state, RUN_EVIDENCE_READER_RESOLVER_KEY, lambda request: _InvisibleRunReader()) + + response = _propose(client) + + assert response.status_code == 403 + assert response.json()["detail"].startswith("SOURCE_NOT_VISIBLE") + assert host._proposals == {} + + +def test_denied_run_reader_is_403(tmp_path): + """The Gateway's resolver raises PermissionError without ``runs:read``.""" + client, _host = _client(tmp_path) + + def denied(request): + raise PermissionError("run evidence requires runs:read") + + setattr(client.app.state, RUN_EVIDENCE_READER_RESOLVER_KEY, denied) + + response = _propose(client) + + assert response.status_code == 403 + + +def test_visible_run_anchor_still_reaches_the_service(tmp_path): + """A visible anchor must not be blocked by the new reader check.""" + client, _host = _client(tmp_path) + setattr(client.app.state, RUN_EVIDENCE_READER_RESOLVER_KEY, lambda request: _VisibleRunReader()) + + response = _propose(client) + + assert response.status_code == 200 diff --git a/frontend/src/core/extensions/bundled-documentation.json b/frontend/src/core/extensions/bundled-documentation.json index 9fecf2781d..27dd921e0a 100644 --- a/frontend/src/core/extensions/bundled-documentation.json +++ b/frontend/src/core/extensions/bundled-documentation.json @@ -583,6 +583,57 @@ }, "docs_url": "https://github.com/bytedance/deer-flow/tree/main/examples/deerflow-extension-retrieval#readme" }, + "deerflow-extension-skill-evolution": { + "schema_version": 1, + "id": "catalog/skillEvolution", + "locales": { + "zh-CN": { + "title": "证据驱动的技能进化", + "description": "在回归用例与宿主校验通过后才提议技能修改,对比新旧结果,并可回滚到已确认的版本。", + "usage": [ + { + "title": "安装并授权", + "description": "通过 make extension-install 安装示例,在 config.yaml 中配置 skill mutation 与 evidence 授权后重启 Gateway。" + }, + { + "title": "记录回归用例", + "description": "在包的 cases 目录添加 JSON 用例,用 file_exists、text_contains、text_not_contains 等静态检查描述这次修改要修复什么。" + }, + { + "title": "提出、评估、应用与回滚", + "description": "调用 /api/extension-skill-evolution/proposals 提交候选版本,查看 justified、unjustified 或 failed 结论;通过后 apply,必要时 revert 到已确认版本。" + } + ], + "requirements": [ + "需要支持 skill_mutations 的持久化数据库、run_events.backend: db,以及单实例 POSIX 本地磁盘。", + "需要在 config.yaml 中显式授予 skill mutation 与 evidence 访问权限。" + ] + }, + "en-US": { + "title": "Evidence-gated skill evolution", + "description": "Propose skill edits only after a regression case and host checks pass, compare old and new results, and roll back to a known-good revision.", + "usage": [ + { + "title": "Install and grant access", + "description": "Install the example with make extension-install, configure the skill mutation and evidence grants in config.yaml, then restart Gateway." + }, + { + "title": "Record a regression case", + "description": "Add a JSON case under the package's cases directory using static checks such as file_exists, text_contains and text_not_contains to state what the change fixes." + }, + { + "title": "Propose, evaluate, apply and roll back", + "description": "Submit a candidate to /api/extension-skill-evolution/proposals, inspect the justified, unjustified or failed verdict, apply an accepted revision, and revert when needed." + } + ], + "requirements": [ + "Requires a durable database backend, run_events.backend: db, and a single POSIX host with local disk.", + "Requires explicit skill mutation and evidence grants in config.yaml." + ] + } + }, + "docs_url": "https://github.com/bytedance/deer-flow/tree/main/examples/deerflow-extension-skill-evolution#readme" + }, "deerflow-extension-weknora": { "schema_version": 1, "id": "catalog/weknora", diff --git a/frontend/src/core/extensions/catalog.ts b/frontend/src/core/extensions/catalog.ts index 0c1896a14c..14c72c2847 100644 --- a/frontend/src/core/extensions/catalog.ts +++ b/frontend/src/core/extensions/catalog.ts @@ -78,6 +78,11 @@ export const extensionCatalog = [ package: "deerflow-extension-knowledge-center", namespace: "knowledge-center", }, + { + key: "skillEvolution", + package: "deerflow-extension-skill-evolution", + namespace: null, + }, { key: "lightrag", package: "deerflow-extension-lightrag", namespace: null }, { key: "weknora", package: "deerflow-extension-weknora", namespace: null }, { key: "example", package: "deerflow-extension-example", namespace: null }, diff --git a/frontend/src/core/i18n/locales/en-US.ts b/frontend/src/core/i18n/locales/en-US.ts index c8d9146caa..7b98ca62e0 100644 --- a/frontend/src/core/i18n/locales/en-US.ts +++ b/frontend/src/core/i18n/locales/en-US.ts @@ -200,6 +200,10 @@ export const enUS: Translations = { "deerflow-extension-knowledge-center", "en-US", ), + skillEvolution: bundledCopy( + "deerflow-extension-skill-evolution", + "en-US", + ), lightrag: bundledCopy("deerflow-extension-lightrag", "en-US"), weknora: bundledCopy("deerflow-extension-weknora", "en-US"), example: bundledCopy("deerflow-extension-example", "en-US"), diff --git a/frontend/src/core/i18n/locales/types.ts b/frontend/src/core/i18n/locales/types.ts index 05e382973a..62abce084b 100644 --- a/frontend/src/core/i18n/locales/types.ts +++ b/frontend/src/core/i18n/locales/types.ts @@ -153,6 +153,7 @@ export interface Translations { | "knowledgeManagement" | "knowledgePlatform" | "knowledgeCenter" + | "skillEvolution" | "weknora" | "lightrag" | "example", diff --git a/frontend/src/core/i18n/locales/zh-CN.ts b/frontend/src/core/i18n/locales/zh-CN.ts index eef406641e..325bd7d9db 100644 --- a/frontend/src/core/i18n/locales/zh-CN.ts +++ b/frontend/src/core/i18n/locales/zh-CN.ts @@ -188,6 +188,10 @@ export const zhCN: Translations = { "deerflow-extension-knowledge-center", "zh-CN", ), + skillEvolution: bundledCopy( + "deerflow-extension-skill-evolution", + "zh-CN", + ), lightrag: bundledCopy("deerflow-extension-lightrag", "zh-CN"), weknora: bundledCopy("deerflow-extension-weknora", "zh-CN"), example: bundledCopy("deerflow-extension-example", "zh-CN"),