From d1d0958beab733dee8d9d73df72418e7c66b7477 Mon Sep 17 00:00:00 2001 From: Ryker_Feng <90562015+RykerFeng@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:17:02 +0800 Subject: [PATCH] feat(skills): add guarded evolution host APIs (#5645) * feat(skills): add guarded evolution host APIs * docs(agents): avoid growing saturated guidance chains * fix(skills): address mutation host review findings * fix(skills): order name fence before projection lock * fix(skills): preserve composite toggle lock order * fix(skills): address catalog and pagination review feedback * docs: keep subagent guidance within merged instruction budget --------- Co-authored-by: Willem Jiang --- README.md | 20 + backend/app/gateway/app.py | 2 + backend/app/gateway/deps.py | 9 + backend/app/gateway/routers/AGENTS.md | 2 + .../app/gateway/routers/skill_mutations.py | 52 + backend/app/gateway/routers/skills.py | 24 +- backend/app/gateway/services.py | 13 + .../deerflow_extension_api/__init__.py | 21 +- .../completed_run_evidence.py | 90 ++ .../deerflow_extension_api/contracts.py | 4 + .../host_capabilities.py | 12 + .../deerflow_extension_api/skill_mutations.py | 161 ++++ .../deerflow/agents/lead_agent/prompt.py | 8 + .../skill_activation_middleware.py | 7 + backend/packages/harness/deerflow/client.py | 7 +- .../harness/deerflow/extensions/AGENTS.md | 6 + .../extensions/completed_run_evidence.py | 322 +++++++ .../harness/deerflow/extensions/gateway.py | 13 +- .../deerflow/extensions/host_access.py | 91 ++ .../deerflow/extensions/host_capabilities.py | 151 +++ .../harness/deerflow/extensions/loader.py | 16 +- .../deerflow/persistence/migrations/AGENTS.md | 15 +- .../versions/0045_completed_run_evidence.py | 48 + .../versions/0046_skill_mutations.py | 161 ++++ .../deerflow/persistence/models/__init__.py | 5 + .../deerflow/persistence/models/run_event.py | 3 + .../harness/deerflow/persistence/run/model.py | 26 +- .../harness/deerflow/persistence/run/sql.py | 113 ++- .../persistence/skill_mutations/__init__.py | 1 + .../persistence/skill_mutations/model.py | 89 ++ .../harness/deerflow/runtime/AGENTS.md | 17 + .../deerflow/runtime/events/evidence.py | 13 + .../deerflow/runtime/events/store/db.py | 51 +- .../harness/deerflow/runtime/journal.py | 80 +- .../harness/deerflow/runtime/runs/manager.py | 78 +- .../harness/deerflow/runtime/runs/schemas.py | 5 +- .../deerflow/runtime/runs/store/base.py | 4 + .../deerflow/runtime/runs/store/memory.py | 8 + .../harness/deerflow/runtime/runs/worker.py | 32 + .../harness/deerflow/skills/AGENTS.md | 25 + .../harness/deerflow/skills/export.py | 4 +- .../deerflow/skills/mutations/__init__.py | 1 + .../deerflow/skills/mutations/assets.py | 148 +++ .../deerflow/skills/mutations/codec.py | 89 ++ .../deerflow/skills/mutations/enrollment.py | 65 ++ .../deerflow/skills/mutations/guard.py | 186 ++++ .../deerflow/skills/mutations/history.py | 103 ++ .../deerflow/skills/mutations/publication.py | 174 ++++ .../deerflow/skills/mutations/recovery.py | 170 ++++ .../deerflow/skills/mutations/repository.py | 127 +++ .../deerflow/skills/mutations/scan_budget.py | 38 + .../deerflow/skills/mutations/scanner.py | 111 +++ .../deerflow/skills/mutations/service.py | 608 ++++++++++++ .../deerflow/skills/mutations/topology.py | 81 ++ .../deerflow/skills/mutations/validation.py | 73 ++ .../deerflow/skills/mutations/workers.py | 67 ++ .../harness/deerflow/skills/projection.py | 155 +-- .../deerflow/skills/security_scanner.py | 17 +- .../deerflow/skills/storage/__init__.py | 10 +- .../skills/storage/local_skill_storage.py | 20 +- .../storage/user_scoped_skill_storage.py | 59 +- .../harness/deerflow/subagents/AGENTS.md | 3 +- .../deerflow/tools/skill_manage_tool.py | 20 +- .../blocking_io/test_skill_mutation_host.py | 122 +++ .../fixtures/skill_evolution_consumer.py | 25 + .../fixtures/skill_mutation_crash_worker.py | 80 ++ backend/tests/test_client.py | 46 +- .../test_completed_evidence_admission.py | 156 +++ backend/tests/test_completed_run_evidence.py | 895 ++++++++++++++++++ .../test_db_event_store_lock_lifecycle.py | 13 +- backend/tests/test_extension_host_access.py | 166 ++++ ...est_gateway_extension_service_lifecycle.py | 40 + backend/tests/test_gateway_run_recovery.py | 3 + backend/tests/test_gateway_services.py | 10 +- backend/tests/test_mcp_lifecycle_writers.py | 2 +- ...st_migration_0026_mcp_task_lease_tokens.py | 2 +- ...migration_0027_0028_notification_outbox.py | 2 +- ...st_migration_0029_scheduler_agent_tasks.py | 8 +- ...migration_0032_activity_and_task_events.py | 4 +- ..._migration_0034_run_event_seq_watermark.py | 4 +- .../test_migration_0035_login_throttle.py | 8 +- ...gration_0037_project_document_summaries.py | 9 +- ..._migration_0038_channel_thread_bindings.py | 6 +- ...t_migration_0040_scheduled_group_notice.py | 6 +- .../test_migration_0042_channel_leases.py | 8 +- ..._migration_0044_channel_runtime_configs.py | 2 +- .../test_migration_0046_skill_mutations.py | 56 ++ backend/tests/test_run_event_store.py | 19 +- backend/tests/test_run_worker_rollback.py | 14 +- backend/tests/test_skill_asset_versions.py | 401 ++++++++ .../test_skill_evolution_host_integration.py | 83 ++ backend/tests/test_skill_export.py | 2 +- backend/tests/test_skill_mutation_admin.py | 144 +++ .../tests/test_skill_mutation_async_reads.py | 20 + backend/tests/test_skill_mutation_guard.py | 66 ++ .../test_skill_mutation_host_lifecycle.py | 312 ++++++ .../tests/test_skill_mutation_package_cost.py | 24 + backend/tests/test_skill_mutation_postgres.py | 69 ++ .../test_skill_mutation_process_crash.py | 50 + backend/tests/test_skill_mutation_scanner.py | 72 ++ backend/tests/test_skill_mutation_topology.py | 42 + .../tests/test_skill_mutation_validation.py | 117 +++ backend/tests/test_skill_mutation_workers.py | 83 ++ backend/tests/test_skill_mutations.py | 734 ++++++++++++++ backend/tests/test_skill_sdk_enrollment.py | 151 +++ .../tests/test_user_scoped_skill_storage.py | 37 + config.example.yaml | 13 + docs/skill-evolution-host.md | 224 +++++ 108 files changed, 8269 insertions(+), 215 deletions(-) create mode 100644 backend/app/gateway/routers/skill_mutations.py create mode 100644 backend/packages/extension-api/deerflow_extension_api/completed_run_evidence.py create mode 100644 backend/packages/extension-api/deerflow_extension_api/host_capabilities.py create mode 100644 backend/packages/extension-api/deerflow_extension_api/skill_mutations.py create mode 100644 backend/packages/harness/deerflow/extensions/completed_run_evidence.py create mode 100644 backend/packages/harness/deerflow/extensions/host_access.py create mode 100644 backend/packages/harness/deerflow/extensions/host_capabilities.py create mode 100644 backend/packages/harness/deerflow/persistence/migrations/versions/0045_completed_run_evidence.py create mode 100644 backend/packages/harness/deerflow/persistence/migrations/versions/0046_skill_mutations.py create mode 100644 backend/packages/harness/deerflow/persistence/skill_mutations/__init__.py create mode 100644 backend/packages/harness/deerflow/persistence/skill_mutations/model.py create mode 100644 backend/packages/harness/deerflow/runtime/events/evidence.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/__init__.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/assets.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/codec.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/enrollment.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/guard.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/history.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/publication.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/recovery.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/repository.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/scan_budget.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/scanner.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/service.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/topology.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/validation.py create mode 100644 backend/packages/harness/deerflow/skills/mutations/workers.py create mode 100644 backend/tests/blocking_io/test_skill_mutation_host.py create mode 100644 backend/tests/fixtures/skill_evolution_consumer.py create mode 100644 backend/tests/fixtures/skill_mutation_crash_worker.py create mode 100644 backend/tests/test_completed_evidence_admission.py create mode 100644 backend/tests/test_completed_run_evidence.py create mode 100644 backend/tests/test_extension_host_access.py create mode 100644 backend/tests/test_migration_0046_skill_mutations.py create mode 100644 backend/tests/test_skill_asset_versions.py create mode 100644 backend/tests/test_skill_evolution_host_integration.py create mode 100644 backend/tests/test_skill_mutation_admin.py create mode 100644 backend/tests/test_skill_mutation_async_reads.py create mode 100644 backend/tests/test_skill_mutation_guard.py create mode 100644 backend/tests/test_skill_mutation_host_lifecycle.py create mode 100644 backend/tests/test_skill_mutation_package_cost.py create mode 100644 backend/tests/test_skill_mutation_postgres.py create mode 100644 backend/tests/test_skill_mutation_process_crash.py create mode 100644 backend/tests/test_skill_mutation_scanner.py create mode 100644 backend/tests/test_skill_mutation_topology.py create mode 100644 backend/tests/test_skill_mutation_validation.py create mode 100644 backend/tests/test_skill_mutation_workers.py create mode 100644 backend/tests/test_skill_mutations.py create mode 100644 backend/tests/test_skill_sdk_enrollment.py create mode 100644 docs/skill-evolution-host.md diff --git a/README.md b/README.md index 2d3e6fc183..2c0c1cbb61 100644 --- a/README.md +++ b/README.md @@ -1915,6 +1915,24 @@ classifies the sanitized text a remote tool result shows the model, PII-redacted `pii_redaction` is enabled, and adds an advisory warning through a lifecycle state update; it imports only the extension contract. +Extensions may opt into owner-scoped completed-run evidence and safe updates to +existing user-custom skills through `host_access` in operator-controlled +`config.yaml`. These optional APIs stage an immutable candidate, check its exact +contents, and publish with version conflict detection and crash recovery. They +do **not** implement a learning/evolution plugin. Automatic changes are limited to +the body and description of `SKILL.md`; changing permissions, supporting files, +or enabled state is out of scope. The host contract requires `deerflow-extension-api==0.2.7`. See [Skill evolution host API](docs/skill-evolution-host.md) +for deployment requirements, grants, recovery, and the plugin integration flow. +Once an owner is enrolled, use the Gateway for skill management: standalone SDK +processes without its durable mutation runtime refuse enrolled skill access, +including custom-skill YAML diagnostics. Gateway diagnostics recover pending +publications before scanning. +Skill discovery preserves user scope when the caller uses the default configuration. +Admin operation lists return `next_cursor: null` at the end of pagination. +Keyed resume retries preserve the original run's evidence origin and agent identity. +This host slice follows up the [Skill Self-Evolution RFC](https://github.com/bytedance/deer-flow/issues/1865) +and the bilingual [Plugin Host APIs RFC](https://github.com/bytedance/deer-flow/issues/5539). + Full-stack contributions can additionally provide browser pages, conversation actions, authenticated backend operations and model tools through the [plugin APIs](docs/full-stack-plugins.md). The independent @@ -2371,6 +2389,8 @@ async with runtime: Gateway run finalization also retains ownership through caller cancellation: it finishes terminal persistence, stream completion, and resource cleanup before propagating the interruption. Backend I/O must terminate or enforce its own timeout. Gateway shutdown includes workers still finalizing after their terminal status commits; an expired shutdown drain budget is logged and does not cancel their outstanding writes. +Terminal cleanup releases the run journal and its message-sequence callback even when diagnostic logs retain a checkpoint-inspection traceback. + The factory still does not load YAML or create SQL infrastructure: the caller supplies the config snapshot, repository, and lifecycle. Because it accepts a caller-owned `system_prompt`, direct integrations also own any model-visible wording about those limits; the default middleware enforces the runtime limits regardless. The factory does not mount the Gateway owner-scoped HTTP routes or Web UI, so direct applications must expose their own result API/UI if they need those surfaces. For ordinary delegation only, `SubagentRuntime(...)` needs no asynchronous startup. Administrators can add, edit, disable, and delete reusable worker definitions from **Settings → Subagents**. Built-in and `config.yaml` definitions remain visible there as read-only entries. The default Lead Agent can use every enabled runtime sub-agent; each page-created Custom Agent can instead allow all, none, or a selected set. That selection is enforced both in the model-visible directory and by the server-side `task` tool. Managed definitions are deployment-wide in this version and follow `agent_storage.backend`: atomic files for a local deployment or the shared application database for multiple instances. diff --git a/backend/app/gateway/app.py b/backend/app/gateway/app.py index 1787cbe7af..efa9a67b4a 100644 --- a/backend/app/gateway/app.py +++ b/backend/app/gateway/app.py @@ -49,6 +49,7 @@ from app.gateway.routers import ( projects, runs, scheduled_tasks, + skill_mutations, skills, subagent_batches, subagents, @@ -1330,6 +1331,7 @@ This gateway provides runtime endpoints for agent runs plus custom endpoints for # Skills API is mounted at /api/skills app.include_router(skills.router) + app.include_router(skill_mutations.router) # First-party integrations API is mounted at /api/integrations app.include_router(integrations.router) diff --git a/backend/app/gateway/deps.py b/backend/app/gateway/deps.py index 230bb67838..a00b6ac6e3 100644 --- a/backend/app/gateway/deps.py +++ b/backend/app/gateway/deps.py @@ -854,6 +854,14 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen from deerflow.extensions.gateway import start_services, stop_services extensions = getattr(app.state, "extensions", EMPTY_EXTENSIONS) + from deerflow.extensions.host_capabilities import HostCapabilities + + host_capabilities = HostCapabilities(config, app.state.run_store, app.state.run_event_store) + app.state.skill_mutation_host = host_capabilities + # Own partial startup and cancellation before acquiring any locks or + # worker resources. LIFO shutdown stops plugins before host recovery. + stack.push_async_callback(host_capabilities.close) + await host_capabilities.start() attempted_services: list[tuple[str, Any]] = [] async def stop_extension_services() -> None: @@ -875,6 +883,7 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen config, sf, run_evidence_reader=app.state.run_evidence_reader, + host_capabilities=host_capabilities.bindings, attempted_services=attempted_services, ) ) diff --git a/backend/app/gateway/routers/AGENTS.md b/backend/app/gateway/routers/AGENTS.md index 7f7c191999..fbe21724e8 100644 --- a/backend/app/gateway/routers/AGENTS.md +++ b/backend/app/gateway/routers/AGENTS.md @@ -16,6 +16,8 @@ create-only: collisions return 409, never overwrite or upsert. Ordinary creation and import share `_persist_new_agent`, which uses `run_drained_write` so a cancelled request still drains the owned persistence worker. +Skill mutation operation lists return `next_cursor: null` when `has_more` is false, including empty pages after a supplied `after_id`. The next cursor is the final returned operation only when a further page exists. + `GET /api/agent-tool-groups` projects only configured group names for the custom-Agent settings allowlist. Never expose tool provider paths, credentials, or raw tool configuration through this catalog. diff --git a/backend/app/gateway/routers/skill_mutations.py b/backend/app/gateway/routers/skill_mutations.py new file mode 100644 index 0000000000..6a2242de91 --- /dev/null +++ b/backend/app/gateway/routers/skill_mutations.py @@ -0,0 +1,52 @@ +"""Host-admin inspection and reconciliation, with no force-overwrite endpoint.""" + +from dataclasses import asdict +from typing import Annotated + +from deerflow_extension_api import HostCapabilityError +from fastapi import APIRouter, HTTPException, Path, Query, Request + +from app.gateway.deps import require_admin_user + +router = APIRouter(prefix="/api/skill-mutations", tags=["skill-mutations"]) +OperationId = Annotated[str, Path(pattern=r"^[a-f0-9]{32}$")] +OwnerId = Annotated[str, Path(min_length=1, max_length=128, pattern=r"^[A-Za-z0-9_-]+$")] + + +async def _invoke(request, method, *args, **kwargs): + await require_admin_user(request, detail="Admin privileges required to inspect or recover skill publications.") + host = getattr(request.app.state, "skill_mutation_host", None) + if host is None or host.recovery is None: + raise HTTPException(status_code=503, detail="UNAVAILABLE") + try: + return await host.workers.run(lambda _check: getattr(host.recovery, method)(*args, **kwargs)) + except HostCapabilityError as exc: + status = {"NOT_FOUND_OR_FORBIDDEN": 404, "NOT_FOUND": 404, "NEEDS_REPAIR": 409, "REVISION_CONFLICT": 409}.get(exc.code, 503) + raise HTTPException(status_code=status, detail=exc.code) from None + except Exception: + # SQL parameters may contain package bytes. No raw exception response. + raise HTTPException(status_code=503, detail="UNAVAILABLE") from None + + +@router.get("/operations") +async def list_operations(request: Request, limit: Annotated[int, Query(ge=1, le=100)] = 50, after_id: Annotated[str | None, Query(pattern=r"^[a-f0-9]{32}$")] = None): + rows = await _invoke(request, "list_operations", limit=limit + 1, after_id=after_id) + items = rows[:limit] + has_more = len(rows) > limit + return {"items": [asdict(row) for row in items], "has_more": has_more, "next_cursor": items[-1].operation_id if has_more else None} + + +@router.get("/operations/{operation_id}") +async def get_operation(request: Request, operation_id: OperationId): + return asdict(await _invoke(request, "get_operation", operation_id)) + + +@router.post("/operations/{operation_id}/recover") +async def recover_operation(request: Request, operation_id: OperationId): + return asdict(await _invoke(request, "recover_operation", operation_id)) + + +@router.post("/owners/{owner_id}/recover") +async def recover_owner(request: Request, owner_id: OwnerId): + await _invoke(request, "recover_owner", owner_id) + return {"owner_id": owner_id, "reconciled": True} diff --git a/backend/app/gateway/routers/skills.py b/backend/app/gateway/routers/skills.py index d44a0b5418..a0cfaf1b55 100644 --- a/backend/app/gateway/routers/skills.py +++ b/backend/app/gateway/routers/skills.py @@ -910,12 +910,12 @@ async def rollback_custom_skill(skill_name: str, body: SkillRollbackRequest, req await asyncio.to_thread(storage.validate_skill_markdown_content, skill_name, target_content) static_findings = await _scan_static_skill_markdown_or_raise(skill_name, target_content, app_config=config) scan = await scan_skill_content(target_content, executable=False, location=f"{skill_name}/{SKILL_MD_FILE}", app_config=config, static_findings=static_findings) + from deerflow.skills.mutations.guard import read_optional_text def _read_current_content() -> str | None: - # Worker thread: the post-scan read of the file being replaced is - # blocking filesystem IO (#5747), same rule as the history read. - skill_file = storage.get_custom_skill_file(skill_name) - return skill_file.read_text(encoding="utf-8") if skill_file.exists() else None + # Called off-loop, inside the mutation critical section for writes. + # Preserve managed-owner readiness and projection-lock admission. + return read_optional_text(storage, storage.get_custom_skill_file(skill_name)) if scan.decision == "block": # The blocked path overwrites nothing, so the entry is purely @@ -1021,12 +1021,12 @@ def _write_extensions_skill_state( """Read-modify-write a skill's enabled state in the shared extensions_config.json. Blocking filesystem IO: always call this via ``asyncio.to_thread``. It takes - the public projection lock before the process-local and cross-process - extensions config locks. The first keeps the enabled-only view synchronized - across workers; the latter two prevent this router and the MCP router from - interleaving writes to the shared file. All locks are held by the worker, so - request cancellation cannot release them while the write or projection - rebuild is still running. + the same-name mutation fence before the public projection, process-local, + and cross-process extensions config locks. The projection lock keeps the + enabled-only view synchronized across workers; the latter two prevent this + router and the MCP router from interleaving writes to the shared file. All + locks are held by the worker, so request cancellation cannot release them + while the write or projection rebuild is still running. """ from contextlib import nullcontext @@ -1040,7 +1040,9 @@ def _write_extensions_skill_state( config_path = Path.cwd().parent / "extensions_config.json" logger.info(f"No existing extensions config found. Creating new config at: {config_path}") - with projection_update: + from deerflow.skills.mutations.guard import managed_global_state_write + + with managed_global_state_write(storage, skill_name), projection_update: with extensions_config_write_lock, extensions_config_file_lock(config_path): # The projection lock is cross-process, but the singleton cache is # not. Existing files are therefore re-read under the lock, raw, so diff --git a/backend/app/gateway/services.py b/backend/app/gateway/services.py index 2e5dedbc58..ed45476def 100644 --- a/backend/app/gateway/services.py +++ b/backend/app/gateway/services.py @@ -93,6 +93,7 @@ from deerflow.runtime.journal import build_checkpoint_history_seed_events from deerflow.runtime.keyed_lock import KeyedLockTable from deerflow.runtime.run_origin import DEERFLOW_ORIGIN_KEY, make_origin from deerflow.runtime.runs.naming import resolve_root_run_name +from deerflow.runtime.runs.schemas import EVIDENCE_ORIGINS from deerflow.runtime.secret_context import ( LegacyRunMetadataSecretError, redact_config_secrets, @@ -1862,6 +1863,7 @@ async def start_run( *, idempotency_key: str | None = None, require_existing_thread: bool = False, + evidence_origin: str | None = None, scheduled_task_runtime: Mapping[str, Any] | None = None, ) -> RunRecord: """Create a RunRecord and launch the background agent task. @@ -1877,6 +1879,9 @@ async def start_run( require_existing_thread : bool Reject a missing thread instead of auto-creating metadata. Internal notification runs use this so a deleted chat cannot be resurrected. + evidence_origin : str | None + Trusted Python entry-point provenance, never copied from request data. + Internal launches without an explicit origin remain unknown. """ # Cancel-capability gate. interrupt/rollback strategies terminate an already # active run — runs:cancel capability, not runs:create — so a create-only @@ -1976,6 +1981,9 @@ async def start_run( # so import-lock waiters cannot starve unrelated default-executor work. # Keep this before admission so import failures cannot create a run. agent_factory = await run_assembly(resolve_agent_factory, body.assistant_id) + admitted_origin = evidence_origin or ("unknown" if is_internal_caller else "interactive") + if admitted_origin not in EVIDENCE_ORIGINS: + raise ValueError("Unsupported host evidence origin") command = getattr(body, "command", None) if command and command.get("resume") is not None: graph_input = Command(resume=command["resume"]) @@ -2356,6 +2364,10 @@ async def start_run( model_name=model_name, user_id=owner_user_id, idempotency_key=idempotency_key, + evidence_origin=admitted_origin, + # Canonical agent keys, not routing aliases or display names. + # Bootstrap and unresolved legacy identities stay read-only. + evidence_agent_id=(None if scope_runtime_config.get("is_bootstrap") else (_DEFAULT_ASSISTANT_ID if scope_assistant_id == _DEFAULT_ASSISTANT_ID else getattr(agent_config, "name", None))), ) except RunIdempotencyUnsupported as exc: raise HTTPException(status_code=503, detail=str(exc)) from exc @@ -2507,6 +2519,7 @@ async def launch_scheduled_thread_run( thread_id, request, idempotency_key=idempotency_key, + evidence_origin="scheduled", scheduled_task_runtime=scheduled_task_runtime, ) return {"run_id": record.run_id, "thread_id": record.thread_id} diff --git a/backend/packages/extension-api/deerflow_extension_api/__init__.py b/backend/packages/extension-api/deerflow_extension_api/__init__.py index 4ff7481f28..2546d0cba9 100644 --- a/backend/packages/extension-api/deerflow_extension_api/__init__.py +++ b/backend/packages/extension-api/deerflow_extension_api/__init__.py @@ -28,6 +28,7 @@ from deerflow_extension_api.compaction import ( CompactionEvent, ContextCompactionObserver, ) +from deerflow_extension_api.completed_run_evidence import CompletedRunEvent, CompletedRunEventPage, CompletedRunEvidenceReader, CompletedRunSnapshot, EvidenceLimits from deerflow_extension_api.contracts import ( ExtensionInstall, ExtensionRegistry, @@ -44,6 +45,7 @@ from deerflow_extension_api.contracts import ( TaskOutcome, extension, ) +from deerflow_extension_api.host_capabilities import HostCapabilityError from deerflow_extension_api.model_invocation import ( ModelInvocationError, ModelInvocationFailed, @@ -95,6 +97,7 @@ from deerflow_extension_api.runtime_bridge import ( task_store_from_runtime, ) from deerflow_extension_api.settings import SettingsField +from deerflow_extension_api.skill_mutations import AssessmentRef, AssetRevision, BundleFile, HostCheckResult, MutationCapabilities, Operation, Proposal, ProposalBundle, SkillMutationService, SkillRevisionView from deerflow_extension_api.state import ExtensionData #: Contract version. Before 1.0, minors may break and patches are additive. @@ -126,8 +129,24 @@ __all__ = [ "ModelTool", "PluginToolResult", "PluginContribution", - "ToolContext", "SettingsField", + "ToolContext", + "CompletedRunEvent", + "CompletedRunEventPage", + "CompletedRunEvidenceReader", + "CompletedRunSnapshot", + "EvidenceLimits", + "HostCapabilityError", + "AssessmentRef", + "AssetRevision", + "BundleFile", + "HostCheckResult", + "MutationCapabilities", + "Operation", + "Proposal", + "ProposalBundle", + "SkillMutationService", + "SkillRevisionView", "API_VERSION", "EXTENSION_PLUGIN_AUTHZ_RESOLVER_ASYNC_KEY", "EXTENSION_PLUGIN_AUTHZ_RESOLVER_KEY", diff --git a/backend/packages/extension-api/deerflow_extension_api/completed_run_evidence.py b/backend/packages/extension-api/deerflow_extension_api/completed_run_evidence.py new file mode 100644 index 0000000000..9a34a50a35 --- /dev/null +++ b/backend/packages/extension-api/deerflow_extension_api/completed_run_evidence.py @@ -0,0 +1,90 @@ +"""Immutable, bounded evidence projections. References never confer authorization.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +from deerflow_extension_api.run_evidence import RunPage + + +@dataclass(frozen=True) +class EvidenceLimits: + max_events: int = 200 + max_bytes: int = 256 * 1024 + + def __post_init__(self) -> None: + if type(self.max_events) is not int or not 1 <= self.max_events <= 2000: + raise ValueError("max_events must be between 1 and 2000") + if type(self.max_bytes) is not int or not 1024 <= self.max_bytes <= 1024 * 1024: + raise ValueError("max_bytes must be between 1024 and 1048576, including the envelope") + + +@dataclass(frozen=True) +class CompletedRunSnapshot: + snapshot_ref: str = "" + evidence_revision: str = "" + retention_revision: int = 0 + thread_id: str = "" + run_id: str = "" + owner_id: str = "" + agent_id: str | None = None + origin: str = "unknown" + status: str = "" + stop_reason: str | None = None + seal_state: str = "partial" + seal_error: str | None = None + upper_event_seq: int = 0 + event_count: int = 0 + coverage: str = "lead-journal-v1" + coverage_limits: tuple[str, ...] = ( + "No complete background subagent, remote MCP, or external side-effect coverage.", + "Prior conversation and attachment bodies are excluded.", + "Skill loading and actual content hashes are unknown unless recorded.", + ) + expires_at: str | None = None + skill_observations: tuple[str, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "coverage_limits", tuple(self.coverage_limits)) + object.__setattr__(self, "skill_observations", tuple(self.skill_observations)) + + +@dataclass(frozen=True) +class CompletedRunEvent: + """JSON strings keep nested event data immutable and detached from the host. + + ``content_sha256`` hashes the persisted UTF-8 content, before any preview. + ``content_bytes`` is its persisted byte count, not the JSON envelope size. + """ + + event_id: str = "" + seq: int = 0 + event_type: str = "" + category: str = "" + content_json: str = "null" + metadata_json: str = "{}" + content_bytes: int = 0 + content_sha256: str = "" + truncated: bool = False + + +@dataclass(frozen=True) +class CompletedRunEventPage: + items: tuple[CompletedRunEvent, ...] = () + next_cursor: str | None = None + has_more: bool = False + + def __post_init__(self) -> None: + object.__setattr__(self, "items", tuple(self.items)) + + +class CompletedRunEvidenceReader(Protocol): + async def list_changed_runs(self, *, cursor: str | None = None, limit: int = 200) -> RunPage: + raise NotImplementedError + + async def get_snapshot(self, *, thread_id: str, run_id: str, limits: EvidenceLimits = EvidenceLimits()) -> CompletedRunSnapshot: + raise NotImplementedError + + async def read_events(self, *, snapshot_ref: str, cursor: str | None = None, limits: EvidenceLimits = EvidenceLimits()) -> CompletedRunEventPage: + raise NotImplementedError diff --git a/backend/packages/extension-api/deerflow_extension_api/contracts.py b/backend/packages/extension-api/deerflow_extension_api/contracts.py index 3ef0e849d2..47090864b7 100644 --- a/backend/packages/extension-api/deerflow_extension_api/contracts.py +++ b/backend/packages/extension-api/deerflow_extension_api/contracts.py @@ -20,9 +20,11 @@ from deerflow_extension_api.state import ExtensionData if TYPE_CHECKING: # pragma: no cover - typing only from deerflow_extension_api.assembly import AgentAssemblyObserver from deerflow_extension_api.compaction import ContextCompactionObserver + from deerflow_extension_api.completed_run_evidence import CompletedRunEvidenceReader from deerflow_extension_api.model_invocation import ModelInvoker from deerflow_extension_api.placement import AgentBuildContext, MiddlewarePlacement from deerflow_extension_api.run_evidence import RunEvidenceReader + from deerflow_extension_api.skill_mutations import SkillMutationService F = TypeVar("F", bound=Callable[..., Any]) @@ -169,6 +171,8 @@ class ExtensionRuntimeDeps: session_factory: Any | None = None run_evidence_reader: RunEvidenceReader | None = None model_invoker: ModelInvoker | None = None + completed_run_evidence: CompletedRunEvidenceReader | None = None + skill_mutations: SkillMutationService | None = None class ExtensionService(Protocol): diff --git a/backend/packages/extension-api/deerflow_extension_api/host_capabilities.py b/backend/packages/extension-api/deerflow_extension_api/host_capabilities.py new file mode 100644 index 0000000000..6229eb3c3d --- /dev/null +++ b/backend/packages/extension-api/deerflow_extension_api/host_capabilities.py @@ -0,0 +1,12 @@ +"""Dependency-free errors shared by optional host capabilities.""" + +from __future__ import annotations + + +class HostCapabilityError(RuntimeError): + """A stable machine-readable failure, optionally carrying retry guidance.""" + + def __init__(self, code: str, message: str = "", *, retry_after: float | None = None) -> None: + self.code = code + self.retry_after = retry_after + super().__init__(f"{code}: {message}" if message else code) diff --git a/backend/packages/extension-api/deerflow_extension_api/skill_mutations.py b/backend/packages/extension-api/deerflow_extension_api/skill_mutations.py new file mode 100644 index 0000000000..e332008128 --- /dev/null +++ b/backend/packages/extension-api/deerflow_extension_api/skill_mutations.py @@ -0,0 +1,161 @@ +"""Host-owned, conditional updates of existing custom skills; no learning policy.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + + +@dataclass(frozen=True) +class AssetRevision: + incarnation_id: str + mutation_seq: int + content_digest: str + + +@dataclass(frozen=True) +class SkillRevisionView: + target_ref: str + owner_id: str + name: str + revision: AssetRevision + content: str + enabled: bool + + +@dataclass(frozen=True) +class MutationCapabilities: + operations: tuple[str, ...] + protocol_version: str = "skill-mutations-v1" + max_main_bytes: int = 131072 + max_package_bytes: int = 16777216 + max_package_files: int = 256 + max_sources: int = 20 + max_pending_proposals: int = 20 + max_model_scan_files: int = 32 + max_model_scan_bytes: int = 1048576 + max_model_scan_file_bytes: int = 131072 + max_concurrent_scans: int = 4 + max_owner_plugin_concurrent_scans: int = 2 + max_owner_plugin_scans_per_hour: int = 20 + check_timeout_seconds: int = 120 + proposal_ttl_seconds: int = 86400 + check_ttl_seconds: int = 900 + rollback_ttl_seconds: int = 2592000 + topology: str = "posix-single-host-local-filesystem" + + def __post_init__(self): + object.__setattr__(self, "operations", tuple(self.operations)) + + +@dataclass(frozen=True) +class Proposal: + proposal_id: str + owner_id: str + target_ref: str + name: str + base_revision: AssetRevision + candidate_hash: str + state: str + expires_at: str + source_refs: tuple[str, ...] + operation_id: str | None = None + + def __post_init__(self): + object.__setattr__(self, "source_refs", tuple(self.source_refs)) + + +@dataclass(frozen=True) +class BundleFile: + path: str + content: bytes + sha256: str + executable: bool + + def __post_init__(self): + object.__setattr__(self, "content", bytes(self.content)) + + +@dataclass(frozen=True) +class ProposalBundle: + proposal_id: str + baseline: tuple[BundleFile, ...] + candidate: tuple[BundleFile, ...] + base_revision: AssetRevision + candidate_hash: str + + def __post_init__(self): + object.__setattr__(self, "baseline", tuple(self.baseline)) + object.__setattr__(self, "candidate", tuple(self.candidate)) + + +@dataclass(frozen=True) +class HostCheckResult: + proposal_id: str + decision: str + candidate_hash: str + base_revision: AssetRevision + policy_version: str + expires_at: str + reason_code: str + + +@dataclass(frozen=True) +class AssessmentRef: + candidate_hash: str + base_revision: AssetRevision + evaluator: str + evaluator_version: str + report_id: str + + +@dataclass(frozen=True) +class Operation: + operation_id: str + owner_id: str + name: str + publication: str + views: str + before_revision: AssetRevision + after_revision: AssetRevision + generation: int + proposal_id: str | None = None + reverts_operation_id: str | None = None + error_code: str | None = None + superseded_by_generation: int | None = None + + +class SkillMutationService(Protocol): + @property + def capabilities(self) -> MutationCapabilities: + raise NotImplementedError + + async def read_skill(self, *, source_ref: str, name: str) -> SkillRevisionView: + raise NotImplementedError + + async def stage(self, *, source_refs: tuple[str, ...], target_ref: str, expected_revision: AssetRevision, content: str, idempotency_key: str) -> Proposal: + raise NotImplementedError + + async def get_proposal(self, *, proposal_id: str) -> Proposal: + raise NotImplementedError + + async def read_proposal_bundle(self, *, proposal_id: str, max_bytes: int = 33554432) -> ProposalBundle: + raise NotImplementedError + + async def check(self, *, proposal_id: str) -> HostCheckResult: + raise NotImplementedError + + async def commit(self, *, proposal_id: str, idempotency_key: str, assessment_ref: AssessmentRef | None = None) -> Operation: + raise NotImplementedError + + async def discard(self, *, proposal_id: str) -> Proposal: + raise NotImplementedError + + async def revert(self, *, operation_id: str, expected_current_revision: AssetRevision, idempotency_key: str) -> Operation: + raise NotImplementedError + + async def get_operation(self, *, operation_id: str) -> Operation: + raise NotImplementedError + + async def find_operation(self, *, idempotency_key: str, method: str = "commit") -> Operation | None: + raise NotImplementedError diff --git a/backend/packages/harness/deerflow/agents/lead_agent/prompt.py b/backend/packages/harness/deerflow/agents/lead_agent/prompt.py index 0ad06297c9..0bdccd0a92 100644 --- a/backend/packages/harness/deerflow/agents/lead_agent/prompt.py +++ b/backend/packages/harness/deerflow/agents/lead_agent/prompt.py @@ -282,6 +282,14 @@ def get_enabled_skills_for_config(app_config: AppConfig | None = None, user_id: to load public + user-level custom skills. Otherwise falls back to the global storage (public + global custom fallback). """ + from deerflow.skills.mutations.guard import owner_is_managed + + if user_id is not None and (app_config is None or owner_is_managed(user_id)): + # Without an explicit config, resolve the user storage on each call. + # Enrolled owners also bypass the process-local LRU. The guarded loader + # checks durable readiness, so peer publications cannot leave a stale + # catalog or treat PREPARED bytes as a committed generation. + return list(get_or_new_user_skill_storage(user_id, app_config=app_config).load_skills(enabled_only=True)) if app_config is None: return _get_enabled_skills() diff --git a/backend/packages/harness/deerflow/agents/middlewares/skill_activation_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/skill_activation_middleware.py index a7d4bcfef1..d839a5e2a6 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/skill_activation_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/skill_activation_middleware.py @@ -279,6 +279,13 @@ class SkillActivationMiddleware(AgentMiddleware): @staticmethod def _read_skill_content(skill_file: Path, skills_root: Path, *, storage: SkillStorage | None = None) -> str: + from deerflow.skills.mutations.guard import managed_read + + with managed_read(storage): + return SkillActivationMiddleware._read_skill_content_unlocked(skill_file, skills_root, storage=storage) + + @staticmethod + def _read_skill_content_unlocked(skill_file: Path, skills_root: Path, *, storage: SkillStorage | None = None) -> str: if skill_file.name != SKILL_MD_FILE: raise ValueError(f"Expected {SKILL_MD_FILE}, got {skill_file.name}") # Use the storage's path validation if available — UserScopedSkillStorage diff --git a/backend/packages/harness/deerflow/client.py b/backend/packages/harness/deerflow/client.py index 75f44b9e21..2f570c43d7 100644 --- a/backend/packages/harness/deerflow/client.py +++ b/backend/packages/harness/deerflow/client.py @@ -1574,10 +1574,11 @@ class DeerFlowClient: if config_path is None: raise FileNotFoundError("Cannot locate extensions_config.json. Set DEER_FLOW_EXTENSIONS_CONFIG_PATH or ensure it exists in the project root.") + from deerflow.skills.mutations.guard import managed_global_state_write from deerflow.skills.projection import skill_projection_mutation removal_names = (name,) if not enabled else () - with skill_projection_mutation(storage, "public", remove_names=removal_names): + with managed_global_state_write(storage, name), skill_projection_mutation(storage, "public", remove_names=removal_names): with extensions_config_write_lock, extensions_config_file_lock(config_path): # The projection lock is cross-process, but the singleton # cache is not. Reload raw from disk under the config lock. @@ -1590,10 +1591,12 @@ class DeerFlowClient: storage.set_skill_enabled_state(name, enabled) else: # Fallback for non-user-scoped storage (unlikely in practice) + from deerflow.skills.mutations.guard import managed_global_state_write + config_path = ExtensionsConfig.resolve_config_path() if config_path is None: raise FileNotFoundError("Cannot locate extensions_config.json. Set DEER_FLOW_EXTENSIONS_CONFIG_PATH or ensure it exists in the project root.") - with extensions_config_write_lock, extensions_config_file_lock(config_path): + with managed_global_state_write(storage, name), extensions_config_write_lock, extensions_config_file_lock(config_path): self._write_skill_enabled_state(config_path, name, enabled) # Invalidate the prompt cache for this caller (and for all users if diff --git a/backend/packages/harness/deerflow/extensions/AGENTS.md b/backend/packages/harness/deerflow/extensions/AGENTS.md index 2b586db3c2..d134099015 100644 --- a/backend/packages/harness/deerflow/extensions/AGENTS.md +++ b/backend/packages/harness/deerflow/extensions/AGENTS.md @@ -440,6 +440,12 @@ to delegation. Browser fields are allowlisted. Code is trusted, not sandboxed. within 64 KiB and snapshot artifacts. Keep identity, policy/deadline and citation gates. Test ToolNode and checkpoint replay. See `docs/full-stack-plugins.md`. +Evolution host grants (`host_access.evidence` / `skill_mutations`) compose with +`host_access.model_invocation` through `HostAccess`. Evolution grants require a +unique stable name and entry point; model-only grants retain their per-install +identity and permit repeated entry points. Keep all granted dependencies when +wrapping services with `ModelInvocationService`. The additive evolution contracts +ship in extension API 0.2.7, after 0.2.6's PluginToolResult contract. Full Agent run control is an optional `deerflow_extension_api.AgentRuns` handle on action/tool contexts and the request resolver. Gateway owns principal binding, revocation and ordinary route admission in `app/gateway/extension_agent_runs.py`. diff --git a/backend/packages/harness/deerflow/extensions/completed_run_evidence.py b/backend/packages/harness/deerflow/extensions/completed_run_evidence.py new file mode 100644 index 0000000000..f9a7e15bc5 --- /dev/null +++ b/backend/packages/harness/deerflow/extensions/completed_run_evidence.py @@ -0,0 +1,322 @@ +"""Owner-bound durable run evidence, with fixed boundaries and bounded reads.""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import json +from dataclasses import asdict, replace +from datetime import UTC, datetime +from time import monotonic +from uuid import uuid4 + +from deerflow_extension_api.completed_run_evidence import CompletedRunEvent, CompletedRunEventPage, CompletedRunSnapshot, EvidenceLimits +from deerflow_extension_api.host_capabilities import HostCapabilityError +from deerflow_extension_api.run_evidence import RunPage, RunStatusView +from sqlalchemy import LargeBinary, Text, and_, cast, func, or_, select, text + +from deerflow.persistence.models.run_event import RunEventRow +from deerflow.persistence.run.model import CompletedRunSnapshotRow, RunRow +from deerflow.persistence.run.sql import RunRepository +from deerflow.runtime.events.store.db import DbRunEventStore +from deerflow.runtime.secret_context import redact_metadata_secrets +from deerflow.utils.time import coerce_iso + + +def _json(value) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + +def _size(value) -> int: + return len(_json(asdict(value)).encode("utf-8")) + + +def _snapshot(payload: dict) -> CompletedRunSnapshot: + return CompletedRunSnapshot(**{**payload, "coverage_limits": tuple(payload["coverage_limits"]), "skill_observations": tuple(payload["skill_observations"])}) + + +class HostCompletedRunEvidenceReader: + """Bindings come from trusted installation grants, never call arguments. + + This is a supported capability boundary for trusted in-process plugins, + not a sandbox against code holding a raw database session factory. + + New events carry an ingestion digest. Legacy digest computation has a + cumulative per-page cap of 1 MiB, 64 SQL chunks, and 2 seconds (including + query waits). Above those caps we raise LIMIT_EXCEEDED, never invent a hash + or trust a digest supplied in event metadata. + """ + + def __init__(self, run_store, event_store, *, plugin_id: str, owner_ids) -> None: + if not isinstance(plugin_id, str) or not plugin_id.strip(): + raise ValueError("a stable plugin identity is required") + if not owner_ids or isinstance(owner_ids, (str, bytes)) or any(not isinstance(owner, str) or not owner.strip() or owner == "*" for owner in owner_ids): + raise ValueError("an explicit nonempty owner allowlist is required") + self._owners = frozenset(owner_ids) + self._scope = hashlib.sha256(_json([plugin_id, sorted(self._owners)]).encode()).hexdigest() + self._sf = run_store._sf if isinstance(run_store, RunRepository) and isinstance(event_store, DbRunEventStore) and run_store._sf is event_store._sf else None + + @property + def scope_digest(self) -> str: + """Stable host-internal scope binding used by transactional consumers.""" + return self._scope + + def _supported(self) -> None: + if self._sf is None: + raise HostCapabilityError("UNSUPPORTED", "completed evidence requires durable run and event stores on the same database") + + def _cursor(self, kind: str, position, reference: str = "") -> str: + return base64.urlsafe_b64encode(_json([1, self._scope, kind, reference, position]).encode()).decode().rstrip("=") + + def _position(self, cursor: str | None, kind: str, reference: str = ""): + if cursor is None: + return None + try: + if not isinstance(cursor, str) or len(cursor) > 2048: + raise ValueError + value = json.loads(base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True)) + if not isinstance(value, list) or len(value) != 5 or value[:4] != [1, self._scope, kind, reference]: + raise ValueError + position = value[4] + if kind == "runs": + if not isinstance(position, list) or len(position) != 2 or type(position[0]) is not int or position[0] < 0 or not isinstance(position[1], str): + raise ValueError + elif type(position) is not int or position < 0: + raise ValueError + return position + except (ValueError, TypeError, UnicodeDecodeError) as exc: + raise HostCapabilityError("INVALID_CURSOR", "cursor does not match this plugin, scope, version, or snapshot") from exc + + async def list_changed_runs(self, *, cursor: str | None = None, limit: int = 200) -> RunPage: + self._supported() + limit = EvidenceLimits(max_events=limit).max_events + seq, run_id = self._position(cursor, "runs") or (-1, "") + stmt = ( + select(RunRow.thread_id, RunRow.run_id, RunRow.status, RunRow.created_at, RunRow.updated_at, RunRow.stop_reason, RunRow.change_seq) + .where( + RunRow.user_id.in_(self._owners), + RunRow.operation_kind == "run", + or_(RunRow.change_seq > seq, and_(RunRow.change_seq == seq, RunRow.run_id > run_id)), + ) + .order_by(RunRow.change_seq, RunRow.run_id) + .limit(limit + 1) + ) + async with self._sf() as session: + rows = (await session.execute(stmt)).all() + items = [] + next_cursor = cursor + for row in rows[:limit]: + item = RunStatusView(thread_id=row.thread_id, run_id=row.run_id, status=row.status, created_at=coerce_iso(row.created_at), updated_at=coerce_iso(row.updated_at), stop_reason=row.stop_reason) + candidate_cursor = self._cursor("runs", [row.change_seq, row.run_id]) + if _size(RunPage(tuple([*items, item]), candidate_cursor, True)) > 1024 * 1024: + break + items.append(item) + next_cursor = candidate_cursor + return RunPage(tuple(items), next_cursor, len(rows) > len(items)) + + async def _begin_read(self, session) -> None: + # sqlite's legacy transaction mode does not BEGIN on SELECT. Explicitly + # start a transaction so owner/revision checks and content share a snapshot. + if session.bind.dialect.name == "sqlite": + await session.execute(text("BEGIN")) + + async def _run(self, session, thread_id: str, run_id: str, *, write: bool = False): + columns = [ + RunRow.thread_id, + RunRow.run_id, + RunRow.user_id, + RunRow.assistant_id, + RunRow.status, + RunRow.stop_reason, + RunRow.created_at, + RunRow.evidence_origin, + RunRow.evidence_agent_id, + RunRow.evidence_seal_state, + RunRow.evidence_seal_error, + RunRow.evidence_revision, + RunRow.evidence_upper_seq, + RunRow.evidence_event_count, + RunRow.evidence_retention_revision, + RunRow.lease_expires_at, + ] + stmt = select(*columns).where(RunRow.thread_id == thread_id, RunRow.run_id == run_id, RunRow.user_id.in_(self._owners), RunRow.operation_kind == "run").with_for_update(read=not write) + row = (await session.execute(stmt)).one_or_none() + if row is None: + raise HostCapabilityError("NOT_FOUND", "run or snapshot is unavailable in this scope") + return row + + @staticmethod + def _seal_state(run) -> str: + seal = run.evidence_seal_state or "partial" + if seal == "pending" and run.status not in ("pending", "running"): + lease = run.lease_expires_at + if lease is None or lease.replace(tzinfo=UTC) <= datetime.now(UTC): + return "partial" + return seal + + @classmethod + def revision_for_run(cls, run) -> str: + # Include lifecycle transitions before a seal exists. Otherwise a + # pending snapshot could hide the subsequent terminal/partial record. + return hashlib.sha256(_json([run.evidence_revision, run.run_id, coerce_iso(run.created_at), run.status, cls._seal_state(run)]).encode()).hexdigest() + + @staticmethod + def _event_conditions(run): + return [RunEventRow.thread_id == run.thread_id, RunEventRow.run_id == run.run_id, or_(RunEventRow.user_id == run.user_id, RunEventRow.user_id.is_(None))] + + async def get_snapshot(self, *, thread_id: str, run_id: str, limits: EvidenceLimits = EvidenceLimits()) -> CompletedRunSnapshot: + self._supported() + if not isinstance(limits, EvidenceLimits): + raise ValueError("limits must be EvidenceLimits") + async with self._sf() as session: + if session.bind.dialect.name == "sqlite": + await session.execute(text("BEGIN IMMEDIATE")) + run = await self._run(session, thread_id, run_id, write=True) + revision = self.revision_for_run(run) + existing = await session.scalar( + select(CompletedRunSnapshotRow).where( + CompletedRunSnapshotRow.run_id == run_id, + CompletedRunSnapshotRow.scope_digest == self._scope, + CompletedRunSnapshotRow.evidence_revision == revision, + CompletedRunSnapshotRow.retention_revision == run.evidence_retention_revision, + ) + ) + if existing is not None: + snap = _snapshot(existing.snapshot_json) + if _size(snap) > limits.max_bytes: + raise HostCapabilityError("LIMIT_EXCEEDED", "snapshot envelope exceeds byte budget") + return snap + seal = self._seal_state(run) + if seal == "sealed": + upper, count = run.evidence_upper_seq, run.evidence_event_count + if upper is None or count is None: + seal = "partial" + if seal != "sealed": + upper, count = (await session.execute(select(func.max(RunEventRow.seq), func.count()).where(*self._event_conditions(run)))).one() + snap = CompletedRunSnapshot( + snapshot_ref=uuid4().hex, + evidence_revision=revision, + retention_revision=run.evidence_retention_revision, + thread_id=thread_id, + run_id=run_id, + owner_id=run.user_id, + agent_id=run.evidence_agent_id, + origin=run.evidence_origin or "unknown", + status=run.status, + stop_reason=run.stop_reason, + seal_state=seal, + seal_error=run.evidence_seal_error, + upper_event_seq=upper or 0, + event_count=count or 0, + ) + if _size(snap) > limits.max_bytes: + raise HostCapabilityError("LIMIT_EXCEEDED", "snapshot envelope exceeds byte budget") + session.add(CompletedRunSnapshotRow(snapshot_ref=snap.snapshot_ref, run_id=run_id, scope_digest=self._scope, evidence_revision=revision, retention_revision=snap.retention_revision, snapshot_json=asdict(snap))) + await session.commit() + return snap + + @staticmethod + def _content_bytes(session): + if session.bind.dialect.name == "postgresql": + return func.convert_to(RunEventRow.content, "UTF8") + return cast(RunEventRow.content, LargeBinary) + + async def _content_hash(self, session, event_id: int, size: int, budget: dict, raw: bytes | None = None) -> str: + chunks = (size + 16383) // 16384 if raw is None else 0 + remaining_time = budget["deadline"] - monotonic() + if size > budget["bytes"] or chunks > budget["queries"] or remaining_time <= 0: + raise HostCapabilityError("LIMIT_EXCEEDED", "legacy event digest exceeds the per-page verification budget") + budget["bytes"] -= size + budget["queries"] -= chunks + if raw is not None: + return hashlib.sha256(raw).hexdigest() + digest = hashlib.sha256() + try: + async with asyncio.timeout(remaining_time): + for start in range(1, size + 1, 16384): + chunk = await session.scalar(select(func.substr(self._content_bytes(session), start, 16384)).where(RunEventRow.id == event_id)) + digest.update(chunk or b"") + except TimeoutError as exc: + raise HostCapabilityError("LIMIT_EXCEEDED", "legacy event digest verification deadline exceeded") from exc + return digest.hexdigest() + + async def _resolve_snapshot(self, session, snapshot_ref: str): + stored = await session.scalar(select(CompletedRunSnapshotRow).where(CompletedRunSnapshotRow.snapshot_ref == snapshot_ref, CompletedRunSnapshotRow.scope_digest == self._scope)) + if stored is None: + raise HostCapabilityError("NOT_FOUND", "snapshot is unavailable in this scope") + snap = _snapshot(stored.snapshot_json) + run = await self._run(session, snap.thread_id, snap.run_id) + if run.user_id != snap.owner_id or run.evidence_retention_revision != snap.retention_revision or self.revision_for_run(run) != snap.evidence_revision: + raise HostCapabilityError("EVIDENCE_EXPIRED", "snapshot revision was replaced or its events were removed") + return snap, run + + async def resolve_snapshot(self, snapshot_ref: str) -> CompletedRunSnapshot: + """Internal host composition hook; authorization is checked anew per call.""" + self._supported() + async with self._sf() as session: + await self._begin_read(session) + snap, _run = await self._resolve_snapshot(session, snapshot_ref) + return snap + + async def read_events(self, *, snapshot_ref: str, cursor: str | None = None, limits: EvidenceLimits = EvidenceLimits()) -> CompletedRunEventPage: + self._supported() + if not isinstance(limits, EvidenceLimits): + raise ValueError("limits must be EvidenceLimits") + after = self._position(cursor, "events", snapshot_ref) or 0 + legacy_budget = {"bytes": 1024 * 1024, "queries": 64, "deadline": monotonic() + 2} + async with self._sf() as session: + await self._begin_read(session) + snap, run = await self._resolve_snapshot(session, snapshot_ref) + conditions = [*self._event_conditions(run), RunEventRow.seq > after, RunEventRow.seq <= snap.upper_event_seq] + length = (lambda col: func.octet_length(col)) if session.bind.dialect.name == "postgresql" else (lambda col: func.length(cast(col, LargeBinary))) + metadata_text = cast(RunEventRow.event_metadata, Text) + # Metadata and body lengths are projected separately; ORM never + # hydrates all rows (or an entire giant event) for a bounded page. + rows = ( + await session.execute( + select(RunEventRow.id, RunEventRow.seq, RunEventRow.event_type, RunEventRow.category, RunEventRow.content_sha256, length(RunEventRow.content).label("body_size"), length(metadata_text).label("meta_size")) + .where(*conditions) + .order_by(RunEventRow.seq) + .limit(limits.max_events + 1) + ) + ).all() + items = [] + next_cursor = cursor + for row in rows[: limits.max_events]: + remaining = limits.max_bytes - _size(CompletedRunEventPage(tuple(items), next_cursor, True)) + oversized = (row.body_size or 0) + (row.meta_size or 0) > max(0, remaining - 700) // 2 + body_limit = 128 if oversized else limits.max_bytes + raw_bytes, raw_meta = (await session.execute(select(func.substr(self._content_bytes(session), 1, body_limit), func.substr(metadata_text, 1, 2 if oversized else limits.max_bytes)).where(RunEventRow.id == row.id))).one() + raw = (raw_bytes or b"").decode("utf-8", errors="ignore" if oversized else "strict") + metadata = {} if oversized else json.loads(raw_meta or "{}") + content = raw or "" + if not oversized and (metadata.get("content_is_json") or metadata.get("content_is_dict")): + try: + content = json.loads(content) + except ValueError: + pass + digest = row.content_sha256 or await self._content_hash(session, row.id, row.body_size or 0, legacy_budget, None if oversized else raw_bytes or b"") + item = CompletedRunEvent( + event_id=str(row.id), + seq=row.seq, + event_type=row.event_type, + category=row.category, + content_json=_json(content), + metadata_json=_json(redact_metadata_secrets(metadata)), + content_bytes=row.body_size or 0, + content_sha256=digest, + truncated=oversized, + ) + candidate_cursor = self._cursor("events", row.seq, snapshot_ref) + candidate = CompletedRunEventPage(tuple([*items, item]), candidate_cursor, True) + if _size(candidate) > limits.max_bytes: + if items: + break + item = replace(item, content_json='""', metadata_json="{}", truncated=True) + candidate = CompletedRunEventPage((item,), candidate_cursor, True) + if _size(candidate) > limits.max_bytes: + raise HostCapabilityError("LIMIT_EXCEEDED", "event identity envelope exceeds byte budget") + items.append(item) + next_cursor = candidate_cursor + return CompletedRunEventPage(tuple(items), next_cursor, len(rows) > len(items)) diff --git a/backend/packages/harness/deerflow/extensions/gateway.py b/backend/packages/harness/deerflow/extensions/gateway.py index d0bf2c903f..5d3730e105 100644 --- a/backend/packages/harness/deerflow/extensions/gateway.py +++ b/backend/packages/harness/deerflow/extensions/gateway.py @@ -579,6 +579,7 @@ async def start_services( session_factory: Any | None, *, run_evidence_reader: Any | None = None, + host_capabilities: dict[str, tuple[Any | None, Any | None]] | None = None, attempted_services: list[tuple[str, Any]] | None = None, ) -> list[Diagnostic]: """Start extension services in registration order, failing open per item.""" @@ -586,14 +587,24 @@ async def start_services( if not extensions.services: return diagnostics - deps = ExtensionRuntimeDeps( + base_deps = ExtensionRuntimeDeps( app_store=extensions.app_store, policy=project_host_policy(app_config), session_factory=session_factory, run_evidence_reader=run_evidence_reader, ) + source_deps = {} for entry in extensions.services: source, service = entry + evidence, mutations = (host_capabilities or {}).get(source, (None, None)) + if evidence is None and mutations is None: + deps = base_deps # Preserve the legacy shared-dependency identity. + else: + from dataclasses import replace + + if source not in source_deps: + source_deps[source] = replace(base_deps, completed_run_evidence=evidence, skill_mutations=mutations) + deps = source_deps[source] if attempted_services is not None: # Record before awaiting start(): a service may acquire resources # and then fail or be cancelled, so it still owns stop(). diff --git a/backend/packages/harness/deerflow/extensions/host_access.py b/backend/packages/harness/deerflow/extensions/host_access.py new file mode 100644 index 0000000000..ce99e9db2a --- /dev/null +++ b/backend/packages/harness/deerflow/extensions/host_access.py @@ -0,0 +1,91 @@ +"""Restart-bound, host-owned grants; not a sandbox for trusted Python code.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections import Counter +from dataclasses import dataclass +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from deerflow.extensions.model_access import ExtensionHostAccess + + +class EvidenceAccess(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + owners: tuple[str, ...] = () + + @field_validator("owners") + @classmethod + def exact_identifiers(cls, values): + if len(values) > 1000 or len(set(values)) != len(values): + raise ValueError("grant lists must be bounded and unique") + if any(not value or len(value) > 128 or value.strip() != value or "*" in value or any(ord(c) < 32 for c in value) for value in values): + raise ValueError("grant lists require exact nonempty identifiers, without wildcards") + return values + + +class SkillMutationAccess(EvidenceAccess): + operations: tuple[Literal["stage", "check", "commit", "revert"], ...] = () + trigger_agents: tuple[str, ...] = () + target_skills: tuple[str, ...] = () + topology: Literal["single_host_local"] | None = Field( + default=None, + description="Operator assertion: all writers upgraded; one POSIX host and shared local disk, not NFS/CSI. Required for mutation service.", + ) + + @field_validator("trigger_agents", "target_skills") + @classmethod + def exact_targets(cls, values): + return cls.exact_identifiers(values) + + @field_validator("target_skills") + @classmethod + def skill_names(cls, values): + if any(not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name) or len(name) > 64 for name in values): + raise ValueError("target_skills must contain canonical skill names") + return values + + +class HostAccess(ExtensionHostAccess): + model_config = ConfigDict(extra="forbid", frozen=True) + evidence: EvidenceAccess = Field(default_factory=EvidenceAccess) + skill_mutations: SkillMutationAccess = Field(default_factory=SkillMutationAccess) + + @property + def granted(self): + return bool(self.evidence.owners or self.skill_mutations.owners or self.skill_mutations.operations) + + +@dataclass(frozen=True) +class BoundHostAccess: + plugin_id: str + access: HostAccess + + def allows(self, operation: str, owner_id: str, name: str) -> bool: + grant = self.access.skill_mutations + return operation in grant.operations and owner_id in grant.owners and name in grant.target_skills + + +def bind_host_access(specs) -> dict[str, BoundHostAccess]: + """Source attribution is host registry data, never a plugin-supplied ID. + + Ungranted legacy duplicate entry points stay legal. Once a source is + granted, all of its enabled registrations must identify one installation. + Changing the name or entry point deliberately creates a new durable identity. + """ + enabled = [spec for spec in specs if spec.enabled] + names = Counter(spec.name for spec in enabled) + entries = Counter(spec.use for spec in enabled) + bindings = {} + for spec in enabled: + if not spec.host_access.granted: + continue + if not spec.name or names[spec.name] != 1 or entries[spec.use] != 1: + raise ValueError("host_access requires a unique explicit plugin name and entry point") + identity = json.dumps([spec.name, spec.use], ensure_ascii=True, separators=(",", ":")) + bindings[spec.use] = BoundHostAccess(hashlib.sha256(identity.encode()).hexdigest(), spec.host_access) + return bindings diff --git a/backend/packages/harness/deerflow/extensions/host_capabilities.py b/backend/packages/harness/deerflow/extensions/host_capabilities.py new file mode 100644 index 0000000000..492fdf07cd --- /dev/null +++ b/backend/packages/harness/deerflow/extensions/host_capabilities.py @@ -0,0 +1,151 @@ +"""Gateway-owned capability assembly and plugin-independent recovery lifetime.""" + +from __future__ import annotations + +import asyncio +import logging + +from deerflow_extension_api import HostCapabilityError +from sqlalchemy import select + +from deerflow.extensions.completed_run_evidence import HostCompletedRunEvidenceReader +from deerflow.extensions.host_access import bind_host_access +from deerflow.persistence.skill_mutations.model import SkillAssetRow, SkillOwnerRow +from deerflow.skills.mutations.guard import SkillMutationRuntime, configure_mutation_runtime +from deerflow.skills.mutations.repository import SkillMutationRepository +from deerflow.skills.mutations.topology import PublicationTopology, mutation_session_factory +from deerflow.skills.mutations.workers import MutationWorkers +from deerflow.utils.file_io import await_drained + +logger = logging.getLogger(__name__) + + +class HostCapabilities: + """Register close() before start(); extension services must stop before close(). + + Grants are restart-bound. Persisted enrollment is intentionally independent + of the current plugin list: uninstalling a plugin cannot turn off recovery + or let an old managed writer bypass an unresolved publication. + """ + + def __init__(self, config, run_store, event_store, *, root=None, storage_factory=None, recovery_factory=None, service_factory=None): + self.config = config + self.run_store = run_store + self.event_store = event_store + self.root = root + self.storage_factory = storage_factory + self.recovery_factory = recovery_factory + self.service_factory = service_factory + self.bindings = {} + self.runtime = None + self.recovery = None + self.workers = None + self._engine = None + self._topology = None + self._maintenance = None + self._services = [] + + async def start(self): + access = bind_host_access(getattr(self.config, "plugins", ())) + mutation_bindings = {source: binding for source, binding in access.items() if binding.access.skill_mutations.owners} + for binding in mutation_bindings.values(): + if binding.access.skill_mutations.topology != "single_host_local": + raise HostCapabilityError("UNSUPPORTED_TOPOLOGY", "Skill mutations require an explicit single_host_local operator assertion") + if not set(binding.access.skill_mutations.owners).issubset(binding.access.evidence.owners): + raise HostCapabilityError("INVALID_GRANT", "Skill mutation owners must also have completed-evidence access") + + owners = set() + sf = getattr(self.run_store, "_sf", None) + if sf is not None: + async with sf() as session: + owners.update(await session.scalars(select(SkillOwnerRow.owner_id))) + owners.update(await session.scalars(select(SkillAssetRow.owner_id).distinct())) + for binding in mutation_bindings.values(): + owners.update(binding.access.skill_mutations.owners) + + for source, binding in access.items(): + evidence = None + if binding.access.evidence.owners: + evidence = HostCompletedRunEvidenceReader(self.run_store, self.event_store, plugin_id=binding.plugin_id, owner_ids=binding.access.evidence.owners) + if source in mutation_bindings: + # Refuse a working-looking automatic API on unsupported stores. + evidence._supported() + self.bindings[source] = (evidence, None) + + if not owners: + return + if self.config.database.backend not in {"sqlite", "postgres"}: + raise HostCapabilityError("UNSUPPORTED_TOPOLOGY") + + self.workers = MutationWorkers() + + def initialize(_check): + from deerflow.config.paths import get_paths + + self._topology = PublicationTopology(self.config.database.backend, self.root or get_paths().base_dir) + self._topology.acquire() + self._engine, sessions = mutation_session_factory(self.config.database.app_sync_sqlalchemy_url, postgres_schema=self.config.database.postgres_schema) + self.runtime = SkillMutationRuntime(SkillMutationRepository(sessions), owners=frozenset(owners)) + if self.storage_factory is None: + from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage + + self.storage_factory = lambda owner: UserScopedSkillStorage(owner, app_config=self.config) + factory = self.recovery_factory + if factory is None: + from deerflow.skills.mutations.recovery import SkillMutationRecovery + + factory = SkillMutationRecovery + self.recovery = factory(self.runtime, self.storage_factory) + configure_mutation_runtime(self.runtime) + self.recovery.recover_all() + self.recovery.collect_garbage() + + await self.workers.run(initialize) + if mutation_bindings: + from deerflow.config import get_app_config + from deerflow.skills.mutations.scanner import CandidateScanner + + factory = self.service_factory + if factory is None: + from deerflow.skills.mutations.service import HostSkillMutationService + + factory = HostSkillMutationService + scanner = CandidateScanner(get_app_config) + for source, binding in mutation_bindings.items(): + evidence, _ = self.bindings[source] + service = factory(binding, runtime=self.runtime, evidence=evidence, storage_factory=self.storage_factory, scanner=scanner, workers=self.workers, recovery=self.recovery) + self._services.append(service) + self.bindings[source] = (evidence, service) + self._maintenance = asyncio.create_task(self._maintain(), name="skill-mutation-recovery") + + async def _maintain(self): + while True: + await asyncio.sleep(60) + try: + await self.workers.run(lambda _check: (self.recovery.recover_all(), self.recovery.collect_garbage())) + except Exception: + # No candidate bytes, model diagnostics, DB parameters or paths. + logger.warning("Skill mutation maintenance unavailable; owner readiness barriers remain active") + + async def close(self): + async def drain(): + if self._maintenance is not None: + self._maintenance.cancel() + await asyncio.gather(self._maintenance, return_exceptions=True) + self._maintenance = None + for service in self._services: + close = getattr(service, "close", None) + if close is not None: + await close() + if self.workers is not None: + await self.workers.close() + if self.runtime is not None: + configure_mutation_runtime(None) + if self._engine is not None: + await asyncio.to_thread(self._engine.dispose) + self._engine = None + if self._topology is not None: + await asyncio.to_thread(self._topology.close) + self._topology = None + + await await_drained(drain()) diff --git a/backend/packages/harness/deerflow/extensions/loader.py b/backend/packages/harness/deerflow/extensions/loader.py index 41bba4d447..a4b27200a6 100644 --- a/backend/packages/harness/deerflow/extensions/loader.py +++ b/backend/packages/harness/deerflow/extensions/loader.py @@ -14,9 +14,10 @@ from dataclasses import dataclass from typing import Any, Literal from deerflow_extension_api import API_VERSION -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator -from deerflow.extensions.model_access import ExtensionHostAccess, ModelInvocationScope +from deerflow.extensions.host_access import HostAccess, bind_host_access +from deerflow.extensions.model_access import ModelInvocationScope from deerflow.extensions.registry import ExtensionRegistry, LoadedExtensions from deerflow.persistence.migrations._env_filters import register_extension_table_prefix from deerflow.reflection import resolve_variable @@ -44,11 +45,18 @@ class ExtensionSpec(BaseModel): description="Installed Python distribution recorded by the extension manager", ) use: str = Field(description="Entry point path, e.g. 'my_extension:install'") - host_access: ExtensionHostAccess = Field(default_factory=ExtensionHostAccess) config: dict[str, Any] = Field( default_factory=dict, description="Extension-private configuration, passed to install() verbatim", ) + host_access: HostAccess = Field(default_factory=HostAccess) + + @model_validator(mode="after") + def _validate_host_identity(self): + if self.host_access.granted and (not self.name or self.name.strip() != self.name or len(self.name) > 128): + raise ValueError("host_access requires an explicit stable plugin name") + return self + required: bool = Field( default=False, description="When true, a load failure aborts startup instead of being skipped", @@ -156,6 +164,8 @@ def load_extensions(specs: Sequence[ExtensionSpec]) -> tuple[LoadedExtensions, l the Gateway still starts. `required: true` flips that to fail-closed for extensions whose absence changes behaviour rather than just observability. """ + # Validate attribution before importing any extension, even optional ones. + bind_host_access(specs) registry = ExtensionRegistry() diagnostics: list[Diagnostic] = [] loaded_sources: list[str] = [] diff --git a/backend/packages/harness/deerflow/persistence/migrations/AGENTS.md b/backend/packages/harness/deerflow/persistence/migrations/AGENTS.md index ad447b9212..f2af7c34f8 100644 --- a/backend/packages/harness/deerflow/persistence/migrations/AGENTS.md +++ b/backend/packages/harness/deerflow/persistence/migrations/AGENTS.md @@ -29,11 +29,12 @@ The empty-DB path keeps using `create_all` because `Base.metadata` is the only a `0028_parked_attempts` → `0029_scheduler_agent_tasks` → `0030_notification_claim_tokens` → `0031_scheduled_streak_boundary` → `0032_activity_and_task_events` → `0033_batch_result_artifact` → `0034_run_event_seq_watermark` → -`0035_login_throttle` → `0036_run_idempotency_request` → `0037_project_document_summaries` → -`0038_channel_thread_bindings` → `0039_user_disabled` → -`0040_scheduled_group_notice` → `0041_im_notice_preferences` → -`0042_channel_leases` → `0043_runtime_im_notices` → -`0044_channel_runtime_configs` (current head). The preference +`0035_login_throttle` → `0036_run_idempotency_request` → +`0037_project_document_summaries` → `0038_channel_thread_bindings` → +`0039_user_disabled` → `0040_scheduled_group_notice` → +`0041_im_notice_preferences` → `0042_channel_leases` → +`0043_runtime_im_notices` → `0044_channel_runtime_configs` → +`0045_completed_run_evidence` → `0046_skill_mutations` (current head). The preference revision adds a separate owner/key table with a cascading users foreign key and does not alter users; the project-documents revision adds a new owner-scoped shelf table, and the MCP lease-token revision adds two nullable token columns to @@ -192,7 +193,6 @@ on installs that never enabled it. The convention is: - `extensions/loader.py::load_extensions` — registers each spec's `table_prefix` with `register_extension_table_prefix()` - Tests: `tests/test_persistence_bootstrap.py` (branches), `tests/test_persistence_bootstrap_concurrency.py` (concurrency), `tests/test_persistence_bootstrap_regression.py` (issue #3682), `tests/test_persistence_migrations_env.py` (filter, including extension-owned tables), `tests/test_extension_loader.py::TestTablePrefixRegistration` (spec-to-filter wiring), `tests/blocking_io/test_persistence_bootstrap.py` (asyncio.to_thread anchor), `tests/test_migration_0004_run_ownership_dedupe.py` + `tests/test_migration_0007_scheduled_run_active_dedupe.py` (dedupe-before-unique-index pre-steps), `tests/test_migration_0025_repair_run_change_seq.py` (issue #5516 skipped-revision heal) - `0041_im_notice_preferences` adds the owner/provider proactive-notification boolean table without a users FK (local synthetic IM owners are supported). Upgrade/drop are table-guarded; fresh bootstrap registers the ORM model. @@ -200,3 +200,6 @@ It follows 0040 without reparenting history, has no backfill and leaves the forward-compatibility floor unchanged. Downgrade removes the saved choices; reenabling a provider never reopens notifications canceled before downgrade. Tests: `test_migration_0041_im_notice_preferences.py`. + +- `0045_completed_run_evidence` adds nullable evidence sealing fields and completed-run snapshots after `0044_channel_runtime_configs`. +- `0046_skill_mutations` adds durable owner enrollment, asset revisions, proposals, publication operations and scan leases. The single-head pin lives in `tests/test_migration_0046_skill_mutations.py`. diff --git a/backend/packages/harness/deerflow/persistence/migrations/versions/0045_completed_run_evidence.py b/backend/packages/harness/deerflow/persistence/migrations/versions/0045_completed_run_evidence.py new file mode 100644 index 0000000000..1fc070a955 --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/migrations/versions/0045_completed_run_evidence.py @@ -0,0 +1,48 @@ +"""Add explicit evidence seal and durable bounded snapshot references.""" + +import sqlalchemy as sa +from alembic import op + +revision = "0045_completed_run_evidence" +down_revision = "0044_channel_runtime_configs" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + from deerflow.persistence.migrations._helpers import safe_add_column + + safe_add_column("run_events", sa.Column("content_sha256", sa.String(64), nullable=True)) + for column in ( + sa.Column("evidence_origin", sa.String(32), nullable=True), + sa.Column("evidence_agent_id", sa.String(128), nullable=True), + sa.Column("evidence_seal_state", sa.String(16), nullable=True), + sa.Column("evidence_seal_error", sa.String(512), nullable=True), + sa.Column("evidence_revision", sa.String(64), nullable=True), + sa.Column("evidence_upper_seq", sa.BigInteger(), nullable=True), + sa.Column("evidence_event_count", sa.BigInteger(), nullable=True), + sa.Column("evidence_retention_revision", sa.BigInteger(), nullable=False, server_default=sa.text("0")), + ): + safe_add_column("runs", column) + if "completed_run_snapshots" not in sa.inspect(op.get_bind()).get_table_names(): + op.create_table( + "completed_run_snapshots", + sa.Column("snapshot_ref", sa.String(64), primary_key=True), + sa.Column("run_id", sa.String(64), nullable=False), + sa.Column("scope_digest", sa.String(64), nullable=False), + sa.Column("evidence_revision", sa.String(64), nullable=False), + sa.Column("retention_revision", sa.BigInteger(), nullable=False), + sa.Column("snapshot_json", sa.JSON(), nullable=False), + sa.UniqueConstraint("run_id", "scope_digest", "evidence_revision", "retention_revision", name="uq_completed_run_snapshot_revision"), + ) + op.create_index("ix_completed_run_snapshots_run_id", "completed_run_snapshots", ["run_id"]) + + +def downgrade() -> None: + from deerflow.persistence.migrations._helpers import safe_drop_column + + safe_drop_column("run_events", "content_sha256") + if "completed_run_snapshots" in sa.inspect(op.get_bind()).get_table_names(): + op.drop_table("completed_run_snapshots") + for name in ("evidence_retention_revision", "evidence_event_count", "evidence_upper_seq", "evidence_revision", "evidence_seal_error", "evidence_seal_state", "evidence_agent_id", "evidence_origin"): + safe_drop_column("runs", name) diff --git a/backend/packages/harness/deerflow/persistence/migrations/versions/0046_skill_mutations.py b/backend/packages/harness/deerflow/persistence/migrations/versions/0046_skill_mutations.py new file mode 100644 index 0000000000..7fdbdef4ec --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/migrations/versions/0046_skill_mutations.py @@ -0,0 +1,161 @@ +"""skill mutation publication journal + +Revision ID: 0046_skill_mutations +Revises: 0045_completed_run_evidence +Create Date: 2026-09-21 11:28:24.923105 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0046_skill_mutations" +down_revision: str | Sequence[str] | None = "0045_completed_run_evidence" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _create_table(name, *columns): + # Legacy databases may already contain tables from an older create_all. + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table(name): + op.create_table(name, *columns) + else: + existing = {column["name"] for column in inspector.get_columns(name)} + required = {column.name for column in columns if isinstance(column, sa.Column)} + if not required <= existing: + raise RuntimeError(f"Incompatible preexisting skill mutation table: {name}") + + +def _create_index(name, table, columns): + if name not in {index["name"] for index in sa.inspect(op.get_bind()).get_indexes(table)}: + op.create_index(name, table, columns, unique=False) + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + _create_table( + "skill_mutation_assets", + sa.Column("target_ref", sa.String(length=32), nullable=False), + sa.Column("owner_id", sa.String(length=36), nullable=False), + sa.Column("name", sa.String(length=64), nullable=False), + sa.Column("incarnation_id", sa.String(length=32), nullable=False), + sa.Column("mutation_seq", sa.Integer(), nullable=False), + sa.Column("content_digest", sa.String(length=64), nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=False), + sa.Column("deleted", sa.Boolean(), nullable=False), + sa.Column("mutating", sa.Boolean(), nullable=False), + sa.Column("operation_id", sa.String(length=32), nullable=True), + sa.ForeignKeyConstraint(["owner_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("target_ref"), + sa.UniqueConstraint("owner_id", "name", name="uq_skill_asset_owner_name"), + ) + _create_index("ix_skill_mutation_assets_owner_id", "skill_mutation_assets", ["owner_id"]) + + _create_table( + "skill_mutation_operations", + sa.Column("operation_id", sa.String(length=32), nullable=False), + sa.Column("plugin_id", sa.String(length=128), nullable=False), + sa.Column("owner_id", sa.String(length=36), nullable=False), + sa.Column("target_ref", sa.String(length=32), nullable=False), + sa.Column("name", sa.String(length=64), nullable=False), + sa.Column("method", sa.String(length=16), nullable=False), + sa.Column("idempotency_key", sa.String(length=128), nullable=False), + sa.Column("request_hash", sa.String(length=64), nullable=False), + sa.Column("proposal_id", sa.String(length=32), nullable=True), + sa.Column("reverts_operation_id", sa.String(length=32), nullable=True), + sa.Column("before_operation_id", sa.String(length=32), nullable=True), + sa.Column("before_revision", sa.JSON(), nullable=False), + sa.Column("after_revision", sa.JSON(), nullable=False), + sa.Column("before_blob", sa.LargeBinary(), nullable=True), + sa.Column("after_blob", sa.LargeBinary(), nullable=True), + sa.Column("generation", sa.Integer(), nullable=False), + sa.Column("publication", sa.String(length=16), nullable=False), + sa.Column("views", sa.String(length=16), nullable=False), + sa.Column("superseded_by_generation", sa.Integer(), nullable=True), + sa.Column("created_at", sa.Float(), nullable=False), + sa.Column("rollback_expires_at", sa.Float(), nullable=False), + sa.Column("assessment", sa.JSON(), nullable=True), + sa.Column("error_code", sa.String(length=64), nullable=True), + sa.ForeignKeyConstraint(["owner_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("operation_id"), + sa.UniqueConstraint("plugin_id", "method", "idempotency_key", name="uq_skill_operation_key"), + sa.UniqueConstraint("proposal_id"), + ) + _create_index("ix_skill_mutation_operations_owner_id", "skill_mutation_operations", ["owner_id"]) + _create_index("ix_skill_mutation_operations_publication", "skill_mutation_operations", ["publication"]) + + _create_table( + "skill_mutation_owners", + sa.Column("owner_id", sa.String(length=36), nullable=False), + sa.Column("generation", sa.Integer(), nullable=False), + sa.Column("deleting", sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint(["owner_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("owner_id"), + ) + _create_table( + "skill_mutation_proposals", + sa.Column("proposal_id", sa.String(length=32), nullable=False), + sa.Column("plugin_id", sa.String(length=128), nullable=False), + sa.Column("owner_id", sa.String(length=36), nullable=False), + sa.Column("target_ref", sa.String(length=32), nullable=False), + sa.Column("name", sa.String(length=64), nullable=False), + sa.Column("base_revision", sa.JSON(), nullable=False), + sa.Column("sources", sa.JSON(), nullable=False), + sa.Column("candidate_hash", sa.String(length=64), nullable=False), + sa.Column("package_blob", sa.LargeBinary(), nullable=True), + sa.Column("candidate_blob", sa.LargeBinary(), nullable=True), + sa.Column("state", sa.String(length=16), nullable=False), + sa.Column("expires_at", sa.Float(), nullable=False), + sa.Column("check_result", sa.JSON(), nullable=True), + sa.Column("idempotency_key", sa.String(length=128), nullable=False), + sa.Column("request_hash", sa.String(length=64), nullable=False), + sa.Column("operation_id", sa.String(length=32), nullable=True), + sa.ForeignKeyConstraint(["owner_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("proposal_id"), + sa.UniqueConstraint("plugin_id", "idempotency_key", name="uq_skill_proposal_key"), + ) + _create_index("ix_skill_mutation_proposals_owner_id", "skill_mutation_proposals", ["owner_id"]) + + _create_table( + "skill_mutation_scan_attempts", + sa.Column("token", sa.String(length=32), nullable=False), + sa.Column("plugin_id", sa.String(length=128), nullable=False), + sa.Column("owner_id", sa.String(length=36), nullable=False), + sa.Column("subject", sa.String(length=80), nullable=False), + sa.Column("started_at", sa.Float(), nullable=False), + sa.Column("lease_until", sa.Float(), nullable=False), + sa.ForeignKeyConstraint(["owner_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("token"), + ) + _create_index("ix_skill_mutation_scan_attempts_owner_id", "skill_mutation_scan_attempts", ["owner_id"]) + _create_index("ix_skill_mutation_scan_attempts_plugin_id", "skill_mutation_scan_attempts", ["plugin_id"]) + + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("skill_mutation_scan_attempts") + with op.batch_alter_table("skill_mutation_proposals", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_skill_mutation_proposals_owner_id")) + + op.drop_table("skill_mutation_proposals") + op.drop_table("skill_mutation_owners") + with op.batch_alter_table("skill_mutation_operations", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_skill_mutation_operations_publication")) + batch_op.drop_index(batch_op.f("ix_skill_mutation_operations_owner_id")) + + op.drop_table("skill_mutation_operations") + with op.batch_alter_table("skill_mutation_assets", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_skill_mutation_assets_owner_id")) + + op.drop_table("skill_mutation_assets") + # ### end Alembic commands ### diff --git a/backend/packages/harness/deerflow/persistence/models/__init__.py b/backend/packages/harness/deerflow/persistence/models/__init__.py index 3f8fa7ddb6..8605ba57fc 100644 --- a/backend/packages/harness/deerflow/persistence/models/__init__.py +++ b/backend/packages/harness/deerflow/persistence/models/__init__.py @@ -36,6 +36,7 @@ from deerflow.persistence.run.model import RunChangeClockRow, RunRow from deerflow.persistence.scheduled_task_events.model import ScheduledTaskEventRow from deerflow.persistence.scheduled_task_runs.model import ScheduledTaskRunRow from deerflow.persistence.scheduled_tasks.model import ScheduledTaskRow +from deerflow.persistence.skill_mutations.model import SkillAssetRow, SkillOperationRow, SkillOwnerRow, SkillProposalRow from deerflow.persistence.subagent_batches.model import SubagentBatchItemRow, SubagentBatchRow from deerflow.persistence.thread_meta.model import ThreadMetaRow from deerflow.persistence.thread_reads.model import ThreadReadMarkerRow, ThreadReadVersionRow @@ -64,6 +65,10 @@ __all__ = [ "RunEventThreadSeqRow", "RunChangeClockRow", "RunRow", + "SkillAssetRow", + "SkillOperationRow", + "SkillOwnerRow", + "SkillProposalRow", "ScheduledTaskEventRow", "ScheduledTaskRow", "ScheduledTaskRunRow", diff --git a/backend/packages/harness/deerflow/persistence/models/run_event.py b/backend/packages/harness/deerflow/persistence/models/run_event.py index 479602858f..ee06eb242d 100644 --- a/backend/packages/harness/deerflow/persistence/models/run_event.py +++ b/backend/packages/harness/deerflow/persistence/models/run_event.py @@ -25,6 +25,9 @@ class RunEventRow(Base): category: Mapped[str] = mapped_column(String(RUN_EVENT_CATEGORY_MAX_LENGTH), nullable=False) # Category values and semantics are defined by runtime/events/catalog.py content: Mapped[str] = mapped_column(Text, default="") + # Digest of the exact persisted UTF-8 content, after serialization/truncation. + # NULL denotes legacy ingestion; event metadata is never trusted as a digest. + content_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True) event_metadata: Mapped[dict] = mapped_column(JSON, default=dict) seq: Mapped[int] = mapped_column(nullable=False) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC)) diff --git a/backend/packages/harness/deerflow/persistence/run/model.py b/backend/packages/harness/deerflow/persistence/run/model.py index 5f4ba3a896..790c28d095 100644 --- a/backend/packages/harness/deerflow/persistence/run/model.py +++ b/backend/packages/harness/deerflow/persistence/run/model.py @@ -3,8 +3,9 @@ from __future__ import annotations from datetime import UTC, datetime +from uuid import uuid4 -from sqlalchemy import JSON, BigInteger, DateTime, Index, Integer, String, Text, text +from sqlalchemy import JSON, BigInteger, DateTime, Index, Integer, String, Text, UniqueConstraint, text from sqlalchemy.orm import Mapped, mapped_column from deerflow.persistence.base import Base @@ -61,6 +62,14 @@ class RunRow(Base): created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC)) updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) change_seq: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0, server_default=text("0")) + evidence_origin: Mapped[str | None] = mapped_column(String(32), nullable=True) + evidence_agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True) + evidence_seal_state: Mapped[str | None] = mapped_column(String(16), nullable=True) + evidence_seal_error: Mapped[str | None] = mapped_column(String(512), nullable=True) + evidence_revision: Mapped[str | None] = mapped_column(String(64), nullable=True, default=lambda: uuid4().hex) + evidence_upper_seq: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + evidence_event_count: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + evidence_retention_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0, server_default=text("0")) # Server-owned origin kind of the run (``deerflow_origin``: schedule, # im_channel, github, extension, mcp_notification). NULL means an # interactive or legacy run; legacy rows are never backfilled. @@ -95,3 +104,18 @@ class RunChangeClockRow(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True) value: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0, server_default=text("0")) + + +class CompletedRunSnapshotRow(Base): + """Small durable boundaries only; never a copy of event contents.""" + + __tablename__ = "completed_run_snapshots" + + snapshot_ref: Mapped[str] = mapped_column(String(64), primary_key=True) + run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + scope_digest: Mapped[str] = mapped_column(String(64), nullable=False) + evidence_revision: Mapped[str] = mapped_column(String(64), nullable=False) + retention_revision: Mapped[int] = mapped_column(BigInteger, nullable=False) + snapshot_json: Mapped[dict] = mapped_column(JSON, nullable=False) + + __table_args__ = (UniqueConstraint("run_id", "scope_digest", "evidence_revision", "retention_revision", name="uq_completed_run_snapshot_revision"),) diff --git a/backend/packages/harness/deerflow/persistence/run/sql.py b/backend/packages/harness/deerflow/persistence/run/sql.py index 685101cfb2..340a26461b 100644 --- a/backend/packages/harness/deerflow/persistence/run/sql.py +++ b/backend/packages/harness/deerflow/persistence/run/sql.py @@ -10,12 +10,13 @@ from __future__ import annotations import json from datetime import UTC, datetime, timedelta from typing import Any +from uuid import uuid4 -from sqlalchemy import and_, case, delete, func, or_, select, update +from sqlalchemy import and_, case, delete, func, or_, select, text, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker -from deerflow.persistence.run.model import RunChangeClockRow, RunRow +from deerflow.persistence.run.model import CompletedRunSnapshotRow, RunChangeClockRow, RunRow from deerflow.runtime.run_origin import origin_kind_of from deerflow.runtime.runs.store.base import ( LeaseRenewal, @@ -138,6 +139,8 @@ class RunRepository(RunStore): owner_worker_id: str | None = None, lease_expires_at: str | None = None, idempotency_key: str | None = None, + evidence_origin: str = "unknown", + evidence_agent_id: str | None = None, idempotency_request: dict[str, Any] | None = None, ): """Insert or update a run row. @@ -152,6 +155,8 @@ class RunRepository(RunStore): lease_dt = datetime.fromisoformat(lease_expires_at) if lease_expires_at else None values = { "thread_id": thread_id, + "evidence_origin": evidence_origin, + "evidence_agent_id": evidence_agent_id, "assistant_id": assistant_id, "user_id": resolved_user_id, "model_name": self._normalize_model_name(model_name), @@ -175,12 +180,53 @@ class RunRepository(RunStore): values["change_seq"] = await self._next_change_seq(session) row = await session.get(RunRow, run_id) if row is None: - session.add(RunRow(run_id=run_id, created_at=created, **values)) + session.add(RunRow(run_id=run_id, created_at=created, evidence_seal_state="pending", **values)) else: for key, value in values.items(): setattr(row, key, value) await session.commit() + async def seal_evidence(self, run_id: str, *, owner_worker_id: str, receipt) -> bool: + """Persist a completed drain and changed-run position in one fenced transaction.""" + from deerflow.runtime.events.evidence import JournalSealReceipt + + if not isinstance(receipt, JournalSealReceipt) or receipt.session_factory is not self._sf or receipt.run_id != run_id: + return False + async with self._sf() as session: + change_seq = await self._next_change_seq(session) + result = await session.execute( + update(RunRow) + .where( + RunRow.run_id == run_id, + RunRow.thread_id == receipt.thread_id, + RunRow.owner_worker_id == owner_worker_id, + or_(RunRow.lease_expires_at.is_(None), RunRow.lease_expires_at > datetime.now(UTC)), + RunRow.evidence_retention_revision == receipt.retention_revision, + RunRow.status.in_(("success", "error", "timeout", "interrupted")), + RunRow.evidence_seal_state == "pending", + ) + .values(evidence_seal_state="sealed", evidence_revision=uuid4().hex, evidence_upper_seq=receipt.upper_event_seq, evidence_event_count=receipt.event_count, change_seq=change_seq) + ) + await session.commit() + return result.rowcount != 0 + + async def mark_evidence_partial(self, run_id: str, *, owner_worker_id: str, error: str) -> bool: + """Make a failed seal discoverable without changing business outcome.""" + async with self._sf() as session: + change_seq = await self._next_change_seq(session) + result = await session.execute( + update(RunRow) + .where( + RunRow.run_id == run_id, + RunRow.owner_worker_id == owner_worker_id, + or_(RunRow.lease_expires_at.is_(None), RunRow.lease_expires_at > datetime.now(UTC)), + RunRow.evidence_seal_state == "pending", + ) + .values(evidence_seal_state="partial", evidence_seal_error=error[:512], evidence_revision=uuid4().hex, change_seq=change_seq) + ) + await session.commit() + return result.rowcount != 0 + async def get( self, run_id, @@ -386,11 +432,14 @@ class RunRepository(RunStore): ): resolved_user_id = resolve_user_id(user_id, method_name="RunRepository.delete") async with self._sf() as session: - row = await session.get(RunRow, run_id) + if session.bind.dialect.name == "sqlite": + await session.execute(text("BEGIN IMMEDIATE")) + row = await session.get(RunRow, run_id, with_for_update=True) if row is None: return if resolved_user_id is not None and row.user_id != resolved_user_id: return + await session.execute(delete(CompletedRunSnapshotRow).where(CompletedRunSnapshotRow.run_id == run_id)) await session.delete(row) await session.commit() @@ -426,6 +475,14 @@ class RunRepository(RunStore): conditions.append(RunRow.user_id == resolved_user_id) async with self._sf() as session: + if session.bind.dialect.name == "sqlite": + await session.execute(text("BEGIN IMMEDIATE")) + # The run-row lock serializes snapshot creation and cleanup on PG; + # SQLite's immediate transaction supplies the equivalent fence. + # Acquire locks in a separate statement: READ COMMITTED must take a + # fresh snapshot after waiting for an in-flight snapshot creator. + await session.execute(select(RunRow.run_id).where(*conditions).with_for_update()) + await session.execute(delete(CompletedRunSnapshotRow).where(CompletedRunSnapshotRow.run_id.in_(select(RunRow.run_id).where(*conditions)))) count = await session.scalar(select(func.count()).select_from(RunRow).where(*conditions)) or 0 if count: await session.execute(delete(RunRow).where(*conditions)) @@ -793,6 +850,8 @@ class RunRepository(RunStore): values: dict[str, Any] = { "status": "error", "error": error, + "evidence_seal_state": "partial", + "evidence_seal_error": "Worker lease expired without a completed journal seal.", "updated_at": datetime.now(UTC), } if stop_reason is not None: @@ -811,6 +870,47 @@ class RunRepository(RunStore): await session.commit() return result.rowcount != 0 + async def recover_unsealed_evidence( + self, + *, + before: str | None = None, + grace_seconds: int = 10, + exclude_run_ids: tuple[str, ...] = (), + ) -> int: + """Publish crash-interrupted terminal seals without changing business status. + + Clock-first locking matches other run mutations. The bounded scan and + conditional update run in one transaction, so a renewed lease wins. + """ + before_dt = datetime.fromisoformat(before) if isinstance(before, str) else before or datetime.now(UTC) + cutoff = datetime.now(UTC) - timedelta(seconds=grace_seconds) + predicates = ( + RunRow.operation_kind == "run", + RunRow.status.in_(("success", "error", "timeout", "interrupted")), + RunRow.evidence_seal_state == "pending", + RunRow.created_at <= before_dt, + _lease_expired_or_null(RunRow.lease_expires_at, cutoff), + RunRow.run_id.not_in(exclude_run_ids), + ) + async with self._sf() as session: + change_seq = await self._next_change_seq(session) + candidates = tuple((await session.scalars(select(RunRow.run_id).where(*predicates).order_by(RunRow.created_at, RunRow.run_id).limit(200))).all()) + if not candidates: + await session.rollback() + return 0 + result = await session.execute( + update(RunRow) + .where(RunRow.run_id.in_(candidates), *predicates) + .values( + evidence_seal_state="partial", + evidence_seal_error="Worker lease expired without a completed journal seal.", + evidence_revision=uuid4().hex, + change_seq=change_seq, + ) + ) + await session.commit() + return result.rowcount + async def list_inflight_with_expired_lease( self, *, @@ -854,6 +954,8 @@ class RunRepository(RunStore): created_at: str | None = None, grace_seconds: int = 10, idempotency_key: str | None = None, + evidence_origin: str = "unknown", + evidence_agent_id: str | None = None, idempotency_request: dict[str, Any] | None = None, ) -> tuple[dict[str, Any], list[dict[str, Any]]]: """Atomically create a run with cross-process thread-uniqueness. @@ -879,6 +981,9 @@ class RunRepository(RunStore): values = { "thread_id": thread_id, + "evidence_origin": evidence_origin, + "evidence_agent_id": evidence_agent_id, + "evidence_seal_state": "pending", "assistant_id": assistant_id, "user_id": resolved_user_id, "model_name": self._normalize_model_name(model_name), diff --git a/backend/packages/harness/deerflow/persistence/skill_mutations/__init__.py b/backend/packages/harness/deerflow/persistence/skill_mutations/__init__.py new file mode 100644 index 0000000000..e12149b64f --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/skill_mutations/__init__.py @@ -0,0 +1 @@ +"""Durable host state for managed Skill changes.""" diff --git a/backend/packages/harness/deerflow/persistence/skill_mutations/model.py b/backend/packages/harness/deerflow/persistence/skill_mutations/model.py new file mode 100644 index 0000000000..d03fdc06f0 --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/skill_mutations/model.py @@ -0,0 +1,89 @@ +"""Host-owned skill version and publication records (Alembic managed).""" + +from __future__ import annotations + +from sqlalchemy import JSON, Boolean, ForeignKey, Integer, LargeBinary, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from deerflow.persistence.base import Base + + +class SkillOwnerRow(Base): + __tablename__ = "skill_mutation_owners" + owner_id: Mapped[str] = mapped_column(String(36), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True) + generation: Mapped[int] = mapped_column(Integer, default=0) + deleting: Mapped[bool] = mapped_column(Boolean, default=False) + + +class SkillAssetRow(Base): + __tablename__ = "skill_mutation_assets" + target_ref: Mapped[str] = mapped_column(String(32), primary_key=True) + owner_id: Mapped[str] = mapped_column(String(36), ForeignKey("users.id", ondelete="CASCADE"), index=True) + name: Mapped[str] = mapped_column(String(64)) + incarnation_id: Mapped[str] = mapped_column(String(32)) + mutation_seq: Mapped[int] = mapped_column(Integer) + content_digest: Mapped[str] = mapped_column(String(64)) + enabled: Mapped[bool] = mapped_column(Boolean) + deleted: Mapped[bool] = mapped_column(Boolean, default=False) + mutating: Mapped[bool] = mapped_column(Boolean, default=False) + operation_id: Mapped[str | None] = mapped_column(String(32), nullable=True) + __table_args__ = (UniqueConstraint("owner_id", "name", name="uq_skill_asset_owner_name"),) + + +class SkillProposalRow(Base): + __tablename__ = "skill_mutation_proposals" + proposal_id: Mapped[str] = mapped_column(String(32), primary_key=True) + plugin_id: Mapped[str] = mapped_column(String(128)) + owner_id: Mapped[str] = mapped_column(String(36), ForeignKey("users.id", ondelete="CASCADE"), index=True) + target_ref: Mapped[str] = mapped_column(String(32)) + name: Mapped[str] = mapped_column(String(64)) + base_revision: Mapped[dict] = mapped_column(JSON) + sources: Mapped[list] = mapped_column(JSON) + candidate_hash: Mapped[str] = mapped_column(String(64)) + package_blob: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True, deferred=True) + candidate_blob: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True, deferred=True) + state: Mapped[str] = mapped_column(String(16)) + expires_at: Mapped[float] + check_result: Mapped[dict | None] = mapped_column(JSON, nullable=True) + idempotency_key: Mapped[str] = mapped_column(String(128)) + request_hash: Mapped[str] = mapped_column(String(64)) + operation_id: Mapped[str | None] = mapped_column(String(32), nullable=True) + __table_args__ = (UniqueConstraint("plugin_id", "idempotency_key", name="uq_skill_proposal_key"),) + + +class SkillOperationRow(Base): + __tablename__ = "skill_mutation_operations" + operation_id: Mapped[str] = mapped_column(String(32), primary_key=True) + plugin_id: Mapped[str] = mapped_column(String(128)) + owner_id: Mapped[str] = mapped_column(String(36), ForeignKey("users.id", ondelete="CASCADE"), index=True) + target_ref: Mapped[str] = mapped_column(String(32)) + name: Mapped[str] = mapped_column(String(64)) + method: Mapped[str] = mapped_column(String(16)) + idempotency_key: Mapped[str] = mapped_column(String(128)) + request_hash: Mapped[str] = mapped_column(String(64)) + proposal_id: Mapped[str | None] = mapped_column(String(32), nullable=True, unique=True) + reverts_operation_id: Mapped[str | None] = mapped_column(String(32), nullable=True) + before_operation_id: Mapped[str | None] = mapped_column(String(32), nullable=True) + before_revision: Mapped[dict] = mapped_column(JSON) + after_revision: Mapped[dict] = mapped_column(JSON) + before_blob: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True, deferred=True) + after_blob: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True, deferred=True) + generation: Mapped[int] = mapped_column(Integer) + publication: Mapped[str] = mapped_column(String(16), index=True) + views: Mapped[str] = mapped_column(String(16)) + superseded_by_generation: Mapped[int | None] = mapped_column(Integer, nullable=True) + created_at: Mapped[float] + rollback_expires_at: Mapped[float] + assessment: Mapped[dict | None] = mapped_column(JSON, nullable=True) + error_code: Mapped[str | None] = mapped_column(String(64), nullable=True) + __table_args__ = (UniqueConstraint("plugin_id", "method", "idempotency_key", name="uq_skill_operation_key"),) + + +class SkillScanAttemptRow(Base): + __tablename__ = "skill_mutation_scan_attempts" + token: Mapped[str] = mapped_column(String(32), primary_key=True) + plugin_id: Mapped[str] = mapped_column(String(128), index=True) + owner_id: Mapped[str] = mapped_column(String(36), ForeignKey("users.id", ondelete="CASCADE"), index=True) + subject: Mapped[str] = mapped_column(String(80)) + started_at: Mapped[float] + lease_until: Mapped[float] diff --git a/backend/packages/harness/deerflow/runtime/AGENTS.md b/backend/packages/harness/deerflow/runtime/AGENTS.md index 142ceadab4..942b2dad67 100644 --- a/backend/packages/harness/deerflow/runtime/AGENTS.md +++ b/backend/packages/harness/deerflow/runtime/AGENTS.md @@ -219,6 +219,11 @@ Check all owners and categories; surviving rows retain the allocation floor. **Terminal run cleanup:** Close streams, journals, and graph references even on cancellation. Details: `backend/docs/runtime-guidance-details.md`. +Clear both the terminal worker's journal and message-sequence stamper references; +the stamper's generation callback captures the journal. Diagnostic tracebacks may +retain finalizer frames, so config/callback removal alone is insufficient. Keep +the DEBUG checkpoint-inspection failure regression in `test_run_worker_rollback.py`. + **`RunManager._runs` holds only records this worker admitted.** A cross-worker idempotent reuse returns the `store_only` row from `_record_from_store()` unregistered: the peer never finalizes or `cleanup()`s it, so a registered copy stays `pending`/`running`, 409s later same-thread admissions, hides the owner's orphan from reconciliation, and sends a peer `cancel()` down the local-owner path. Pinned by `test_peer_idempotent_reuse_*` and `test_peer_cancel_of_reused_run_*` (`tests/test_multi_worker_run_ownership.py`) plus `tests/test_gateway_services.py::test_start_run_peer_idempotent_reuse_*`. **A `RunRecord` owner matches its durable row.** HTTP admissions omit `user_id` and the SQL store stamps the ambient user, so `create()` and `_admit_thread_operation()` resolve an omitted owner from the contextvar, keeping `None` without one — never `get_effective_user_id()`'s `default` bucket. A `None` record made SQL keyed retries on a peer or after `cleanup()` return 500, hid HTTP runs from owner-scoped history, and skipped their MCP `background_tasks` projection. Pinned by `test_run_record_owner_matches_row_stamped_from_context`, `test_keyed_retry_without_explicit_user_*`, the `sql` case of `test_start_run_peer_idempotent_reuse_*`, `test_run_manager_*_admitted_without_explicit_user`, and `test_run_manager_keeps_omitted_owner_unset_without_user_context`. @@ -318,3 +323,15 @@ the admitted group keeps its records on success or completes rollback on failure This is a store-local guarantee, not a change to RunJournal cancellation policy or JSONL's single-process deployment constraint. Regression coverage is in `tests/test_jsonl_event_store_cancellation.py`. + +## Completed-evidence provenance + +Gateway admission and RunManager validate host-owned evidence origin against the shared `EVIDENCE_ORIGINS` in `runtime/runs/schemas.py`. Preserve the string values and reject client metadata as provenance; only trusted Python entry points set the admission argument. + +Idempotent resume reuse preserves both the private request identity and the original evidence origin/agent across worker and store boundaries. A retry must not relabel existing evidence. Memory/SQLite parity is covered in `tests/test_completed_evidence_admission.py`. + +Completed-evidence finalization obeys the journal write-ownership contract: +unknown or cancelled write outcomes cannot yield a receipt or be replayed. Only +proven-not-committed batches may be retried. Normal close stops new evidence +producers and drains already admitted cross-thread callbacks inside its operation +lock before flushing and detaching; do not reacquire that lock through `flush()`. diff --git a/backend/packages/harness/deerflow/runtime/events/evidence.py b/backend/packages/harness/deerflow/runtime/events/evidence.py new file mode 100644 index 0000000000..9ca3786593 --- /dev/null +++ b/backend/packages/harness/deerflow/runtime/events/evidence.py @@ -0,0 +1,13 @@ +"""Internal durable journal drain acknowledgement; not an extension write API.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class JournalSealReceipt: + run_id: str + thread_id: str + upper_event_seq: int + event_count: int + retention_revision: int + session_factory: object diff --git a/backend/packages/harness/deerflow/runtime/events/store/db.py b/backend/packages/harness/deerflow/runtime/events/store/db.py index c8b22ed958..6d55f2b84d 100644 --- a/backend/packages/harness/deerflow/runtime/events/store/db.py +++ b/backend/packages/harness/deerflow/runtime/events/store/db.py @@ -7,6 +7,7 @@ at ``max_trace_content`` bytes to avoid bloating the database. from __future__ import annotations import asyncio +import hashlib import json import logging import re @@ -14,10 +15,12 @@ import weakref from datetime import UTC, datetime from typing import Any -from sqlalchemy import delete, func, or_, select, text +from sqlalchemy import delete, func, or_, select, text, update from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from deerflow.persistence.models.run_event import RunEventRow, RunEventThreadSeqRow +from deerflow.persistence.run.model import CompletedRunSnapshotRow, RunRow +from deerflow.persistence.run.sql import RunRepository from deerflow.runtime.events.message_identity import message_identity from deerflow.runtime.events.store.base import RunEventStore from deerflow.runtime.user_context import AUTO, _AutoSentinel, get_current_user, resolve_user_id @@ -232,6 +235,7 @@ class DbRunEventStore(RunEventStore): event_type=event_type, category=category, content=db_content, + content_sha256=hashlib.sha256(db_content.encode("utf-8")).hexdigest(), event_metadata=metadata, seq=seq, created_at=_parse_created_at(created_at), @@ -269,6 +273,7 @@ class DbRunEventStore(RunEventStore): event_type=e["event_type"], category=category, content=db_content, + content_sha256=hashlib.sha256(db_content.encode("utf-8")).hexdigest(), event_metadata=metadata, seq=seq, created_at=_parse_created_at(e.get("created_at")), @@ -324,6 +329,7 @@ class DbRunEventStore(RunEventStore): event_type=event_type, category=category, content=db_content, + content_sha256=hashlib.sha256(db_content.encode("utf-8")).hexdigest(), event_metadata=metadata, seq=(max_seq or 0) + 1, created_at=_parse_created_at(created_at), @@ -364,6 +370,45 @@ class DbRunEventStore(RunEventStore): rows = list(result.scalars()) return [self._row_to_dict(r) for r in reversed(rows)] + async def evidence_receipt(self, thread_id: str, run_id: str): + """Capture acknowledged durable bounds after the journal's producer drain.""" + from deerflow.runtime.events.evidence import JournalSealReceipt + + async with self._get_write_lock(thread_id): + async with self._sf() as session: + async with session.begin(): + await self._acquire_thread_mutation_fence(session, thread_id) + run = (await session.execute(select(RunRow.user_id, RunRow.evidence_retention_revision).where(RunRow.run_id == run_id, RunRow.thread_id == thread_id))).one_or_none() + if run is None: + return None + bounds = await session.execute( + select(func.max(RunEventRow.seq), func.count()).where( + RunEventRow.thread_id == thread_id, + RunEventRow.run_id == run_id, + or_(RunEventRow.user_id == run.user_id, RunEventRow.user_id.is_(None)), + ) + ) + upper, count = bounds.one() + return JournalSealReceipt(run_id, thread_id, upper or 0, count, run.evidence_retention_revision, self._sf) + + @staticmethod + async def _invalidate_evidence(session, thread_id: str, user_id: str | None, change_seq: int, run_id: str | None = None) -> None: + conditions = [RunRow.thread_id == thread_id] + if user_id is not None: + conditions.append(RunRow.user_id == user_id) + if run_id is not None: + conditions.append(RunRow.run_id == run_id) + await session.execute( + update(RunRow) + .where(*conditions) + .values( + evidence_retention_revision=RunRow.evidence_retention_revision + 1, + evidence_seal_state="partial", + change_seq=change_seq, + ) + ) + await session.execute(delete(CompletedRunSnapshotRow).where(CompletedRunSnapshotRow.run_id.in_(select(RunRow.run_id).where(*conditions)))) + async def list_events( self, thread_id, @@ -547,7 +592,9 @@ class DbRunEventStore(RunEventStore): async with self._get_write_lock(thread_id): async with self._sf() as session: async with session.begin(): + change_seq = await RunRepository._next_change_seq(session) await self._acquire_thread_mutation_fence(session, thread_id) + await self._invalidate_evidence(session, thread_id, resolved_user_id, change_seq) count_conditions = [RunEventRow.thread_id == thread_id] if resolved_user_id is not None: count_conditions.append(RunEventRow.user_id == resolved_user_id) @@ -586,7 +633,9 @@ class DbRunEventStore(RunEventStore): async with self._get_write_lock(thread_id): async with self._sf() as session: async with session.begin(): + change_seq = await RunRepository._next_change_seq(session) max_seq = await self._max_seq_for_thread(session, thread_id) + await self._invalidate_evidence(session, thread_id, resolved_user_id, change_seq, run_id) count_conditions = [RunEventRow.thread_id == thread_id, RunEventRow.run_id == run_id] if resolved_user_id is not None: count_conditions.append(RunEventRow.user_id == resolved_user_id) diff --git a/backend/packages/harness/deerflow/runtime/journal.py b/backend/packages/harness/deerflow/runtime/journal.py index 5ea8e1ffaa..5bb978061d 100644 --- a/backend/packages/harness/deerflow/runtime/journal.py +++ b/backend/packages/harness/deerflow/runtime/journal.py @@ -48,6 +48,7 @@ from deerflow.runtime.events.catalog import ( RUN_START_EVENT, ) from deerflow.runtime.events.store.base import RunEventStore, RunEventStoreNotCommittedError +from deerflow.runtime.user_context import AUTO, _AutoSentinel from deerflow.utils.llm_text import strip_leading_think_blocks from deerflow.utils.messages import message_to_text, restore_original_human_message @@ -319,12 +320,19 @@ class RunJournal(BaseCallbackHandler): flush_threshold: int = 20, progress_reporter: Callable[[dict], Awaitable[None]] | None = None, progress_flush_interval: float = 5.0, + user_id: str | None | _AutoSentinel = AUTO, ): super().__init__() + from deerflow.runtime.events.store.db import DbRunEventStore + self.run_id = run_id self.thread_id = thread_id + self._evidence_owner_id = user_id if isinstance(event_store, DbRunEventStore) else AUTO self._store: RunEventStore | None = event_store self._closed = False + self._accepting_evidence = True + self._producer_lock = threading.Lock() + self._pending_producer_callbacks = 0 self._track_tokens = track_token_usage self._flush_threshold = flush_threshold self._progress_reporter = progress_reporter @@ -570,7 +578,7 @@ class RunJournal(BaseCallbackHandler): tags: list[str] | None = None, **kwargs: Any, ) -> None: - if self._closed: + if self._closed or not self._accepting_evidence: return messages: list[AnyMessage] = [] @@ -956,6 +964,7 @@ class RunJournal(BaseCallbackHandler): return { "thread_id": self.thread_id, "run_id": self.run_id, + **({"user_id": self._evidence_owner_id} if self._evidence_owner_id is not AUTO else {}), "event_type": event_type, "category": category, "content": content, @@ -1029,7 +1038,7 @@ class RunJournal(BaseCallbackHandler): caller: str, ) -> None: """Queue one logical response and merge usage into its canonical callback.""" - if self._closed: + if self._closed or not self._accepting_evidence: return has_usage = self._has_positive_usage(events) @@ -1067,8 +1076,8 @@ class RunJournal(BaseCallbackHandler): # Some providers immediately re-fire on_llm_end with usage filled in. # Defer an incomplete copy until the next event or flush. - def _put(self, *, event_type: str, category: str, content: str | dict = "", metadata: dict | None = None) -> None: - if self._closed: + def _put(self, *, event_type: str, category: str, content: str | dict = "", metadata: dict | None = None, _admitted: bool = False) -> None: + if self._closed or (not self._accepting_evidence and not _admitted): return self._commit_pending_llm_response() self._buffer.append(self._make_event(event_type=event_type, category=category, content=content, metadata=metadata)) @@ -1496,17 +1505,22 @@ class RunJournal(BaseCallbackHandler): if owner_loop.is_closed() or not owner_loop.is_running(): logger.warning("Dropping cross-thread middleware event after run loop shutdown") return - try: - owner_loop.call_soon_threadsafe( - partial( - self._put, - event_type=event_type, - category=MIDDLEWARE_EVENT_PATTERN.category, - content={"name": name, "hook": hook, "action": action, "changes": dict(changes)}, + with self._producer_lock: + if not self._accepting_evidence: + return + self._pending_producer_callbacks += 1 + try: + owner_loop.call_soon_threadsafe( + partial( + self._record_admitted_middleware, + event_type=event_type, + category=MIDDLEWARE_EVENT_PATTERN.category, + content={"name": name, "hook": hook, "action": action, "changes": dict(changes)}, + ) ) - ) - except RuntimeError: - logger.warning("Dropping cross-thread middleware event after run loop shutdown") + except RuntimeError: + self._pending_producer_callbacks -= 1 + logger.warning("Dropping cross-thread middleware event after run loop shutdown") return self._put( @@ -1515,6 +1529,13 @@ class RunJournal(BaseCallbackHandler): content={"name": name, "hook": hook, "action": action, "changes": changes}, ) + def _record_admitted_middleware(self, **kwargs) -> None: + try: + self._put(**kwargs, _admitted=True) + finally: + with self._producer_lock: + self._pending_producer_callbacks -= 1 + def claim_tool_promotions(self, tool_names: Iterable[str]) -> list[str]: """Atomically claim names not yet reported by this run's lead agent.""" candidates = sorted(set(tool_names)) @@ -1697,9 +1718,39 @@ class RunJournal(BaseCallbackHandler): _batch, error = self._quarantined_batches[0] raise RuntimeError(f"RunJournal flush refused: {len(_batch)} event(s) for run {self.run_id} have an unknown durable outcome") from error + async def _stop_evidence_producers(self) -> None: + with self._producer_lock: + self._accepting_evidence = False + while self._pending_producer_callbacks: + await asyncio.sleep(0) + + async def _drain_evidence(self) -> None: + if self._closed: + return + await self._stop_evidence_producers() + await self.flush() + + async def finish_evidence(self): + """Stop declared producers and drain accepted work before capturing bounds. + + This is independent of terminal business status and does not detach the + journal. Proven uncommitted writes remain available for explicit retry; + unknown outcomes stay quarantined and never yield a durable receipt. + Unsupported stores return no durable receipt. + """ + if self._closed: + return None + await self._drain_evidence() + from deerflow.runtime.events.store.db import DbRunEventStore + + if isinstance(self._store, DbRunEventStore): + return await self._store.evidence_receipt(self.thread_id, self.run_id) + return None + def _detach_runtime_dependencies(self) -> None: """Drop every external or potentially cyclic run-scoped reference.""" self._closed = True + self._accepting_evidence = False with self._skill_usage_lock: self._skill_usages.clear() self._store = None @@ -1746,6 +1797,7 @@ class RunJournal(BaseCallbackHandler): raise RuntimeError(f"RunJournal for run {self.run_id} was force-closed; close cannot report success") if self._closed: return + await self._stop_evidence_producers() await self._flush_locked() self._detach_runtime_dependencies() return diff --git a/backend/packages/harness/deerflow/runtime/runs/manager.py b/backend/packages/harness/deerflow/runtime/runs/manager.py index 9a31cfd024..8f296f0f0b 100644 --- a/backend/packages/harness/deerflow/runtime/runs/manager.py +++ b/backend/packages/harness/deerflow/runtime/runs/manager.py @@ -22,7 +22,7 @@ from deerflow.runtime.user_context import AUTO, _AutoSentinel, get_current_user, from deerflow.utils.time import is_lease_expired from deerflow.utils.time import now_iso as _now_iso -from .schemas import DisconnectMode, RunStatus, ThreadOperationKind +from .schemas import EVIDENCE_ORIGINS, DisconnectMode, RunStatus, ThreadOperationKind from .store.base import ( EditReplayVisibility, RunIdempotencyConflict, @@ -267,6 +267,8 @@ class RunRecord: # True only on the caller that recovered an existing idempotent admission; # that caller must not attach a second worker to the durable run. idempotency_reused: bool = False + evidence_origin: str = "unknown" + evidence_agent_id: str | None = None class RunStartOutcome(StrEnum): @@ -366,6 +368,10 @@ class RunManager: "idempotency_key": record.idempotency_key, "goal_verdict": record.goal_verdict, } + if record.evidence_origin != "unknown": + payload["evidence_origin"] = record.evidence_origin + if record.evidence_agent_id is not None: + payload["evidence_agent_id"] = record.evidence_agent_id if record.idempotency_request is not None: payload["idempotency_request"] = record.idempotency_request if record.user_id is not None: @@ -543,6 +549,8 @@ class RunManager: last_ai_message=row.get("last_ai_message"), first_human_message=row.get("first_human_message"), owner_worker_id=row.get("owner_worker_id"), + evidence_origin=row.get("evidence_origin") or "unknown", + evidence_agent_id=row.get("evidence_agent_id"), lease_expires_at=row.get("lease_expires_at"), stop_reason=row.get("stop_reason"), goal_verdict=row.get("goal_verdict"), @@ -656,6 +664,34 @@ class RunManager: except Exception: logger.warning("Failed to persist finalizing progress for %s", run_id, exc_info=True) + @staticmethod + def _validate_evidence_origin(origin: str) -> None: + if origin not in EVIDENCE_ORIGINS: + raise ValueError("invalid evidence origin") + + async def seal_completed_evidence(self, run_id: str, receipt) -> bool: + """Best-effort evidence metadata is independent of business completion.""" + async with self._lock: + record = self._runs.get(run_id) + if record is None or record.ownership_lost: + return False + owner_worker_id = record.owner_worker_id + seal = getattr(self._store, "seal_evidence", None) + if seal is None or owner_worker_id is None: + return False + return await seal(run_id, owner_worker_id=owner_worker_id, receipt=receipt) + + async def mark_evidence_partial(self, run_id: str, *, error: str) -> bool: + async with self._lock: + record = self._runs.get(run_id) + if record is None or record.ownership_lost: + return False + owner_worker_id = record.owner_worker_id + mark_partial = getattr(self._store, "mark_evidence_partial", None) + if mark_partial is None or owner_worker_id is None: + return False + return await mark_partial(run_id, owner_worker_id=owner_worker_id, error=error) + async def create( self, thread_id: str, @@ -666,6 +702,8 @@ class RunManager: kwargs: dict | None = None, multitask_strategy: str = "reject", user_id: str | None = None, + evidence_origin: str = "unknown", + evidence_agent_id: str | None = None, ) -> RunRecord: """Create a new pending run and register it. @@ -678,6 +716,7 @@ class RunManager: """ from deerflow.runtime.assembly_evidence import EVIDENCE_KEY + self._validate_evidence_origin(evidence_origin) run_id = str(uuid.uuid4()) now = _now_iso() user_id = _resolve_record_user_id(user_id) @@ -696,6 +735,8 @@ class RunManager: updated_at=now, owner_worker_id=self._worker_id, lease_expires_at=lease_expires_at, + evidence_origin=evidence_origin, + evidence_agent_id=evidence_agent_id, ) async with self._lock: self._runs[run_id] = record @@ -1613,6 +1654,8 @@ class RunManager: model_name: str | None = None, user_id: str | None = None, idempotency_key: str | None = None, + evidence_origin: str = "unknown", + evidence_agent_id: str | None = None, idempotency_request: dict[str, Any] | None = None, ) -> RunRecord: """Atomically admit a normal agent run for a thread.""" @@ -1627,6 +1670,8 @@ class RunManager: model_name=model_name, user_id=user_id, idempotency_key=idempotency_key, + evidence_origin=evidence_origin, + evidence_agent_id=evidence_agent_id, idempotency_request=idempotency_request, ) @@ -1688,6 +1733,8 @@ class RunManager: model_name: str | None = None, user_id: str | None = None, idempotency_key: str | None = None, + evidence_origin: str = "unknown", + evidence_agent_id: str | None = None, idempotency_request: dict[str, Any] | None = None, ) -> RunRecord: """Atomically check for inflight runs and create a new one. @@ -1706,6 +1753,7 @@ class RunManager: """ from deerflow.runtime.assembly_evidence import EVIDENCE_KEY + self._validate_evidence_origin(evidence_origin) run_id = str(uuid.uuid4()) now = _now_iso() # Resolve before the idempotency checks below compare it with stored rows. @@ -1736,6 +1784,8 @@ class RunManager: owner_worker_id=self._worker_id, lease_expires_at=lease_expires_at, idempotency_key=idempotency_key, + evidence_origin=evidence_origin, + evidence_agent_id=evidence_agent_id, idempotency_request=idempotency_request, ) @@ -1809,6 +1859,10 @@ class RunManager: } if idempotency_key is not None: create_kwargs["idempotency_key"] = idempotency_key + if evidence_origin != "unknown": + create_kwargs["evidence_origin"] = evidence_origin + if evidence_agent_id is not None: + create_kwargs["evidence_agent_id"] = evidence_agent_id if idempotency_request is not None: create_kwargs["idempotency_request"] = idempotency_request try: @@ -1843,6 +1897,10 @@ class RunManager: } if idempotency_key is not None: create_kwargs["idempotency_key"] = idempotency_key + if evidence_origin != "unknown": + create_kwargs["evidence_origin"] = evidence_origin + if evidence_agent_id is not None: + create_kwargs["evidence_agent_id"] = evidence_agent_id if idempotency_request is not None: create_kwargs["idempotency_request"] = idempotency_request # Interrupt / rollback: store-side claim + insert in one @@ -2005,6 +2063,24 @@ class RunManager: if self._store is None: return [] grace_seconds = self._run_ownership_config.grace_seconds if self._run_ownership_config else 10 + recover_evidence = getattr(self._store, "recover_unsealed_evidence", None) + if callable(recover_evidence): + try: + evidence_before = before or _now_iso() + while True: + async with self._lock: + local_finalizing = tuple(r.run_id for r in self._runs.values() if not r.ownership_lost and (r.finalizing or (r.task is not None and not r.task.done()))) + recovered_count = await self._call_store_with_retry( + "recover_unsealed_evidence", + "*", + lambda: recover_evidence(before=evidence_before, grace_seconds=grace_seconds, exclude_run_ids=local_finalizing), + ) + # Drain at startup too: single-worker mode has no heartbeat. + if not recovered_count: + break + except Exception: + # Evidence availability must not prevent normal run recovery. + logger.warning("Failed to recover orphaned completed-run evidence", exc_info=True) try: rows = await self._call_store_with_retry( "list_inflight_with_expired_lease", diff --git a/backend/packages/harness/deerflow/runtime/runs/schemas.py b/backend/packages/harness/deerflow/runtime/runs/schemas.py index b5bb31d546..eaf19427f1 100644 --- a/backend/packages/harness/deerflow/runtime/runs/schemas.py +++ b/backend/packages/harness/deerflow/runtime/runs/schemas.py @@ -1,7 +1,10 @@ -"""Run status and disconnect mode enums.""" +"""Run status, disconnect modes and host-owned evidence provenance.""" from enum import StrEnum +# Shared by Gateway admission and RunManager; values remain wire-format strings. +EVIDENCE_ORIGINS = frozenset({"interactive", "scheduled", "extension_evaluation", "unknown"}) + class ThreadOperationKind(StrEnum): """Kind of operation holding exclusive admission for a thread.""" diff --git a/backend/packages/harness/deerflow/runtime/runs/store/base.py b/backend/packages/harness/deerflow/runtime/runs/store/base.py index 399d1623f4..15e7b54c77 100644 --- a/backend/packages/harness/deerflow/runtime/runs/store/base.py +++ b/backend/packages/harness/deerflow/runtime/runs/store/base.py @@ -159,6 +159,8 @@ class RunStore(abc.ABC): owner_worker_id: str | None = None, lease_expires_at: str | None = None, idempotency_key: str | None = None, + evidence_origin: str = "unknown", + evidence_agent_id: str | None = None, idempotency_request: dict[str, Any] | None = None, ) -> None: pass @@ -452,6 +454,8 @@ class RunStore(abc.ABC): created_at: str | None = None, grace_seconds: int = 10, idempotency_key: str | None = None, + evidence_origin: str = "unknown", + evidence_agent_id: str | None = None, idempotency_request: dict[str, Any] | None = None, ) -> tuple[dict[str, Any], list[dict[str, Any]]]: """Atomically create an active thread operation with cross-process uniqueness. diff --git a/backend/packages/harness/deerflow/runtime/runs/store/memory.py b/backend/packages/harness/deerflow/runtime/runs/store/memory.py index 68da3b1347..700c457789 100644 --- a/backend/packages/harness/deerflow/runtime/runs/store/memory.py +++ b/backend/packages/harness/deerflow/runtime/runs/store/memory.py @@ -69,6 +69,8 @@ class MemoryRunStore(RunStore): owner_worker_id=None, lease_expires_at=None, idempotency_key=None, + evidence_origin="unknown", + evidence_agent_id=None, idempotency_request=None, ): now = datetime.now(UTC).isoformat() @@ -77,6 +79,8 @@ class MemoryRunStore(RunStore): "run_id": run_id, "thread_id": thread_id, "assistant_id": assistant_id, + "evidence_origin": evidence_origin, + "evidence_agent_id": evidence_agent_id, "user_id": user_id, "model_name": model_name, "status": status, @@ -505,6 +509,8 @@ class MemoryRunStore(RunStore): created_at: str | None = None, grace_seconds: int = 10, idempotency_key: str | None = None, + evidence_origin: str = "unknown", + evidence_agent_id: str | None = None, idempotency_request: dict[str, Any] | None = None, ) -> tuple[dict[str, Any], list[dict[str, Any]]]: from deerflow.runtime.runs.manager import ConflictError @@ -583,6 +589,8 @@ class MemoryRunStore(RunStore): "run_id": run_id, "thread_id": thread_id, "assistant_id": assistant_id, + "evidence_origin": evidence_origin, + "evidence_agent_id": evidence_agent_id, "user_id": user_id, "model_name": model_name, "status": "pending", diff --git a/backend/packages/harness/deerflow/runtime/runs/worker.py b/backend/packages/harness/deerflow/runtime/runs/worker.py index 87f2ead9bf..e4f6cf2016 100644 --- a/backend/packages/harness/deerflow/runtime/runs/worker.py +++ b/backend/packages/harness/deerflow/runtime/runs/worker.py @@ -988,6 +988,7 @@ async def run_agent( accessor: CheckpointStateAccessor | None = None rollback_point: RollbackPoint | None = None journal = None + seq_stamper: _MessageSeqStamper | None = None runtime_ctx: dict[str, Any] | None = None runtime: Any | None = None agent: Any | None = None @@ -1004,6 +1005,7 @@ async def run_agent( # checkpoint failures / cancellation while waiting did not write an empty # completion snapshot into RunStore. persist_completion = False + evidence_execution_started = False completion_data: dict[str, Any] | None = None # Buffers subagent step events for batched persistence (#3779); assigned once # streaming starts and flushed in the finally block. Pre-bound to None so the @@ -1072,6 +1074,7 @@ async def run_agent( thread_id=thread_id, event_store=event_store, track_token_usage=getattr(run_events_config, "track_token_usage", True), + user_id=record.user_id, progress_reporter=lambda snapshot: run_manager.update_run_progress(run_id, **snapshot), ) @@ -1567,6 +1570,7 @@ async def run_agent( if stand_down_reason.startswith("thread_changed"): final_goal_verdict.update(satisfied=False, blocker="missing_evidence", relied_on_assumption=False) + evidence_execution_started = True await _stream_once(graph_input, initial_runnable_config) while not record.abort_event.is_set() and not llm_error_fallback_message and (journal is None or not journal.had_llm_error_fallback): continuation_input = await _prepare_goal_continuation_input( @@ -1691,6 +1695,7 @@ async def run_agent( async def finalize() -> None: nonlocal completion_hook_task + nonlocal journal, seq_stamper nonlocal accessor, agent, agent_result, cancel_action, checkpoint_rollback_completed nonlocal completion_data, deferred_finalization_interrupt, delivery_content, goal_completion, goal_evaluator_model nonlocal graph_input, pre_run_workspace_snapshot, produced_output_paths, rollback_point, runtime @@ -1976,6 +1981,29 @@ async def run_agent( "Extension task-stop notification interrupted for run %s; completing cleanup first", run_id, ) + if not record.ownership_lost and journal is not None: + evidence_failure = None + try: + evidence_receipt = await journal.finish_evidence() + if evidence_receipt is not None: + if not evidence_execution_started: + evidence_failure = "Lead graph execution did not begin; producer coverage is partial." + elif not await run_manager.seal_completed_evidence(run_id, evidence_receipt): + evidence_failure = "The owning worker could not persist the journal seal." + except BaseException as exc: + evidence_failure = f"Journal seal failed ({type(exc).__name__})." + if not isinstance(exc, Exception): + deferred_finalization_interrupt = _defer_finalization_interrupt(deferred_finalization_interrupt, exc) + logger.warning("Failed to seal completed-run evidence for %s (non-fatal)", run_id, exc_info=True) + if evidence_failure is not None: + try: + await run_manager.mark_evidence_partial(run_id, error=evidence_failure) + except Exception: + logger.warning("Failed to persist partial evidence for %s (non-fatal)", run_id, exc_info=True) + except BaseException as exc: + deferred_finalization_interrupt = _defer_finalization_interrupt(deferred_finalization_interrupt, exc) + logger.warning("Partial evidence persistence interrupted for %s; completing cleanup first", run_id) + if record.finalizing: await run_manager.set_finalizing(run_id, False) @@ -2015,6 +2043,10 @@ async def run_agent( ) # Drop graph and per-run payload references before the terminal # worker task itself becomes collectable. + # Diagnostic tracebacks can retain this frame. Release both + # the journal and the stamper callback that closes over it. + seq_stamper = None + journal = None agent = None agent_result = None accessor = None diff --git a/backend/packages/harness/deerflow/skills/AGENTS.md b/backend/packages/harness/deerflow/skills/AGENTS.md index ce3f11478c..2ff5239a86 100644 --- a/backend/packages/harness/deerflow/skills/AGENTS.md +++ b/backend/packages/harness/deerflow/skills/AGENTS.md @@ -54,3 +54,28 @@ Lets a caller pass per-request, short-lived end-user credentials (e.g. an ERP to - Build archives from captured bytes and require the preview's revision. Preserve file contents, empty directories, and normalized executable flags; import must not restore privileged permission bits. - Keep traversal, YAML parsing, and archive construction bounded and cancellable. Reject unsupported filesystem operations rather than following links; report limits instead of truncating results. Preserve YAML preflight before object construction. - Export includes raw saved files and is not a secret audit. Return relative paths and generic errors without leaking source content. See the [export API contract](../../../../docs/API.md#export-a-custom-skill) for response fields and limits. + +### Evolution host checks and recovery + +`mutations/service.py` shares one shielded check per proposal. A cancelled waiter +does not cancel the shared scan; a cancelled scan or timeout must propagate and +drain token-fenced lease cleanup, leaving no cached verdict. Keep the scan attempt +for hourly quota accounting. Cleanup must not erase a replacement worker's lease. +Single-operation recovery propagates an unresolved owner barrier as `NEEDS_REPAIR` +(HTTP 409); only the host-wide recovery sweep suppresses it to continue with other +owners. Publication and derived-view status remain separate. +`tests/blocking_io/test_skill_mutation_host.py` exercises the real host lifetime, +evidence pages, scanner, publication, revert and admin routes under Blockbuster; +only remote moderation is stubbed. Keep new async paths inside that gate. + +Standalone SDK/storage processes have no publication runtime. `mutations/enrollment.py` checks the configured application DB on each access and refuses enrolled owners with `MUTATION_RUNTIME_REQUIRED`; global toggles check all persisted owners. Never cache negative enrollment or infer it from the plugin list. Reads and writes must keep these guards, and the local storage factory must retain its AppConfig so the probe uses the same DB as the Gateway. `tests/test_skill_sdk_enrollment.py` exercises fresh real SDK processes without injected runtime state. + +- `get_enabled_skills_for_config` preserves user scope even when `app_config` is omitted; only calls without both config and user use the nonblocking global cache. Enrolled owners always bypass the local catalog LRU and pass the durable readiness guard. + +Owned custom-skill YAML diagnostics use the same managed-read guard as catalog +reads: recover pending publications before scanning, and refuse standalone reads +for enrolled owners without a mutation runtime. Keep the scan off the event loop. + +Projection readers and mutation writers share the reentrant owner lock. Both use +`utils.file_lock.retry_until_locked` for contention-only OS lock retries; preserve +deadlines, cancellation checks and the shared descriptor for nested reads. diff --git a/backend/packages/harness/deerflow/skills/export.py b/backend/packages/harness/deerflow/skills/export.py index 5b496c5306..6237f24bbd 100644 --- a/backend/packages/harness/deerflow/skills/export.py +++ b/backend/packages/harness/deerflow/skills/export.py @@ -366,12 +366,14 @@ def _open_directory_chain(path, budget): def _capture(storage, name, snapshot, budget): + from deerflow.skills.mutations.guard import managed_read + if not isinstance(name, str) or len(name) > 64 or not _NAME.fullmatch(name): raise SkillExportError(422, "skill_export_unsupported", "Invalid skill name.") root = storage.get_custom_skill_dir(name) budget.check() try: - with skill_projection_read_lock(storage, timeout=LOCK_TIMEOUT_SECONDS, check=budget.check): + with skill_projection_read_lock(storage, timeout=LOCK_TIMEOUT_SECONDS, check=budget.check), managed_read(storage): if not hasattr(os, "O_NOFOLLOW") or os.open not in os.supports_dir_fd or os.scandir not in os.supports_fd: raise SkillExportError(422, "skill_export_unsupported", "This platform cannot safely capture skill files.") parent_fd = _open_directory_chain(root.parent, budget) diff --git a/backend/packages/harness/deerflow/skills/mutations/__init__.py b/backend/packages/harness/deerflow/skills/mutations/__init__.py new file mode 100644 index 0000000000..d9714bc838 --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/__init__.py @@ -0,0 +1 @@ +"""Host-managed conditional updates of existing user custom skills.""" diff --git a/backend/packages/harness/deerflow/skills/mutations/assets.py b/backend/packages/harness/deerflow/skills/mutations/assets.py new file mode 100644 index 0000000000..e01ee04ed5 --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/assets.py @@ -0,0 +1,148 @@ +"""Bounded byte snapshots for managed custom packages (POSIX, no links).""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +from dataclasses import dataclass +from pathlib import Path + +from deerflow.skills.mutations.validation import MAX_MAIN_BYTES + +MAX_FILES = 256 +MAX_PACKAGE_BYTES = 16 * 1024 * 1024 +MAX_DEPTH = 24 + + +@dataclass(frozen=True) +class PackageFile: + path: str + content: bytes + executable: bool + + @property + def digest(self) -> str: + return hashlib.sha256(self.content).hexdigest() + + +@dataclass(frozen=True) +class PackageSnapshot: + files: tuple[PackageFile, ...] + + @property + def main_content(self) -> str: + return next(item.content for item in self.files if item.path == "SKILL.md").decode("utf-8") + + @property + def digest(self) -> str: + manifest = [(item.path, item.digest, item.executable) for item in self.files] + return hashlib.sha256(json.dumps(manifest, ensure_ascii=False, separators=(",", ":")).encode("utf-8")).hexdigest() + + def with_main(self, content: str) -> PackageSnapshot: + return PackageSnapshot(tuple(PackageFile(item.path, content.encode("utf-8"), item.executable) if item.path == "SKILL.md" else item for item in self.files)) + + +def capture_package(root: Path) -> PackageSnapshot: + """Caller owns mutation guard. Never read outside the opened package tree.""" + if os.name != "posix": + raise ValueError("UNSUPPORTED_ASSET") + files: list[PackageFile] = [] + total = 0 + entries = 0 + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + + def walk(directory_fd: int, prefix: str = "", depth: int = 0) -> None: + nonlocal total, entries + if depth > MAX_DEPTH: + raise ValueError("QUOTA_EXCEEDED") + # Bound directory enumeration, including empty directories, before sort. + names = [] + with os.scandir(directory_fd) as iterator: + for entry in iterator: + entries += 1 + if entries > MAX_FILES * 2: + raise ValueError("QUOTA_EXCEEDED") + names.append(entry.name) + for name in sorted(names): + if "\\" in name or any(ord(c) < 32 for c in name): + raise ValueError("UNSUPPORTED_ASSET") + path = f"{prefix}/{name}" if prefix else name + if len(path.encode("utf-8")) > 1024: + raise ValueError("QUOTA_EXCEEDED") + info = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + if not stat.S_ISDIR(info.st_mode) and not stat.S_ISREG(info.st_mode): + raise ValueError("UNSUPPORTED_ASSET") + fd = os.open(name, flags | (os.O_DIRECTORY if stat.S_ISDIR(info.st_mode) else 0), dir_fd=directory_fd) + try: + opened = os.fstat(fd) + if (info.st_dev, info.st_ino, info.st_mode) != (opened.st_dev, opened.st_ino, opened.st_mode): + raise ValueError("REVISION_CONFLICT") + if stat.S_ISDIR(opened.st_mode): + walk(fd, path, depth + 1) + continue + if len(files) >= MAX_FILES: + raise ValueError("QUOTA_EXCEEDED") + limit = min(MAX_PACKAGE_BYTES - total, MAX_MAIN_BYTES if path == "SKILL.md" else MAX_PACKAGE_BYTES) + if opened.st_size > limit: + raise ValueError("QUOTA_EXCEEDED") + data = bytearray() + while True: + chunk = os.read(fd, min(65536, limit + 1 - len(data))) + if not chunk: + break + data.extend(chunk) + if len(data) > limit: + raise ValueError("QUOTA_EXCEEDED") + after = os.fstat(fd) + if (opened.st_size, opened.st_mtime_ns, opened.st_ctime_ns) != (after.st_size, after.st_mtime_ns, after.st_ctime_ns): + raise ValueError("REVISION_CONFLICT") + total += len(data) + files.append(PackageFile(path, bytes(data), bool(opened.st_mode & 0o111))) + finally: + os.close(fd) + + try: + root_fd = open_directory_chain(root) + try: + walk(root_fd) + finally: + os.close(root_fd) + except OSError as exc: + raise ValueError("UNSUPPORTED_ASSET") from exc + result = PackageSnapshot(tuple(sorted(files, key=lambda item: item.path))) + try: + result.main_content + except (StopIteration, UnicodeError) as exc: + raise ValueError("UNSUPPORTED_ASSET") from exc + return result + + +def open_directory_chain(path: Path) -> int: + """Return an anchored directory FD; reject links in every ancestor. + + Publication also uses this helper: validating only the package's final + component would let a linked owner/custom root escape the owner guard. + Caller owns the returned descriptor. + """ + path = path.absolute() + if ".." in path.parts or len(path.parts) > 128 or len(os.fsencode(path)) > 4096: + raise ValueError("UNSUPPORTED_ASSET") + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_NONBLOCK + fd = os.open(path.anchor, flags) + try: + for part in path.parts[1:]: + before = os.stat(part, dir_fd=fd, follow_symlinks=False) + if not stat.S_ISDIR(before.st_mode): + raise ValueError("UNSUPPORTED_ASSET") + child = os.open(part, flags, dir_fd=fd) + os.close(fd) + fd = child + after = os.fstat(fd) + if (before.st_dev, before.st_ino, before.st_mode) != (after.st_dev, after.st_ino, after.st_mode): + raise ValueError("REVISION_CONFLICT") + return fd + except BaseException: + os.close(fd) + raise diff --git a/backend/packages/harness/deerflow/skills/mutations/codec.py b/backend/packages/harness/deerflow/skills/mutations/codec.py new file mode 100644 index 0000000000..af770e2600 --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/codec.py @@ -0,0 +1,89 @@ +"""Detached database encodings. Blob storage never enters skill loading roots.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +from datetime import UTC, datetime + +from deerflow_extension_api.skill_mutations import AssetRevision, HostCheckResult, Operation, Proposal + +from deerflow.skills.mutations.assets import MAX_FILES, MAX_PACKAGE_BYTES, PackageFile, PackageSnapshot +from deerflow.skills.mutations.validation import MAX_MAIN_BYTES + + +def fingerprint(value) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode()).hexdigest() + + +def encode_package(package: PackageSnapshot) -> bytes: + return json.dumps([(item.path, base64.b64encode(item.content).decode("ascii"), item.executable) for item in package.files], separators=(",", ":")).encode() + + +def decode_package(blob: bytes) -> PackageSnapshot: + try: + if not isinstance(blob, bytes) or len(blob) > 24 * 1024 * 1024: + raise ValueError + rows = json.loads(blob) + if not isinstance(rows, list) or not 1 <= len(rows) <= MAX_FILES: + raise ValueError + files, total, previous = [], 0, "" + for item in rows: + if not isinstance(item, list) or len(item) != 3: + raise ValueError + path, content, executable = item + if not isinstance(path, str) or not path or len(path.encode()) > 1024 or path <= previous or "\\" in path or any(ord(c) < 32 for c in path) or any(part in {"", ".", ".."} for part in path.split("/")): + raise ValueError + if type(executable) is not bool or not isinstance(content, str): + raise ValueError + data = base64.b64decode(content, validate=True) + total += len(data) + if total > MAX_PACKAGE_BYTES or (path == "SKILL.md" and len(data) > MAX_MAIN_BYTES): + raise ValueError + files.append(PackageFile(path, data, executable)) + previous = path + result = PackageSnapshot(tuple(files)) + result.main_content + return result + except (ValueError, TypeError, StopIteration, UnicodeError, RecursionError) as exc: + raise ValueError("BLOB_INTEGRITY_ERROR") from exc + + +def operation_packages(row): + before, after = decode_package(row.before_blob), decode_package(row.after_blob) + if before.digest != row.before_revision.get("content_digest") or after.digest != row.after_revision.get("content_digest") or before.digest == after.digest: + raise ValueError("BLOB_INTEGRITY_ERROR") + return before, after + + +def timestamp(value: float) -> str: + return datetime.fromtimestamp(value, UTC).isoformat() + + +def proposal_view(row) -> Proposal: + return Proposal( + row.proposal_id, row.owner_id, row.target_ref, row.name, AssetRevision(**row.base_revision), row.candidate_hash, row.state, timestamp(row.expires_at), tuple(source["snapshot_ref"] for source in row.sources), row.operation_id + ) + + +def operation_view(row) -> Operation: + return Operation( + row.operation_id, + row.owner_id, + row.name, + row.publication, + row.views, + AssetRevision(**row.before_revision), + AssetRevision(**row.after_revision), + row.generation, + row.proposal_id, + row.reverts_operation_id, + row.error_code, + row.superseded_by_generation, + ) + + +def check_view(row) -> HostCheckResult: + result = row.check_result + return HostCheckResult(row.proposal_id, result["decision"], row.candidate_hash, AssetRevision(**row.base_revision), result["policy_version"], timestamp(result["expires_at"]), result["reason_code"]) diff --git a/backend/packages/harness/deerflow/skills/mutations/enrollment.py b/backend/packages/harness/deerflow/skills/mutations/enrollment.py new file mode 100644 index 0000000000..300b82dcec --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/enrollment.py @@ -0,0 +1,65 @@ +"""Fail closed when an embedded process lacks the enrolled owner's runtime. + +Enrollment belongs to the application database, not the current plugin list or +process. Do not cache negative lookups: a Gateway can enroll an owner after an +SDK client has already been constructed. Probes never create/migrate a database. +""" + +from __future__ import annotations + +from pathlib import Path +from urllib.parse import quote + +from deerflow_extension_api.host_capabilities import HostCapabilityError +from sqlalchemy import create_engine, inspect, select, union_all +from sqlalchemy.engine import make_url +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.pool import NullPool + +from deerflow.persistence.skill_mutations.model import SkillAssetRow, SkillOwnerRow + + +def require_unenrolled(storage, *, global_scope: bool = False) -> None: + config = getattr(storage, "_app_config", None) + database = getattr(config, "database", None) + if database is None or database.backend == "memory": + return + owner_id = getattr(storage, "user_id", None) + if owner_id is None and not global_scope: + return + engine = None + try: + url = make_url(database.app_sync_sqlalchemy_url) + if database.backend == "sqlite": + path = Path(database.sqlite_path) + if not path.exists(): + return + # URI read-only mode closes the exists/connect creation race and + # avoids journal/schema changes in an uninitialized SDK process. + url = url.set(database="file:" + quote(str(path), safe="/"), query={"mode": "ro", "uri": "true"}) + args = {"timeout": 5} + else: + from deerflow.persistence.postgres_schema import build_psycopg_options + + options = build_psycopg_options(database.postgres_schema) or "" + args = {"connect_timeout": 5, "options": f"{options} -c lock_timeout=5000 -c statement_timeout=5000".strip()} + # A short-lived connection has no process-wide SDK teardown requirement + # and cannot retain an enrollment snapshot across later skill calls. + engine = create_engine(url, connect_args=args, poolclass=NullPool) + with engine.connect() as connection: + schema = inspect(connection) + queries = [] + for model in (SkillOwnerRow, SkillAssetRow): + if schema.has_table(model.__tablename__): + query = select(model.owner_id) + queries.append(query if global_scope else query.where(model.owner_id == owner_id)) + enrolled = bool(queries) and connection.execute(union_all(*queries).limit(1)).first() is not None + if enrolled: + raise HostCapabilityError("MUTATION_RUNTIME_REQUIRED", "Use the Gateway for enrolled skill access; this process has no durable mutation runtime") + except (SQLAlchemyError, OSError) as exc: + # Unavailable enrollment is not proof that a write is unmanaged. Do not + # expose a DB URL or driver diagnostics in the public exception. + raise HostCapabilityError("UNAVAILABLE", "Cannot verify durable skill enrollment") from exc + finally: + if engine is not None: + engine.dispose() diff --git a/backend/packages/harness/deerflow/skills/mutations/guard.py b/backend/packages/harness/deerflow/skills/mutations/guard.py new file mode 100644 index 0000000000..d58aa8c823 --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/guard.py @@ -0,0 +1,186 @@ +"""One host-owned coordination point for managed storage readers and writers.""" + +from __future__ import annotations + +import hashlib +from contextlib import ExitStack, contextmanager, nullcontext + +from deerflow_extension_api.host_capabilities import HostCapabilityError + +from deerflow.skills.mutations.assets import capture_package +from deerflow.skills.mutations.enrollment import require_unenrolled +from deerflow.skills.mutations.repository import revision + +_runtime: SkillMutationRuntime | None = None + + +def configure_mutation_runtime(runtime: SkillMutationRuntime | None) -> None: + """Gateway startup/shutdown only. Recovery remains independent of plugins.""" + global _runtime + _runtime = runtime + + +def runtime_for(storage) -> SkillMutationRuntime | None: + runtime = _runtime + if runtime is None: + require_unenrolled(storage) + return runtime if runtime is not None and getattr(storage, "user_id", None) in runtime.owners else None + + +def owner_is_managed(owner_id: str | None) -> bool: + return _runtime is not None and owner_id in _runtime.owners + + +@contextmanager +def managed_read(storage): + runtime = runtime_for(storage) + if runtime is None: + yield + return + from deerflow.skills.projection import skill_projection_read_lock + + with skill_projection_read_lock(storage): + runtime.ensure_readable(storage) + yield + + +def managed_global_read(storage): + """Only enrolled owners need global -> owner ordering around view rebuilds.""" + if runtime_for(storage) is None: + return nullcontext() + from deerflow.skills.projection import _projection_lock, get_skill_projection_paths + + return _projection_lock(get_skill_projection_paths(storage).public.parent, timeout=5.0) + + +def read_optional_text(storage, path): + """Canonical reads for existing edit/rollback audit paths, inside readiness.""" + from deerflow.skills.storage.skill_storage import read_text_or_none + + with managed_read(storage): + return read_text_or_none(path) if path.exists() else None + + +@contextmanager +def managed_name_writes(storage, names, *, global_scope=False, timeout=None): + """Serialize one Skill name across public state and enrolled owner writes.""" + runtime = _runtime + owner_id = getattr(storage, "user_id", None) + if runtime is None: + require_unenrolled(storage, global_scope=global_scope) + if runtime is None or not names or (not global_scope and owner_id not in runtime.owners): + yield + return + from deerflow.skills.projection import _projection_lock, get_skill_projection_paths + + lock_root = get_skill_projection_paths(storage).public.parent / ".skill-mutation-names" + with ExitStack() as stack: + for name in sorted(set(names)): + token = hashlib.sha256(name.encode("utf-8")).hexdigest() + stack.enter_context(_projection_lock(lock_root / token, timeout=timeout)) + yield + + +@contextmanager +def managed_global_state_write(storage, name): + """A global same-name enable toggle also changes each user's custom asset. + + Acquire before config locks: name fence -> global projection -> sorted owner guards -> DB. + A name fence prevents concurrent same-name creation. Only owners that have + the affected custom asset are then reserved before changing the shared flag. + """ + runtime = _runtime + if runtime is None: + require_unenrolled(storage, global_scope=True) + yield + return + from deerflow.skills.projection import _projection_lock, get_skill_projection_paths, skill_projection_read_lock + from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage + + with managed_name_writes(storage, (name,), global_scope=True, timeout=5.0), _projection_lock(get_skill_projection_paths(storage).public.parent, timeout=5.0), ExitStack() as stack: + storages = [] + for owner in sorted(runtime.owners): + scoped = UserScopedSkillStorage(owner, host_path=str(storage.get_skills_root_path())) + if not scoped.get_custom_skill_file(name).exists(): + continue + stack.enter_context(skill_projection_read_lock(scoped)) + runtime.ensure_readable(scoped) + if scoped.get_custom_skill_file(name).exists(): + storages.append(scoped) + for scoped in storages: + stack.enter_context(runtime.writer(scoped, (name,))) + yield + + +class SkillMutationRuntime: + def __init__(self, repository, *, owners: frozenset[str]): + self.repository = repository + self.owners = owners + self.recover_locked = None + + def ensure_readable(self, storage): + """Caller owns the owner guard; lazy recovery never rebuilds views.""" + try: + self.repository.check_readable(storage.user_id) + except HostCapabilityError as exc: + if exc.code != "NEEDS_REPAIR" or self.recover_locked is None: + raise + self.recover_locked(storage) + self.repository.check_readable(storage.user_id) + + @staticmethod + def state(storage, name): + root = storage.get_custom_skill_dir(name) + exists = root.exists() + if not exists: + return "", False, False + try: + digest = capture_package(root).digest + except ValueError: + # Old APIs retain their broader asset support. Such assets cannot + # enter the new automatic API; each legacy write invalidates CAS. + import uuid + + digest = "unsupported-" + uuid.uuid4().hex + from deerflow.config.extensions_config import ExtensionsConfig + + enabled = storage.get_skill_enabled_state(name) and ExtensionsConfig.from_file().is_skill_enabled(name, "custom") + return digest, enabled, True + + def read_revision(self, storage, name): + from deerflow.skills.projection import skill_projection_read_lock + + with skill_projection_read_lock(storage): + digest, enabled, exists = self.state(storage, name) + row = self.repository.observe(storage.user_id, name, digest, enabled, exists=exists) + if not exists: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN") + return revision(row) + + @contextmanager + def writer(self, storage, names): + # The caller already owns the projection/user guard, on one worker. + self.ensure_readable(storage) + for name in names: + digest, enabled, exists = self.state(storage, name) + self.repository.observe(storage.user_id, name, digest, enabled, exists=exists, mark_mutating=True) + try: + yield + except BaseException: + # Keep the marker: the next guarded access must conservatively + # advance the version even if a failed writer restored old bytes. + raise + else: + for name in names: + digest, enabled, exists = self.state(storage, name) + self.repository.finish_writer(storage.user_id, name, digest, enabled, exists=exists) + + +@contextmanager +def managed_writer(storage, names): + runtime = runtime_for(storage) + if runtime is None: + yield + else: + with runtime.writer(storage, names): + yield diff --git a/backend/packages/harness/deerflow/skills/mutations/history.py b/backend/packages/harness/deerflow/skills/mutations/history.py new file mode 100644 index 0000000000..8a7968bf44 --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/history.py @@ -0,0 +1,103 @@ +"""Bounded best-effort metadata mirror; the operation database is authoritative.""" + +import json +import os +import stat +from collections import defaultdict +from dataclasses import asdict + +from deerflow.skills.mutations.assets import open_directory_chain + +MAX_HISTORY_BYTES = 8 * 1024 * 1024 +MAX_HISTORY_LINE_BYTES = 64 * 1024 + + +def mirror_operations(storage, operations): + """Caller owns the owner guard. Never copy retained Skill body blobs here. + + Replay at most the latest 100 operations per owner. An oversized/malformed + history is left untouched, since bounded work cannot prove deduplication. + """ + grouped = defaultdict(list) + for operation in operations: + grouped[operation.name].append(operation) + if not grouped: + return + custom = open_directory_chain(storage.get_custom_skill_dir(next(iter(grouped))).parent) + try: + try: + os.mkdir(".history", 0o700, dir_fd=custom) + except FileExistsError: + pass + directory = os.open(".history", os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=custom) + finally: + os.close(custom) + try: + for name, items in grouped.items(): + storage.validate_skill_name(name) + fd = os.open(name + ".jsonl", os.O_RDWR | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600, dir_fd=directory) + try: + before = os.fstat(fd) + if not stat.S_ISREG(before.st_mode) or before.st_size > MAX_HISTORY_BYTES: + continue + seen, pending, total = set(), b"", 0 + valid = True + while chunk := os.read(fd, min(65536, MAX_HISTORY_BYTES + 1 - total)): + total += len(chunk) + if total > MAX_HISTORY_BYTES: + valid = False + break + lines = (pending + chunk).split(b"\n") + pending = lines.pop() + for line in lines: + if len(line) > MAX_HISTORY_LINE_BYTES: + valid = False + break + if not line.strip(): + continue + try: + record = json.loads(line) + if not isinstance(record, dict): + raise ValueError + identifier = record.get("operation_id") + if isinstance(identifier, str): + seen.add(identifier) + except (ValueError, UnicodeError): + valid = False + break + if not valid or len(pending) > MAX_HISTORY_LINE_BYTES: + valid = False + break + if not valid or pending.strip() or os.fstat(fd).st_size != before.st_size: + continue + records = [] + for operation in reversed(items): + if operation.operation_id in seen: + continue + records.append( + json.dumps( + { + "operation_id": operation.operation_id, + "action": "host_revert" if operation.reverts_operation_id else "host_commit", + "publication": operation.publication, + "prev_revision": asdict(operation.before_revision), + "new_revision": asdict(operation.after_revision), + "scope": "main-file-only", + "file_path": "SKILL.md", + }, + separators=(",", ":"), + ).encode() + + b"\n" + ) + if total + sum(map(len, records)) > MAX_HISTORY_BYTES: + continue + for record in records: + if os.write(fd, record) != len(record): + raise OSError("incomplete history mirror") + if records: + os.fsync(fd) + os.fsync(directory) + finally: + os.close(fd) + finally: + os.close(directory) diff --git a/backend/packages/harness/deerflow/skills/mutations/publication.py b/backend/packages/harness/deerflow/skills/mutations/publication.py new file mode 100644 index 0000000000..8191d5fb5a --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/publication.py @@ -0,0 +1,174 @@ +"""One-file durable publication. The PREPARED journal precedes every replace.""" + +from __future__ import annotations + +import os +import stat + +from deerflow_extension_api.host_capabilities import HostCapabilityError + +from deerflow.persistence.skill_mutations.model import SkillAssetRow, SkillOperationRow, SkillOwnerRow +from deerflow.skills.mutations.assets import capture_package, open_directory_chain +from deerflow.skills.mutations.codec import operation_packages +from deerflow.skills.mutations.repository import supersede_views + + +def replace_main(root, content: bytes, *, operation_id: str) -> None: + """Use the validated directory descriptor throughout; preserve mode bits.""" + directory = open_directory_chain(root) + parent = None + created = False + temporary = ".host-mutation-" + operation_id + try: + parent = open_directory_chain(root.parent) + original = os.stat("SKILL.md", dir_fd=directory, follow_symlinks=False) + if not stat.S_ISREG(original.st_mode): + raise HostCapabilityError("UNSUPPORTED_ASSET") + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=parent) + created = True + try: + offset = 0 + while offset < len(content): + written = os.write(fd, content[offset:]) + if written <= 0: + raise OSError("short publication write") + offset += written + # Preserve regular permissions, never propagate setuid/setgid/sticky. + os.fchmod(fd, stat.S_IMODE(original.st_mode) & 0o777) + os.fsync(fd) + finally: + os.close(fd) + os.replace(temporary, "SKILL.md", src_dir_fd=parent, dst_dir_fd=directory) + os.fsync(directory) + os.fsync(parent) + finally: + try: + if created: + try: + os.unlink(temporary, dir_fd=parent) + except FileNotFoundError: + pass + finally: + os.close(directory) + if parent is not None: + os.close(parent) + + +def assert_temporary_absent(root, operation_id): + parent = open_directory_chain(root.parent) + try: + try: + os.stat(".host-mutation-" + operation_id, dir_fd=parent, follow_symlinks=False) + except FileNotFoundError: + return + raise HostCapabilityError("UNSUPPORTED_ASSET") + finally: + os.close(parent) + + +def durable_current(root): + """Recovery also needs successful barriers before claiming a disk outcome.""" + directory = open_directory_chain(root) + try: + fd = os.open("SKILL.md", os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory) + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): + raise OSError("unsupported main file") + os.fsync(fd) + finally: + os.close(fd) + os.fsync(directory) + finally: + os.close(directory) + parent = open_directory_chain(root.parent) + try: + os.fsync(parent) + finally: + os.close(parent) + + +def cleanup_temporary(root, operation_id): + parent = open_directory_chain(root.parent) + try: + try: + os.unlink(".host-mutation-" + operation_id, dir_fd=parent) + os.fsync(parent) + except FileNotFoundError: + pass + finally: + os.close(parent) + + +def finalize(session, operation, publication: str) -> None: + """Apply reserved counters exactly once, while holding the owner guard.""" + asset = session.get(SkillAssetRow, operation.target_ref) + owner = session.get(SkillOwnerRow, operation.owner_id) + if asset is None or owner is None or asset.operation_id != operation.operation_id: + operation.publication = "NEEDS_REPAIR" + operation.error_code = "ASSET_IDENTITY_CHANGED" + return + operation.publication = publication + if publication == "APPLIED": + after = operation.after_revision + asset.incarnation_id, asset.mutation_seq, asset.content_digest = after["incarnation_id"], after["mutation_seq"], after["content_digest"] + asset.mutating = False + owner.generation = operation.generation + supersede_views(session, operation.owner_id, owner.generation) + operation.error_code = None + elif publication == "ABORTED": + asset.mutating = False + asset.operation_id = operation.before_operation_id + operation.views = "SUPERSEDED" + operation.superseded_by_generation = owner.generation + operation.error_code = "PUBLICATION_ABORTED" + else: + operation.error_code = "DISK_STATE_UNKNOWN" + + +def resolve_prepared(repository, storage, operation_id: str) -> None: + """Classify disk only. Recovery never replays a write.""" + with repository.sessions.begin() as session: + operation = session.get(SkillOperationRow, operation_id) + if operation is None or operation.publication not in {"PREPARED", "NEEDS_REPAIR"}: + return + try: + operation_packages(operation) + except ValueError: + operation.publication = "NEEDS_REPAIR" + operation.error_code = "BLOB_INTEGRITY_ERROR" + return + try: + disk = capture_package(storage.get_custom_skill_dir(operation.name)).digest + except (OSError, ValueError): + disk = None + before, after = operation.before_revision["content_digest"], operation.after_revision["content_digest"] + publication = "ABORTED" if disk == before else "APPLIED" if disk == after else "NEEDS_REPAIR" + if publication != "NEEDS_REPAIR": + try: + durable_current(storage.get_custom_skill_dir(operation.name)) + except (OSError, ValueError): + operation.error_code = "DURABILITY_UNCONFIRMED" + return + finalize(session, operation, publication) + if publication != "NEEDS_REPAIR": + cleanup_temporary(storage.get_custom_skill_dir(operation.name), operation_id) + + +def publish_prepared(repository, storage, operation_id: str) -> None: + with repository.sessions() as session: + operation = session.get(SkillOperationRow, operation_id) + try: + _, package = operation_packages(operation) + except ValueError: + resolve_prepared(repository, storage, operation_id) + return + name = operation.name + try: + replace_main(storage.get_custom_skill_dir(name), package.main_content.encode("utf-8"), operation_id=operation_id) + except Exception: + # A response failure after replace must never look like ordinary refusal. + # Journal classification distinguishes not-written, written and uncertain. + resolve_prepared(repository, storage, operation_id) + return + with repository.sessions.begin() as session: + finalize(session, session.get(SkillOperationRow, operation_id), "APPLIED") diff --git a/backend/packages/harness/deerflow/skills/mutations/recovery.py b/backend/packages/harness/deerflow/skills/mutations/recovery.py new file mode 100644 index 0000000000..8b3e287875 --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/recovery.py @@ -0,0 +1,170 @@ +"""Host-owned recovery, deletion fencing and retention independent of plugins.""" + +from __future__ import annotations + +import time + +from deerflow_extension_api.host_capabilities import HostCapabilityError +from sqlalchemy import and_, delete, exists, literal, or_, select, update + +from deerflow.persistence.skill_mutations.model import SkillAssetRow, SkillOperationRow, SkillOwnerRow, SkillProposalRow, SkillScanAttemptRow +from deerflow.skills.mutations.codec import operation_view +from deerflow.skills.mutations.publication import resolve_prepared +from deerflow.skills.mutations.repository import supersede_views +from deerflow.skills.projection import skill_projection_read_lock + + +class SkillMutationRecovery: + def __init__(self, runtime, storage_factory, *, rebuild_views=None): + self.runtime, self.storage_factory = runtime, storage_factory + self.repository = runtime.repository + if rebuild_views is None: + from deerflow.skills.projection import rebuild_skill_projections + + def rebuild_views(storage): + return rebuild_skill_projections(storage, include_public=False) + + self.rebuild_views = rebuild_views + runtime.recover_locked = self.recover_locked + + def get_operation(self, operation_id): + with self.repository.sessions() as session: + row = session.get(SkillOperationRow, operation_id) + if row is None: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN") + return operation_view(row) + + def list_operations(self, *, limit=50, after_id=None): + if type(limit) is not int or not 1 <= limit <= 101 or (after_id is not None and (not isinstance(after_id, str) or len(after_id) > 32)): + raise HostCapabilityError("INVALID_REQUEST") + with self.repository.sessions() as session: + cursor = session.get(SkillOperationRow, after_id) if after_id else None + if after_id and cursor is None: + return () + after = or_(SkillOperationRow.created_at > cursor.created_at, and_(SkillOperationRow.created_at == cursor.created_at, SkillOperationRow.operation_id > cursor.operation_id)) if cursor is not None else True + rows = session.scalars( + select(SkillOperationRow) + .where( + or_(SkillOperationRow.publication.in_(("PREPARED", "NEEDS_REPAIR")), (SkillOperationRow.publication == "APPLIED") & SkillOperationRow.views.in_(("PENDING", "ERROR"))), + after, + ) + .order_by(SkillOperationRow.created_at, SkillOperationRow.operation_id) + .limit(limit) + ).all() + return tuple(operation_view(row) for row in rows) + + def recover_locked(self, storage): + """Caller owns owner lock. No projection rebuild or plugin callbacks.""" + with self.repository.sessions() as session: + pending = session.scalars(select(SkillOperationRow.operation_id).where(SkillOperationRow.owner_id == storage.user_id, SkillOperationRow.publication.in_(("PREPARED", "NEEDS_REPAIR")))).all() + for operation_id in pending: + resolve_prepared(self.repository, storage, operation_id) + with self.repository.sessions() as session: + uncertain = session.scalars(select(SkillAssetRow.name).where(SkillAssetRow.owner_id == storage.user_id, SkillAssetRow.mutating.is_(True))).all() + for name in uncertain: + digest, enabled, exists = self.runtime.state(storage, name) + self.repository.observe(storage.user_id, name, digest, enabled, exists=exists) + + def refresh_views(self, owner_id): + storage = self.storage_factory(owner_id) + with skill_projection_read_lock(storage): + with self.repository.sessions.begin() as session: + owner = session.get(SkillOwnerRow, owner_id) + if owner is None or owner.deleting: + return + from deerflow.skills.mutations.history import mirror_operations + + recent = session.scalars( + select(SkillOperationRow).where(SkillOperationRow.owner_id == owner_id, SkillOperationRow.publication == "APPLIED").order_by(SkillOperationRow.created_at.desc(), SkillOperationRow.operation_id).limit(100) + ).all() + try: + mirror_operations(storage, [operation_view(row) for row in recent]) + except Exception: + # Mirror availability never changes publication/view status. + pass + generation = owner.generation + supersede_views(session, owner_id, generation) + operations = session.scalars(select(SkillOperationRow.operation_id).where(SkillOperationRow.owner_id == owner_id, SkillOperationRow.publication == "APPLIED", SkillOperationRow.views.in_(("PENDING", "ERROR")))).all() + if not operations: + return + try: + self.rebuild_views(storage) + state = "READY" + except Exception: + state = "ERROR" + with skill_projection_read_lock(storage), self.repository.sessions.begin() as session: + owner = session.get(SkillOwnerRow, owner_id) + for operation_id in operations: + row = session.get(SkillOperationRow, operation_id) + if row is not None: + row.views = state if owner and owner.generation == row.generation == generation else "SUPERSEDED" + if row.views == "SUPERSEDED" and owner: + row.superseded_by_generation = owner.generation + + def recover_owner(self, owner_id): + storage = self.storage_factory(owner_id) + with skill_projection_read_lock(storage): + self.recover_locked(storage) + self.repository.check_readable(owner_id) + self.refresh_views(owner_id) + + def recover_operation(self, operation_id): + operation = self.get_operation(operation_id) + self.recover_owner(operation.owner_id) + return self.get_operation(operation_id) + + def recover_all(self): + for owner_id in self._owners(): + try: + self.recover_owner(owner_id) + except (HostCapabilityError, OSError, TimeoutError): + # Owner readiness remains closed; admin recovery stays reachable. + continue + + def _owners(self): + after = "" + while True: + with self.repository.sessions() as session: + owners = session.scalars(select(SkillOwnerRow.owner_id).where(SkillOwnerRow.owner_id > after).order_by(SkillOwnerRow.owner_id).limit(100)).all() + if not owners: + return + yield from owners + after = owners[-1] + + def quiesce_owner(self, owner_id): + # Accepted workers own this same lock through finalization. Taking it + # drains them before admission is fenced. Deletion need not keep blobs. + storage = self.storage_factory(owner_id) + with skill_projection_read_lock(storage): + self.recover_locked(storage) + with self.repository.sessions.begin() as session: + owner = session.get(SkillOwnerRow, owner_id) + if owner is None: + owner = self.repository.owner(session, owner_id) + owner.deleting = True + + def collect_garbage(self): + now = time.time() + for owner_id in self._owners(): + try: + self._collect_owner_garbage(owner_id, now) + except (HostCapabilityError, OSError, TimeoutError): + # Keep inaccessible owners' blobs intact for a later retry, + # without blocking startup or cleanup for unrelated owners. + continue + + def _collect_owner_garbage(self, owner_id, now): + with skill_projection_read_lock(self.storage_factory(owner_id)), self.repository.sessions.begin() as session: + # Clear columns in SQL: retained idempotency history never causes + # unbounded blob hydration. Owner guard serializes new references. + live_operation = exists(select(SkillOperationRow.operation_id).where(SkillOperationRow.operation_id == SkillProposalRow.operation_id, SkillOperationRow.publication.in_(("PREPARED", "NEEDS_REPAIR")))) + live_scan = exists(select(SkillScanAttemptRow.token).where(SkillScanAttemptRow.subject == literal("check:") + SkillProposalRow.proposal_id, SkillScanAttemptRow.lease_until > now)) + eligible = (SkillProposalRow.owner_id == owner_id, ~live_operation, ~live_scan) + session.execute(update(SkillProposalRow).where(*eligible, or_(SkillProposalRow.expires_at <= now, SkillProposalRow.state.in_(("DISCARDED", "PUBLISHED")))).values(package_blob=None, candidate_blob=None)) + session.execute(update(SkillProposalRow).where(*eligible, SkillProposalRow.expires_at <= now, SkillProposalRow.state == "PENDING").values(state="EXPIRED")) + session.execute( + update(SkillOperationRow) + .where(SkillOperationRow.owner_id == owner_id, SkillOperationRow.rollback_expires_at <= now, SkillOperationRow.publication.not_in(("PREPARED", "NEEDS_REPAIR"))) + .values(before_blob=None, after_blob=None) + ) + session.execute(delete(SkillScanAttemptRow).where(SkillScanAttemptRow.owner_id == owner_id, SkillScanAttemptRow.started_at < now - 3600, SkillScanAttemptRow.lease_until <= now)) diff --git a/backend/packages/harness/deerflow/skills/mutations/repository.py b/backend/packages/harness/deerflow/skills/mutations/repository.py new file mode 100644 index 0000000000..8f97ac48aa --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/repository.py @@ -0,0 +1,127 @@ +"""Short synchronous transactions, always acquired after the owner file guard.""" + +from __future__ import annotations + +import uuid + +from deerflow_extension_api.host_capabilities import HostCapabilityError +from deerflow_extension_api.skill_mutations import AssetRevision +from sqlalchemy import select, update + +from deerflow.persistence.skill_mutations.model import SkillAssetRow, SkillOperationRow, SkillOwnerRow +from deerflow.persistence.user.model import UserRow + + +def revision(row: SkillAssetRow) -> AssetRevision: + return AssetRevision(row.incarnation_id, row.mutation_seq, row.content_digest) + + +def supersede_views(session, owner_id, generation): + session.execute( + update(SkillOperationRow) + .where(SkillOperationRow.owner_id == owner_id, SkillOperationRow.publication.in_(("APPLIED", "NO_CHANGE")), SkillOperationRow.generation < generation, SkillOperationRow.views != "SUPERSEDED") + .values(views="SUPERSEDED", superseded_by_generation=generation) + ) + + +class SkillMutationRepository: + def __init__(self, sessions): + self.sessions = sessions + + def owner(self, session, owner_id: str) -> SkillOwnerRow: + user = session.get(UserRow, owner_id) + if user is None: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN") + row = session.get(SkillOwnerRow, owner_id, with_for_update=True) + if row is None: + row = SkillOwnerRow(owner_id=owner_id, generation=0, deleting=False) + session.add(row) + session.flush() + if row.deleting: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN") + return row + + @staticmethod + def asset(session, owner_id: str, name: str) -> SkillAssetRow | None: + return session.scalar(select(SkillAssetRow).where(SkillAssetRow.owner_id == owner_id, SkillAssetRow.name == name)) + + def observe(self, owner_id: str, name: str, digest: str, enabled: bool, *, exists: bool, mark_mutating: bool = False) -> SkillAssetRow: + with self.sessions.begin() as session: + owner = self.owner(session, owner_id) + row = self.asset(session, owner_id, name) + if row is not None and row.operation_id is not None: + operation = session.get(SkillOperationRow, row.operation_id) + if operation is not None and operation.publication in {"PREPARED", "NEEDS_REPAIR"}: + raise HostCapabilityError("NEEDS_REPAIR") + if row is None: + row = SkillAssetRow( + target_ref=uuid.uuid4().hex, + owner_id=owner_id, + name=name, + incarnation_id=uuid.uuid4().hex, + mutation_seq=1, + content_digest=digest, + enabled=enabled, + deleted=not exists, + mutating=False, + ) + session.add(row) + owner.generation += 1 + elif row.mutating or (row.content_digest, row.enabled, row.deleted) != (digest, enabled, not exists): + if row.deleted and exists: + row.incarnation_id = uuid.uuid4().hex + row.mutation_seq += 1 + row.content_digest, row.enabled, row.deleted = digest, enabled, not exists + row.operation_id = None + owner.generation += 1 + row.mutating = mark_mutating + supersede_views(session, owner_id, owner.generation) + session.flush() + return row + + def finish_writer(self, owner_id: str, name: str, digest: str, enabled: bool, *, exists: bool) -> None: + with self.sessions.begin() as session: + owner = self.owner(session, owner_id) + row = self.asset(session, owner_id, name) + if row is None or not row.mutating: + raise HostCapabilityError("NEEDS_REPAIR") + if (row.content_digest, row.enabled, row.deleted) != (digest, enabled, not exists): + if row.deleted and exists: + row.incarnation_id = uuid.uuid4().hex + row.mutation_seq += 1 + row.content_digest, row.enabled, row.deleted = digest, enabled, not exists + row.operation_id = None + owner.generation += 1 + row.mutating = False + supersede_views(session, owner_id, owner.generation) + + def generation(self, owner_id: str) -> int: + with self.sessions() as session: + row = session.get(SkillOwnerRow, owner_id) + return row.generation if row else 0 + + def check_readable(self, owner_id: str) -> None: + with self.sessions() as session: + if session.get(UserRow, owner_id) is None: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN") + owner = session.get(SkillOwnerRow, owner_id) + if owner is not None and owner.deleting: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN") + blocked = session.scalar( + select(SkillOperationRow.operation_id) + .where( + SkillOperationRow.owner_id == owner_id, + SkillOperationRow.publication.in_(("PREPARED", "NEEDS_REPAIR")), + ) + .limit(1) + ) + uncertain = session.scalar( + select(SkillAssetRow.target_ref) + .where( + SkillAssetRow.owner_id == owner_id, + SkillAssetRow.mutating.is_(True), + ) + .limit(1) + ) + if blocked is not None or uncertain is not None: + raise HostCapabilityError("NEEDS_REPAIR") diff --git a/backend/packages/harness/deerflow/skills/mutations/scan_budget.py b/backend/packages/harness/deerflow/skills/mutations/scan_budget.py new file mode 100644 index 0000000000..419c0551be --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/scan_budget.py @@ -0,0 +1,38 @@ +"""Cross-worker scan leases and hourly cost accounting, including reverts.""" + +import time + +from deerflow_extension_api.host_capabilities import HostCapabilityError +from sqlalchemy import func, select, text + +from deerflow.persistence.skill_mutations.model import SkillScanAttemptRow + + +def reserve_scan(session, *, plugin_id, owner_id, subject, token): + # Called before other statements in this short transaction. The global + # admission mutex guards count+insert on different owners and processes. + if session.bind.dialect.name == "sqlite": + session.execute(text("BEGIN IMMEDIATE")) + elif session.bind.dialect.name == "postgresql": + session.execute(text("SELECT pg_advisory_xact_lock(731129850451)")) + now = time.time() + + def count(*conditions): + return session.scalar(select(func.count()).select_from(SkillScanAttemptRow).where(*conditions)) + + live = SkillScanAttemptRow.lease_until > now + plugin = SkillScanAttemptRow.plugin_id == plugin_id + owner = SkillScanAttemptRow.owner_id == owner_id + if count(live) >= 4 or count(live, plugin, owner) >= 2 or count(live, plugin, owner, SkillScanAttemptRow.subject == subject): + raise HostCapabilityError("BUSY", retry_after=1) + recent = SkillScanAttemptRow.started_at > now - 3600 + if count(plugin, owner, recent) >= 20: + raise HostCapabilityError("QUOTA_EXCEEDED") + session.add(SkillScanAttemptRow(token=token, plugin_id=plugin_id, owner_id=owner_id, subject=subject, started_at=now, lease_until=now + 130)) + + +def release_scan(repository, token): + with repository.sessions.begin() as session: + row = session.get(SkillScanAttemptRow, token) + if row is not None: + row.lease_until = 0 diff --git a/backend/packages/harness/deerflow/skills/mutations/scanner.py b/backend/packages/harness/deerflow/skills/mutations/scanner.py new file mode 100644 index 0000000000..829594cb79 --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/scanner.py @@ -0,0 +1,111 @@ +"""Fail-closed checks of exact detached packages, never model-visible activation.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import tempfile +from dataclasses import dataclass +from pathlib import Path + +from deerflow.skills.mutations.assets import PackageSnapshot +from deerflow.skills.mutations.validation import MAX_MAIN_BYTES, PARSER_VERSION +from deerflow.skills.package_files import is_code_file +from deerflow.skills.security_scanner import scan_skill_content +from deerflow.skills.skillscan.orchestrator import scan_skill_dir +from deerflow.utils.file_io import await_drained + +CHECK_POLICY_VERSION = "skill-mutation-check-v1" +MAX_MODEL_FILES = 32 +MAX_MODEL_BYTES = 1024 * 1024 +CHECK_TIMEOUT_SECONDS = 120 + + +@dataclass(frozen=True) +class ScanVerdict: + decision: str + reason_code: str + policy_version: str + + +class CandidateScanner: + def __init__(self, config_provider): + self._config_provider = config_provider + + @staticmethod + def _policy(config): + # Credentials never enter the returned value. Model configuration and + # explicit code-policy versions invalidate approvals after changes. + models = [model.model_dump(mode="json", exclude={"api_key"}) for model in config.models] + policy = [CHECK_POLICY_VERSION, PARSER_VERSION, config.skill_scan.enabled, config.skill_evolution.moderation_model_name, models] + return hashlib.sha256(json.dumps(policy, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + async def policy_version(self): + return await asyncio.to_thread(self.policy_version_sync) + + def policy_version_sync(self): + """Revalidate policy during short guarded publication admission.""" + return self._policy(self._config_provider()) + + @staticmethod + def _prepare(package: PackageSnapshot): + text_files = [] + total = 0 + for item in package.files: + executable = item.executable or is_code_file(item.path, item.content[:4096]) + try: + content = item.content.decode("utf-8") + except UnicodeError: + if executable or item.path == "SKILL.md": + return None, None + continue # Binary assets still receive the whole-package static scan. + if "\0" in content: + if executable or item.path == "SKILL.md": + return None, None + continue + total += len(item.content) + if len(item.content) > MAX_MAIN_BYTES or total > MAX_MODEL_BYTES or len(text_files) >= MAX_MODEL_FILES: + return None, None + text_files.append((item.path, content, executable)) + with tempfile.TemporaryDirectory(prefix="deerflow-host-check-") as temporary: + root = Path(temporary) + for item in package.files: + target = root / item.path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(item.content) + target.chmod(0o700 if item.executable else 0o600) + result = scan_skill_dir(root) + return result, text_files + + async def scan(self, package: PackageSnapshot, name: str) -> ScanVerdict: + policy = "unavailable" + try: + config = await asyncio.to_thread(self._config_provider) + policy = self._policy(config) + if not config.skill_scan.enabled: + return ScanVerdict("unavailable", "SCANNER_DISABLED", policy) + # This capability never inherits the old manual path's fail-open flag. + config = config.model_copy(deep=True) + config.skill_evolution.security_fail_closed = True + async with asyncio.timeout(CHECK_TIMEOUT_SECONDS): + result, files = await await_drained(asyncio.to_thread(self._prepare, package)) + if result is None: + return ScanVerdict("unavailable", "SCAN_LIMIT_EXCEEDED", policy) + if result["scanner_errors"]: + return ScanVerdict("unavailable", "SCANNER_INCOMPLETE", policy) + if result["blocked"]: + return ScanVerdict("reject", "STATIC_SCAN_REJECTED", policy) + for path, content, executable in files: + findings = [finding for finding in result["findings"] if finding["file"] == path] + scanned = await scan_skill_content(content, executable=executable, location=f"{name}/{path}", app_config=config, static_findings=findings, redact_diagnostics=True) + if not scanned.available: + return ScanVerdict("unavailable", "SCANNER_UNAVAILABLE", policy) + if scanned.decision == "block" or (executable and scanned.decision != "allow"): + return ScanVerdict("reject", "MODEL_SCAN_REJECTED", policy) + return ScanVerdict("allow", "CHECK_PASSED", policy) + except (asyncio.CancelledError, TimeoutError): + raise + except Exception: + # Error details can include model/provider content. Return codes only. + return ScanVerdict("unavailable", "SCANNER_UNAVAILABLE", policy) diff --git a/backend/packages/harness/deerflow/skills/mutations/service.py b/backend/packages/harness/deerflow/skills/mutations/service.py new file mode 100644 index 0000000000..f0c45a417f --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/service.py @@ -0,0 +1,608 @@ +"""Bound P0 host service: exact proposals, fail-closed checks and durable CAS.""" + +from __future__ import annotations + +import asyncio +import json +import os +import time +import uuid +from contextlib import contextmanager +from dataclasses import asdict + +from deerflow_extension_api.host_capabilities import HostCapabilityError +from deerflow_extension_api.skill_mutations import AssessmentRef, AssetRevision, BundleFile, MutationCapabilities, ProposalBundle, SkillRevisionView +from sqlalchemy import func, select, text +from sqlalchemy.exc import IntegrityError, OperationalError + +from deerflow.persistence.run.model import CompletedRunSnapshotRow, RunRow +from deerflow.persistence.skill_mutations.model import SkillAssetRow, SkillOperationRow, SkillOwnerRow, SkillProposalRow +from deerflow.persistence.user.model import UserRow +from deerflow.skills.mutations.assets import MAX_PACKAGE_BYTES, capture_package +from deerflow.skills.mutations.codec import check_view, decode_package, encode_package, fingerprint, operation_packages, operation_view, proposal_view +from deerflow.skills.mutations.publication import assert_temporary_absent, publish_prepared +from deerflow.skills.mutations.recovery import SkillMutationRecovery +from deerflow.skills.mutations.repository import revision +from deerflow.skills.mutations.scan_budget import release_scan, reserve_scan +from deerflow.skills.mutations.validation import MAX_MAIN_BYTES, validate_candidate +from deerflow.skills.mutations.workers import MutationWorkers +from deerflow.skills.projection import skill_projection_read_lock + + +class HostSkillMutationService: + def __init__(self, binding, *, runtime, evidence, storage_factory, scanner, workers=None, recovery=None): + self.binding, self.runtime, self.evidence = binding, runtime, evidence + self.storage_factory, self.scanner = storage_factory, scanner + self.repository = runtime.repository + self._owns_workers = workers is None + self.workers = workers or MutationWorkers() + self.recovery = recovery or SkillMutationRecovery(runtime, storage_factory) + self._checks = {} + self._reverts = set() + self._closed = False + self._closing = False + + @property + def capabilities(self): + return MutationCapabilities(operations=self.binding.access.skill_mutations.operations) + + async def close(self): + self._closing = True + if self._checks or self._reverts: + from deerflow.utils.file_io import await_drained + + await await_drained(asyncio.gather(*tuple(self._checks.values()), *tuple(self._reverts), return_exceptions=True)) + self._closed = True + if self._owns_workers: + await self.workers.close() + + def _authorize(self, owner_id, name, operation=None): + grant = self.binding.access.skill_mutations + allowed = self.binding.allows(operation, owner_id, name) if operation else bool(grant.operations) and owner_id in grant.owners and name in grant.target_skills + if not allowed or owner_id not in self.runtime.owners: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN") + if os.name != "posix" or grant.topology != "single_host_local": + raise HostCapabilityError("UNSUPPORTED_TOPOLOGY") + + def _owner_exists(self, session, owner_id): + owner = session.get(SkillOwnerRow, owner_id) + if session.get(UserRow, owner_id) is None or (owner is not None and owner.deleting): + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN") + + def _row(self, session, model, identifier, operation=None): + row = session.get(model, identifier) + if row is None or row.plugin_id != self.binding.plugin_id: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN") + self._authorize(row.owner_id, row.name, operation) + self._owner_exists(session, row.owner_id) + return row + + async def _work(self, work): + if self._closed: + raise HostCapabilityError("UNAVAILABLE") + + def invoke(check): + try: + return work(check) + except TimeoutError as exc: + raise HostCapabilityError("BUSY", retry_after=1) from exc + except (OperationalError, IntegrityError) as exc: + raise HostCapabilityError("BUSY", retry_after=1) from exc + except ValueError as exc: + code = str(exc) + raise HostCapabilityError(code if code in {"UNSUPPORTED_ASSET", "REVISION_CONFLICT", "QUOTA_EXCEEDED", "INVALID_FRONTMATTER", "FRONTMATTER_CHANGED"} else "INVALID_REQUEST") from exc + + return await self.workers.run(invoke) + + @staticmethod + def _key(value): + if not isinstance(value, str) or not value or len(value) > 128 or any(ord(c) < 32 for c in value): + raise HostCapabilityError("INVALID_REQUEST") + + @contextmanager + def _guard(self, owner_id, check): + storage = self.storage_factory(owner_id) + with skill_projection_read_lock(storage, check=check): + check() + self.recovery.recover_locked(storage) + self.repository.check_readable(owner_id) + yield storage + + def _capture(self, storage, name): + package = capture_package(storage.get_custom_skill_dir(name)) + _, enabled, exists = self.runtime.state(storage, name) + row = self.repository.observe(storage.user_id, name, package.digest, enabled, exists=exists) + if not exists: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN") + return row, package + + async def _source(self, source_ref): + try: + snap = await self.evidence.resolve_snapshot(source_ref) + except HostCapabilityError as exc: + raise HostCapabilityError("SOURCE_STALE_OR_GONE") from exc + grant = self.binding.access.skill_mutations + if snap.owner_id not in grant.owners or snap.agent_id not in grant.trigger_agents or snap.seal_state != "sealed" or snap.origin not in {"interactive", "scheduled"} or snap.coverage != "lead-journal-v1": + raise HostCapabilityError("SOURCE_STALE_OR_GONE") + return asdict(snap) + + def _sources_locked(self, session, sources, owner_id): + """Retain run share locks until PREPARED commits; deletion takes UPDATE.""" + for source in sorted(sources, key=lambda item: item["run_id"]): + stored = session.get(CompletedRunSnapshotRow, source["snapshot_ref"]) + run = session.scalar(select(RunRow).where(RunRow.run_id == source["run_id"]).with_for_update(read=True)) + if ( + stored is None + or stored.scope_digest != self.evidence.scope_digest + or fingerprint(stored.snapshot_json) != fingerprint(source) + or run is None + or run.user_id != owner_id + or run.thread_id != source["thread_id"] + or run.operation_kind != "run" + or run.evidence_seal_state != "sealed" + or run.evidence_origin not in {"interactive", "scheduled"} + or run.evidence_agent_id not in self.binding.access.skill_mutations.trigger_agents + or run.evidence_origin != source["origin"] + or run.evidence_agent_id != source["agent_id"] + or source["owner_id"] != owner_id + or source.get("coverage") != "lead-journal-v1" + or run.evidence_retention_revision != source["retention_revision"] + or self.evidence.revision_for_run(run) != source["evidence_revision"] + ): + raise HostCapabilityError("SOURCE_STALE_OR_GONE") + + @staticmethod + def _begin_fence(session): + if session.bind.dialect.name == "sqlite": + session.execute(text("BEGIN IMMEDIATE")) + + async def read_skill(self, *, source_ref, name): + source = await self._source(source_ref) + self._authorize(source["owner_id"], name) + + def read(check): + with self._guard(source["owner_id"], check) as storage: + with self.repository.sessions() as session: + self._sources_locked(session, [source], source["owner_id"]) + asset, package = self._capture(storage, name) + return SkillRevisionView(asset.target_ref, asset.owner_id, name, revision(asset), package.main_content, asset.enabled) + + return await self._work(read) + + async def stage(self, *, source_refs, target_ref, expected_revision, content, idempotency_key): + self._key(idempotency_key) + if not isinstance(source_refs, (tuple, list)) or not 1 <= len(source_refs) <= 20 or len(set(source_refs)) != len(source_refs): + raise HostCapabilityError("QUOTA_EXCEEDED") + if not isinstance(expected_revision, AssetRevision) or not isinstance(content, str): + raise HostCapabilityError("INVALID_REQUEST") + if len(content.encode("utf-8")) > MAX_MAIN_BYTES: + raise HostCapabilityError("QUOTA_EXCEEDED") + request = fingerprint([list(source_refs), target_ref, asdict(expected_revision), content]) + + def existing(_): + with self.repository.sessions() as session: + row = session.scalar(select(SkillProposalRow).where(SkillProposalRow.plugin_id == self.binding.plugin_id, SkillProposalRow.idempotency_key == idempotency_key)) + if row is None: + return None + self._row(session, SkillProposalRow, row.proposal_id, "stage") + if row.request_hash != request: + raise HostCapabilityError("IDEMPOTENCY_CONFLICT") + return proposal_view(row) + + cached = await self._work(existing) + if cached is not None: + return cached + sources = [await self._source(ref) for ref in source_refs] + owner_id = sources[0]["owner_id"] + if any(source["owner_id"] != owner_id for source in sources): + raise HostCapabilityError("SOURCE_STALE_OR_GONE") + + def create(check): + with self.repository.sessions() as session: + asset = session.get(SkillAssetRow, target_ref) + if asset is None or asset.owner_id != owner_id: + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN") + name = asset.name + self._authorize(owner_id, name, "stage") + with self._guard(owner_id, check) as storage: + cached = existing(check) + if cached: + return cached + asset, baseline = self._capture(storage, name) + if asset.target_ref != target_ref or revision(asset) != expected_revision: + raise HostCapabilityError("REVISION_CONFLICT") + if not asset.enabled: + raise HostCapabilityError("ASSET_DISABLED") + validate_candidate(baseline.main_content, content, name) + candidate = baseline.with_main(content) + if sum(len(item.content) for item in candidate.files) > MAX_PACKAGE_BYTES: + raise HostCapabilityError("QUOTA_EXCEEDED") + with self.repository.sessions.begin() as session: + self._begin_fence(session) + self.repository.owner(session, owner_id) + self._sources_locked(session, sources, owner_id) + count = session.scalar( + select(func.count()) + .select_from(SkillProposalRow) + .where(SkillProposalRow.owner_id == owner_id, SkillProposalRow.plugin_id == self.binding.plugin_id, SkillProposalRow.state == "PENDING", SkillProposalRow.expires_at > time.time()) + ) + if count >= 20: + raise HostCapabilityError("QUOTA_EXCEEDED") + check() + row = SkillProposalRow( + proposal_id=uuid.uuid4().hex, + plugin_id=self.binding.plugin_id, + owner_id=owner_id, + target_ref=target_ref, + name=name, + base_revision=asdict(expected_revision), + sources=sources, + candidate_hash=candidate.digest, + package_blob=encode_package(baseline), + candidate_blob=encode_package(candidate), + state="PENDING", + expires_at=time.time() + 86400, + idempotency_key=idempotency_key, + request_hash=request, + ) + session.add(row) + session.flush() + result = proposal_view(row) + return result + + return await self._work(create) + + async def get_proposal(self, *, proposal_id): + def read(_): + with self.repository.sessions() as session: + return proposal_view(self._row(session, SkillProposalRow, proposal_id)) + + return await self._work(read) + + @staticmethod + def _pending(row): + if row.state != "PENDING" or row.expires_at <= time.time() or row.package_blob is None or row.candidate_blob is None: + raise HostCapabilityError("PROPOSAL_UNAVAILABLE") + + async def read_proposal_bundle(self, *, proposal_id, max_bytes=33554432): + if type(max_bytes) is not int or not 1 <= max_bytes <= 33554432: + raise HostCapabilityError("INVALID_REQUEST") + + def read(_): + with self.repository.sessions() as session: + row = self._row(session, SkillProposalRow, proposal_id) + if row.package_blob is None or row.candidate_blob is None or row.state == "DISCARDED" or row.expires_at <= time.time(): + raise HostCapabilityError("PROPOSAL_UNAVAILABLE") + baseline, candidate = decode_package(row.package_blob), decode_package(row.candidate_blob) + if sum(len(item.content) for package in (baseline, candidate) for item in package.files) > max_bytes: + raise HostCapabilityError("QUOTA_EXCEEDED") + bundles = [tuple(BundleFile(item.path, item.content, item.digest, item.executable) for item in package.files) for package in (baseline, candidate)] + return ProposalBundle(row.proposal_id, *bundles, AssetRevision(**row.base_revision), row.candidate_hash) + + return await self._work(read) + + async def discard(self, *, proposal_id): + owner_id = (await self.get_proposal(proposal_id=proposal_id)).owner_id + + def discard(check): + with skill_projection_read_lock(self.storage_factory(owner_id), check=check), self.repository.sessions.begin() as session: + row = self._row(session, SkillProposalRow, proposal_id, "stage") + if row.operation_id: + raise HostCapabilityError("ALREADY_PUBLISHED") + row.state = "DISCARDED" + return proposal_view(row) + + return await self._work(discard) + + async def check(self, *, proposal_id): + if self._closed or self._closing: + raise HostCapabilityError("UNAVAILABLE") + task = self._checks.get(proposal_id) + if task is None: + task = asyncio.create_task(self._check(proposal_id)) + self._checks[proposal_id] = task + + def finished(done): + self._checks.pop(proposal_id, None) + if not done.cancelled(): + done.exception() + + task.add_done_callback(finished) + return await asyncio.shield(task) + + async def _check(self, proposal_id): + proposal = await self.get_proposal(proposal_id=proposal_id) + policy = await self.scanner.policy_version() + token = uuid.uuid4().hex + + def reserve(check): + with self._guard(proposal.owner_id, check) as storage: + asset, _ = self._capture(storage, proposal.name) + with self.repository.sessions.begin() as session: + row = self._row(session, SkillProposalRow, proposal_id, "check") + self._pending(row) + if revision(asset) != AssetRevision(**row.base_revision): + raise HostCapabilityError("REVISION_CONFLICT") + if not asset.enabled: + raise HostCapabilityError("ASSET_DISABLED") + now = time.time() + previous = row.check_result or {} + if previous.get("policy_version") == policy and previous.get("expires_at", 0) > now: + return check_view(row), None + if previous.get("lease_until", 0) > now: + raise HostCapabilityError("BUSY", retry_after=1) + reserve_scan(session, plugin_id=self.binding.plugin_id, owner_id=row.owner_id, subject="check:" + proposal_id, token=token) + row.check_result = {"token": token, "lease_until": now + 130} + return None, decode_package(row.candidate_blob) + + cached, package = await self._work(reserve) + if cached: + return cached + + def finish(check): + with skill_projection_read_lock(self.storage_factory(proposal.owner_id), check=check), self.repository.sessions.begin() as session: + row = self._row(session, SkillProposalRow, proposal_id, "check") + self._pending(row) + if (row.check_result or {}).get("token") != token: + raise HostCapabilityError("CHECK_REQUIRED") + row.check_result = {"decision": verdict.decision, "reason_code": verdict.reason_code, "policy_version": verdict.policy_version, "expires_at": time.time() + 900} + return check_view(row) + + def cleanup(_check): + try: + with skill_projection_read_lock(self.storage_factory(proposal.owner_id)), self.repository.sessions.begin() as session: + row = session.get(SkillProposalRow, proposal_id) + if row is not None and (row.check_result or {}).get("token") == token: + # An interrupted attempt is not an authoritative verdict. + # Never clear a newer worker's lease or completed result. + row.check_result = None + finally: + release_scan(self.repository, token) + + try: + try: + async with asyncio.timeout(120): + verdict = await self.scanner.scan(package, proposal.name) + except TimeoutError: + raise + except Exception: + from deerflow.skills.mutations.scanner import ScanVerdict + + verdict = ScanVerdict("unavailable", "SCANNER_UNAVAILABLE", policy) + return await self._work(finish) + finally: + from deerflow.utils.file_io import await_drained + + await await_drained(self.workers.run(cleanup)) + + def _existing_operation(self, session, method, key, request): + row = session.scalar(select(SkillOperationRow).where(SkillOperationRow.plugin_id == self.binding.plugin_id, SkillOperationRow.method == method, SkillOperationRow.idempotency_key == key)) + if row is not None: + self._row(session, SkillOperationRow, row.operation_id, method) + if row.request_hash != request: + raise HostCapabilityError("IDEMPOTENCY_CONFLICT") + return row + + def _prepare(self, session, *, owner_id, asset, before, after, method, key, request, proposal=None, reverts=None, assessment=None): + owner = self.repository.owner(session, owner_id) + changed = before.digest != after.digest + before_revision = revision(asset) + after_revision = AssetRevision(before_revision.incarnation_id, before_revision.mutation_seq + int(changed), after.digest) + operation = SkillOperationRow( + operation_id=uuid.uuid4().hex, + plugin_id=self.binding.plugin_id, + owner_id=owner_id, + target_ref=asset.target_ref, + name=asset.name, + method=method, + idempotency_key=key, + request_hash=request, + proposal_id=proposal.proposal_id if proposal else None, + reverts_operation_id=reverts, + before_operation_id=asset.operation_id, + before_revision=asdict(before_revision), + after_revision=asdict(after_revision), + before_blob=encode_package(before), + after_blob=encode_package(after), + generation=owner.generation + int(changed), + publication="PREPARED" if changed else "NO_CHANGE", + views="PENDING" if changed else "READY", + created_at=time.time(), + rollback_expires_at=time.time() + 2592000, + assessment=assessment, + ) + if changed: + assert_temporary_absent(self.storage_factory(owner_id).get_custom_skill_dir(asset.name), operation.operation_id) + session.add(operation) + if changed: + canonical = session.get(SkillAssetRow, asset.target_ref) + canonical.operation_id = operation.operation_id + canonical.mutating = True + if proposal: + proposal.operation_id, proposal.state = operation.operation_id, "PUBLISHED" + session.flush() + return operation.operation_id, changed, operation_view(operation) + + def _publish_result(self, storage, operation_id, changed, prepared): + try: + if changed: + publish_prepared(self.repository, storage, operation_id) + return self.recovery.get_operation(operation_id) + except Exception: + # Admission is already durable. Even an unavailable database must + # return the accepted ID, never a refusal that invites republication. + return prepared + + def _refresh_result(self, owner_id, operation): + try: + self.recovery.refresh_views(owner_id) + return self.recovery.get_operation(operation.operation_id) + except Exception: + return operation + + @staticmethod + def _assessment_payload(assessment_ref): + if assessment_ref is None: + return None + try: + if not isinstance(assessment_ref, AssessmentRef) or not isinstance(assessment_ref.base_revision, AssetRevision): + raise TypeError + assessment = asdict(assessment_ref) + base = assessment["base_revision"] + if ( + not isinstance(assessment["candidate_hash"], str) + or len(assessment["candidate_hash"]) != 64 + or set(base) != {"incarnation_id", "mutation_seq", "content_digest"} + or not isinstance(base["incarnation_id"], str) + or not isinstance(base["content_digest"], str) + or type(base["mutation_seq"]) is not int + or any(not isinstance(assessment[field], str) or not 1 <= len(assessment[field]) <= 512 for field in ("evaluator", "evaluator_version", "report_id")) + or len(json.dumps(assessment, allow_nan=False).encode("utf-8")) > 4096 + ): + raise ValueError + return assessment + except Exception as exc: + raise HostCapabilityError("INVALID_ASSESSMENT") from exc + + async def commit(self, *, proposal_id, idempotency_key, assessment_ref=None): + self._key(idempotency_key) + assessment = self._assessment_payload(assessment_ref) + request = fingerprint([proposal_id, assessment]) + proposal = await self.get_proposal(proposal_id=proposal_id) + + def commit(check): + with skill_projection_read_lock(self.storage_factory(proposal.owner_id), check=check): + storage = self.storage_factory(proposal.owner_id) + self.recovery.recover_locked(storage) + with self.repository.sessions() as session: + row = self._row(session, SkillProposalRow, proposal_id, "commit") + existing = self._existing_operation(session, "commit", idempotency_key, request) + if existing is not None: + return operation_view(existing) + if row.operation_id: + raise HostCapabilityError("PROPOSAL_ALREADY_COMMITTED") + self.repository.check_readable(proposal.owner_id) + asset, baseline = self._capture(storage, proposal.name) + with self.repository.sessions.begin() as session: + self._begin_fence(session) + row = self._row(session, SkillProposalRow, proposal_id, "commit") + self._pending(row) + self.repository.owner(session, row.owner_id) + self._sources_locked(session, row.sources, row.owner_id) + if asset.target_ref != row.target_ref or revision(asset) != AssetRevision(**row.base_revision): + raise HostCapabilityError("REVISION_CONFLICT") + if not asset.enabled: + raise HostCapabilityError("ASSET_DISABLED") + checked = row.check_result or {} + try: + policy = self.scanner.policy_version_sync() + except Exception as exc: + raise HostCapabilityError("CHECK_REQUIRED") from exc + if checked.get("decision") != "allow" or checked.get("expires_at", 0) <= time.time() or checked.get("policy_version") != policy: + raise HostCapabilityError("CHECK_REQUIRED") + if assessment and (assessment["candidate_hash"] != row.candidate_hash or assessment["base_revision"] != row.base_revision): + raise HostCapabilityError("INVALID_ASSESSMENT") + candidate = decode_package(row.candidate_blob) + if baseline.digest != decode_package(row.package_blob).digest or candidate.digest != row.candidate_hash: + raise HostCapabilityError("REVISION_CONFLICT") + validate_candidate(baseline.main_content, candidate.main_content, row.name) + check() # Last cancellation boundary: PREPARED is irrevocable admission. + operation_id, changed, prepared = self._prepare(session, owner_id=row.owner_id, asset=asset, before=baseline, after=candidate, method="commit", key=idempotency_key, request=request, proposal=row, assessment=assessment) + result = self._publish_result(storage, operation_id, changed, prepared) + return self._refresh_result(proposal.owner_id, result) + + return await self._work(commit) + + async def get_operation(self, *, operation_id): + def read(_): + with self.repository.sessions() as session: + return operation_view(self._row(session, SkillOperationRow, operation_id)) + + return await self._work(read) + + async def find_operation(self, *, idempotency_key, method="commit"): + self._key(idempotency_key) + if method not in {"commit", "revert"}: + raise HostCapabilityError("INVALID_REQUEST") + + def read(_): + with self.repository.sessions() as session: + row = session.scalar(select(SkillOperationRow).where(SkillOperationRow.plugin_id == self.binding.plugin_id, SkillOperationRow.method == method, SkillOperationRow.idempotency_key == idempotency_key)) + return operation_view(self._row(session, SkillOperationRow, row.operation_id)) if row else None + + return await self._work(read) + + async def revert(self, *, operation_id, expected_current_revision, idempotency_key): + if self._closing or self._closed: + raise HostCapabilityError("UNAVAILABLE") + task = asyncio.create_task(self._revert(operation_id=operation_id, expected_current_revision=expected_current_revision, idempotency_key=idempotency_key)) + self._reverts.add(task) + task.add_done_callback(self._reverts.discard) + return await task + + async def _revert(self, *, operation_id, expected_current_revision, idempotency_key): + self._key(idempotency_key) + request = fingerprint([operation_id, asdict(expected_current_revision)]) + + def inspect(check): + with self.repository.sessions() as session: + original = self._row(session, SkillOperationRow, operation_id, "revert") + existing = self._existing_operation(session, "revert", idempotency_key, request) + if existing: + return operation_view(existing), None, None + if original.publication != "APPLIED" or original.before_blob is None or original.rollback_expires_at <= time.time(): + raise HostCapabilityError("ROLLBACK_UNAVAILABLE") + try: + before, _ = operation_packages(original) + except ValueError as exc: + raise HostCapabilityError("ROLLBACK_UNAVAILABLE") from exc + return None, operation_view(original), before + + existing, original, restored = await self._work(inspect) + if existing: + return existing + token = uuid.uuid4().hex + + def reserve(check): + with self._guard(original.owner_id, check) as storage: + asset, _ = self._capture(storage, original.name) + if revision(asset) != expected_current_revision or revision(asset) != original.after_revision or asset.operation_id != operation_id: + raise HostCapabilityError("REVISION_CONFLICT") + with self.repository.sessions.begin() as session: + reserve_scan(session, plugin_id=self.binding.plugin_id, owner_id=original.owner_id, subject="revert:" + operation_id, token=token) + + await self._work(reserve) + try: + async with asyncio.timeout(120): + verdict = await self.scanner.scan(restored, original.name) + except Exception as exc: + raise HostCapabilityError("CHECK_REQUIRED") from exc + finally: + await self.workers.run(lambda _: release_scan(self.repository, token)) + if verdict.decision != "allow": + raise HostCapabilityError("CHECK_REQUIRED") + + def revert(check): + with self._guard(original.owner_id, check) as storage: + asset, current = self._capture(storage, original.name) + with self.repository.sessions.begin() as session: + row = self._row(session, SkillOperationRow, operation_id, "revert") + existing = self._existing_operation(session, "revert", idempotency_key, request) + if existing: + return operation_view(existing) + if row.before_blob is None or row.rollback_expires_at <= time.time(): + raise HostCapabilityError("ROLLBACK_UNAVAILABLE") + if revision(asset) != expected_current_revision or revision(asset) != AssetRevision(**row.after_revision) or asset.operation_id != row.operation_id: + raise HostCapabilityError("REVISION_CONFLICT") + if not asset.enabled: + raise HostCapabilityError("ASSET_DISABLED") + if verdict.policy_version != self.scanner.policy_version_sync(): + raise HostCapabilityError("CHECK_REQUIRED") + if restored.digest != decode_package(row.before_blob).digest: + raise HostCapabilityError("ROLLBACK_UNAVAILABLE") + check() + new_id, changed, prepared = self._prepare(session, owner_id=row.owner_id, asset=asset, before=current, after=restored, method="revert", key=idempotency_key, request=request, reverts=row.operation_id) + result = self._publish_result(storage, new_id, changed, prepared) + return self._refresh_result(original.owner_id, result) + + return await self._work(revert) diff --git a/backend/packages/harness/deerflow/skills/mutations/topology.py b/backend/packages/harness/deerflow/skills/mutations/topology.py new file mode 100644 index 0000000000..25d6fa4a6e --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/topology.py @@ -0,0 +1,81 @@ +"""Publication topology admission and a short-transaction durable DB pool.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from deerflow_extension_api.host_capabilities import HostCapabilityError +from sqlalchemy import create_engine, event +from sqlalchemy.engine import make_url +from sqlalchemy.orm import sessionmaker + + +class PublicationTopology: + """Operator attests local single host; SQLite additionally enforces one host. + + A file lock cannot prove that a mount is not NFS/CSI. That remains an explicit + deployment prerequisite, not a capability inferred from PostgreSQL alone. + """ + + def __init__(self, backend: str, root: Path): + self.backend = backend + self.root = root + self._fd: int | None = None + + def acquire(self): + if os.name != "posix" or self.backend not in {"sqlite", "postgres"}: + raise HostCapabilityError("UNSUPPORTED_TOPOLOGY") + if self._fd is not None or self.backend == "postgres": + return + import fcntl + + self.root.mkdir(parents=True, exist_ok=True) + fd = os.open(self.root / ".skill-mutations-gateway.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exc: + os.close(fd) + raise HostCapabilityError("UNSUPPORTED_TOPOLOGY", "SQLite publication requires one Gateway process") from exc + self._fd = fd + + def close(self): + if self._fd is not None: + fd, self._fd = self._fd, None + os.close(fd) + + +def mutation_session_factory(url: str, *, postgres_schema: str = ""): + """Same application DB, dedicated bounded pool and fsync-strict journal writes. + + Do not change PRAGMAs on the shared agent-definition engine. FULL is needed + for PREPARED to be durable before replacing the canonical filesystem file. + """ + parsed = make_url(url) + if parsed.get_backend_name() == "sqlite": + if not parsed.database or parsed.database == ":memory:": + raise HostCapabilityError("UNSUPPORTED_TOPOLOGY") + args = {"check_same_thread": False, "timeout": 5} + elif parsed.get_backend_name() == "postgresql": + from deerflow.config.postgres_schema import validate_postgres_schema + from deerflow.persistence.postgres_schema import build_psycopg_options + + options = build_psycopg_options(validate_postgres_schema(postgres_schema)) or "" + args = {"connect_timeout": 5, "options": f"{options} -c lock_timeout=5000 -c statement_timeout=30000 -c synchronous_commit=on".strip()} + else: + raise HostCapabilityError("UNSUPPORTED_TOPOLOGY") + engine = create_engine(url, connect_args=args, pool_size=4, max_overflow=0, pool_timeout=5, pool_pre_ping=True) + if parsed.get_backend_name() == "sqlite": + + @event.listens_for(engine, "connect") + def setup(connection, _record): + cursor = connection.cursor() + try: + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA synchronous=FULL") + cursor.execute("PRAGMA foreign_keys=ON") + cursor.execute("PRAGMA busy_timeout=5000") + finally: + cursor.close() + + return engine, sessionmaker(engine, expire_on_commit=False) diff --git a/backend/packages/harness/deerflow/skills/mutations/validation.py b/backend/packages/harness/deerflow/skills/mutations/validation.py new file mode 100644 index 0000000000..fabbd63aa2 --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/validation.py @@ -0,0 +1,73 @@ +"""Strict P0 authority-preserving validation; never activate the candidate.""" + +from __future__ import annotations + +import json + +import yaml + +from deerflow.skills.frontmatter import _FRONTMATTER_RE, ALLOWED_FRONTMATTER_PROPERTIES +from deerflow.skills.validation import validate_skill_frontmatter_text + +MAX_MAIN_BYTES = 128 * 1024 +PARSER_VERSION = "skill-mutation-frontmatter-v1" + + +class _StrictLoader(yaml.SafeLoader): + def construct_mapping(self, node, deep=False): + result = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + if not isinstance(key, str) or key in result: + raise ValueError("INVALID_FRONTMATTER") + result[key] = self.construct_object(value_node, deep=deep) + return result + + +def _metadata(content: str) -> dict: + if not isinstance(content, str) or len(content.encode("utf-8")) > MAX_MAIN_BYTES: + raise ValueError("QUOTA_EXCEEDED") + match = _FRONTMATTER_RE.match(content) + if match is None: + raise ValueError("INVALID_FRONTMATTER") + source = match.group(1) + try: + depth = 0 + for count, event in enumerate(yaml.parse(source, Loader=yaml.SafeLoader)): + if count > 4096 or isinstance(event, yaml.events.AliasEvent): + raise ValueError("INVALID_FRONTMATTER") + if isinstance(event, (yaml.events.MappingStartEvent, yaml.events.SequenceStartEvent)): + depth += 1 + elif isinstance(event, (yaml.events.MappingEndEvent, yaml.events.SequenceEndEvent)): + depth -= 1 + if depth > 24: + raise ValueError("INVALID_FRONTMATTER") + metadata = yaml.load(source, Loader=_StrictLoader) + if not isinstance(metadata, dict): + raise ValueError("INVALID_FRONTMATTER") + # Reject values without stable JSON/type semantics (dates, sets, NaN). + json.dumps(metadata, allow_nan=False) + return metadata + except (yaml.YAMLError, ValueError, TypeError, RecursionError, OverflowError) as exc: + raise ValueError("INVALID_FRONTMATTER") from exc + + +def _typed(value): + if isinstance(value, dict): + return (dict, tuple(sorted((key, _typed(item)) for key, item in value.items()))) + if isinstance(value, list): + return (list, tuple(_typed(item) for item in value)) + return (type(value), value) + + +def validate_candidate(baseline: str, candidate: str, name: str) -> None: + """Only body and description can differ; unknown declarations stay intact.""" + before, after = _metadata(baseline), _metadata(candidate) + if _typed({k: v for k, v in before.items() if k != "description"}) != _typed({k: v for k, v in after.items() if k != "description"}): + raise ValueError("FRONTMATTER_CHANGED") + # Reuse runtime/install field validation without discarding unknown values + # from the actual candidate: comparison above already preserves them. + known = {k: v for k, v in after.items() if k in ALLOWED_FRONTMATTER_PROPERTIES} + valid, _, declared_name = validate_skill_frontmatter_text("---\n" + yaml.safe_dump(known) + "---\n") + if not valid or declared_name != name: + raise ValueError("INVALID_FRONTMATTER") diff --git a/backend/packages/harness/deerflow/skills/mutations/workers.py b/backend/packages/harness/deerflow/skills/mutations/workers.py new file mode 100644 index 0000000000..8014678f6e --- /dev/null +++ b/backend/packages/harness/deerflow/skills/mutations/workers.py @@ -0,0 +1,67 @@ +"""Bounded off-loop work. Caller cancellation never releases a worker's locks.""" + +from __future__ import annotations + +import asyncio +import threading +from concurrent.futures import ThreadPoolExecutor +from contextvars import copy_context + +from deerflow_extension_api.host_capabilities import HostCapabilityError + + +class MutationWorkers: + def __init__(self, *, limit: int = 4): + self._slots = threading.BoundedSemaphore(limit) + self._executor = ThreadPoolExecutor(max_workers=limit, thread_name_prefix="skill-mutation") + self._tasks: set[asyncio.Future] = set() + self._closed = False + + async def run(self, work): + if self._closed: + raise HostCapabilityError("UNAVAILABLE") + if not self._slots.acquire(blocking=False): + raise HostCapabilityError("BUSY", retry_after=1) + cancelled = threading.Event() + + def check(): + # Call during admission/lock retries, never after PREPARED. + if cancelled.is_set(): + raise HostCapabilityError("CANCELLED") + + def invoke(): + try: + check() + return work(check) + finally: + self._slots.release() + + context = copy_context() + try: + task = asyncio.get_running_loop().run_in_executor(self._executor, context.run, invoke) + except BaseException: + self._slots.release() + raise + self._tasks.add(task) + + def finished(done): + self._tasks.discard(done) + if not done.cancelled(): + done.exception() # Detached cancellations still consume failures. + + task.add_done_callback(finished) + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + cancelled.set() + raise + + async def close(self): + from deerflow.utils.file_io import await_drained + + self._closed = True + if self._tasks: + await await_drained(asyncio.gather(*tuple(self._tasks), return_exceptions=True)) + # All accepted functions have completed. Signal idle threads to exit + # without making shutdown depend on the shared default executor. + self._executor.shutdown(wait=False, cancel_futures=True) diff --git a/backend/packages/harness/deerflow/skills/projection.py b/backend/packages/harness/deerflow/skills/projection.py index 844f5d8c8e..4e9c1b08db 100644 --- a/backend/packages/harness/deerflow/skills/projection.py +++ b/backend/packages/harness/deerflow/skills/projection.py @@ -33,6 +33,7 @@ except ImportError: # pragma: no cover - Windows _locks_guard = threading.Lock() _process_locks: dict[Path, threading.RLock] = {} +_held_projection_locks = threading.local() _MANIFEST_VERSION = 1 _MAX_REBUILD_ATTEMPTS = 2 _THREAD_PROJECTION_POLICY_VERSION = 1 @@ -98,25 +99,59 @@ def _lock_for(path: Path) -> threading.RLock: @contextmanager -def _projection_lock(root: Path) -> Iterator[None]: +def _projection_lock(root: Path, *, timeout: float | None = None, check=None) -> Iterator[None]: """Serialize projection replacement in-process and across POSIX workers.""" + import time + lock_path = root.parent / f".{root.name}.projection.lock" lock_path.parent.mkdir(parents=True, exist_ok=True) process_lock = _lock_for(lock_path) - with process_lock, lock_path.open("a", encoding="utf-8") as lock_file: - if fcntl is not None: - fcntl.flock(lock_file, fcntl.LOCK_EX) - else: # pragma: no cover - Windows - lock_file.seek(0) - msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1) - try: + key = lock_path.resolve() + deadline = None if timeout is None else time.monotonic() + timeout + while not process_lock.acquire(timeout=0.05): + if check: + check() + if deadline is not None and time.monotonic() >= deadline: + raise TimeoutError("Skill projection lock timeout") + try: + held = getattr(_held_projection_locks, "paths", None) + if held is None: + held = _held_projection_locks.paths = set() + if key in held: + # Opening and flocking a second descriptor deadlocks even though + # the in-process RLock is reentrant. Nested readers share the fd. yield - finally: - if fcntl is not None: - fcntl.flock(lock_file, fcntl.LOCK_UN) - else: # pragma: no cover - Windows - lock_file.seek(0) - msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + return + with lock_path.open("a", encoding="utf-8") as lock_file: + + def acquire_region() -> None: + if fcntl is not None: + fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + else: # pragma: no cover - Windows + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1) + + retry_until_locked( + acquire_region, + interval=0.02, + deadline=deadline, + sleep=time.sleep, + monotonic=time.monotonic, + check=check, + timeout_message="Skill projection lock timeout", + ) + held.add(key) + try: + yield + finally: + held.remove(key) + if fcntl is not None: + fcntl.flock(lock_file, fcntl.LOCK_UN) + else: # pragma: no cover - Windows + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + finally: + process_lock.release() def _copy_into_view(source: str, target: str, *, follow_symlinks: bool = True) -> str: @@ -614,6 +649,19 @@ def ensure_thread_skill_projection( storage: SkillStorage, thread_id: str, allowed_skills: set[str] | None, +) -> SkillProjectionPaths | None: + from deerflow.skills.mutations.guard import managed_global_read, managed_read + + # Global -> owner -> thread. No fresh-view fast path bypasses recovery. + with managed_global_read(storage): + with managed_read(storage): + return _ensure_thread_skill_projection(storage, thread_id, allowed_skills) + + +def _ensure_thread_skill_projection( + storage: SkillStorage, + thread_id: str, + allowed_skills: set[str] | None, ) -> SkillProjectionPaths | None: """Ensure the filesystem view for one run's effective Agent skill policy. @@ -655,6 +703,19 @@ def rebuild_skill_projections( *, include_public: bool = True, include_user: bool = True, +) -> SkillProjectionPaths: + from deerflow.skills.mutations.guard import managed_global_read, managed_read + + with managed_global_read(storage): + with managed_read(storage): + return _rebuild_skill_projections(storage, include_public=include_public, include_user=include_user) + + +def _rebuild_skill_projections( + storage: SkillStorage, + *, + include_public: bool = True, + include_user: bool = True, ) -> SkillProjectionPaths: """Rebuild enabled-only projection scopes visible through ``storage``.""" paths = get_skill_projection_paths(storage) @@ -682,6 +743,14 @@ def _public_projection_is_fresh(storage: SkillStorage, paths: SkillProjectionPat def ensure_skill_projections(storage: SkillStorage) -> SkillProjectionPaths: + from deerflow.skills.mutations.guard import managed_global_read, managed_read + + with managed_global_read(storage): + with managed_read(storage): + return _ensure_skill_projections(storage) + + +def _ensure_skill_projections(storage: SkillStorage) -> SkillProjectionPaths: """Repair stale projection scopes, otherwise leave their inodes untouched.""" paths = get_skill_projection_paths(storage) @@ -729,6 +798,7 @@ def skill_projection_mutation( *, remove: tuple[tuple[SkillCategory, Path], ...] = (), remove_names: tuple[str, ...] = (), + mutation_names: tuple[str, ...] = (), ) -> Iterator[None]: """Hold a projection scope lock across a source/state mutation.""" if not isinstance(storage.get_skills_root_path(), Path): @@ -767,7 +837,9 @@ def skill_projection_mutation( _validate_projection_relative_path(relative_path) removals.add((root, relative_path)) - with _projection_lock(scope_root): + from deerflow.skills.mutations.guard import managed_name_writes + + with managed_name_writes(storage, mutation_names), _projection_lock(scope_root): if remove_names: names = set(remove_names) skills = _load_public_skills(storage, enabled_only=False) if scope == "public" else storage.load_skills(enabled_only=False) @@ -782,7 +854,10 @@ def skill_projection_mutation( _manifest_path(scope_root).unlink(missing_ok=True) for root, relative_path in removals: _remove_projection_relative(root, relative_path) - yield + from deerflow.skills.mutations.guard import managed_writer + + with managed_writer(storage, mutation_names): + yield rebuild() except Exception: _clear_projection_scope(scope_root, *category_roots.values()) @@ -819,50 +894,6 @@ def ensure_public_skill_projection(*, app_config=None) -> bool: @contextmanager def skill_projection_read_lock(storage: SkillStorage, *, timeout: float = 5.0, check=None) -> Iterator[None]: """Bounded, non-mutating acquisition of the existing user projection lock.""" - import time - root = (storage.get_skills_root_path() / "custom") if getattr(storage, "user_id", None) is None else get_skill_projection_paths(storage).custom.parent - lock_path = root.parent / f".{root.name}.projection.lock" - lock_path.parent.mkdir(parents=True, exist_ok=True) - process_lock = _lock_for(lock_path) - deadline = time.monotonic() + timeout - acquired = False - try: - while not acquired: - if check: - check() - acquired = process_lock.acquire(timeout=min(0.05, max(0, deadline - time.monotonic()))) - if not acquired and time.monotonic() >= deadline: - raise TimeoutError("Skill projection lock timeout") - with lock_path.open("a", encoding="utf-8") as lock_file: - locked = False - try: - - def acquire_region() -> None: - if fcntl is not None: - fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) - else: # pragma: no cover - Windows - lock_file.seek(0) - msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1) - - retry_until_locked( - acquire_region, - interval=0.02, - deadline=deadline, - sleep=time.sleep, - monotonic=time.monotonic, - check=check, - timeout_message="Skill projection lock timeout", - ) - locked = True - yield - finally: - if locked: - if fcntl is not None: - fcntl.flock(lock_file, fcntl.LOCK_UN) - else: # pragma: no cover - Windows - lock_file.seek(0) - msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) - finally: - if acquired: - process_lock.release() + with _projection_lock(root, timeout=timeout, check=check): + yield diff --git a/backend/packages/harness/deerflow/skills/security_scanner.py b/backend/packages/harness/deerflow/skills/security_scanner.py index e13b95065e..35a6ff3322 100644 --- a/backend/packages/harness/deerflow/skills/security_scanner.py +++ b/backend/packages/harness/deerflow/skills/security_scanner.py @@ -24,6 +24,7 @@ logger = logging.getLogger(__name__) class ScanResult: decision: str reason: str + available: bool = True def _resolve_fail_closed(app_config: AppConfig | None) -> bool: @@ -101,6 +102,7 @@ async def scan_skill_content( app_config: AppConfig | None = None, static_findings: list[dict[str, Any]] | None = None, attach_tracing: bool = True, + redact_diagnostics: bool = False, ) -> ScanResult: """Screen skill content before it is written to disk. @@ -162,15 +164,18 @@ async def scan_skill_content( decision = str(parsed.get("decision", "")).lower() if decision in {"allow", "warn", "block"}: return ScanResult(decision, str(parsed.get("reason") or "No reason provided.")) - logger.warning("Security scan produced unparseable output: %s", raw[:200]) + if redact_diagnostics: + logger.warning("Security scan produced unparseable output") + else: + logger.warning("Security scan produced unparseable output: %s", raw[:200]) except Exception: - logger.warning("Skill security scan model call failed; applying configured fail-closed/fail-open policy", exc_info=True) + logger.warning("Skill security scan model call failed; applying configured fail-closed/fail-open policy", exc_info=not redact_diagnostics) if model_responded: - return ScanResult("block", "Security scan produced unparseable output; manual review required.") + return ScanResult("block", "Security scan produced unparseable output; manual review required.", available=False) if executable: - return ScanResult("block", "Security scan unavailable for executable content; manual review required.") + return ScanResult("block", "Security scan unavailable for executable content; manual review required.", available=False) if _resolve_fail_closed(app_config): - return ScanResult("block", "Security scan unavailable for skill content; manual review required.") + return ScanResult("block", "Security scan unavailable for skill content; manual review required.", available=False) logger.warning("Security scan unavailable; failing open for non-executable skill content at %s (manual review recommended)", location) - return ScanResult("warn", "Security scan unavailable for non-executable skill content; manual review recommended.") + return ScanResult("warn", "Security scan unavailable for non-executable skill content; manual review recommended.", available=False) diff --git a/backend/packages/harness/deerflow/skills/storage/__init__.py b/backend/packages/harness/deerflow/skills/storage/__init__.py index 0ecd51528e..7f669e63bb 100644 --- a/backend/packages/harness/deerflow/skills/storage/__init__.py +++ b/backend/packages/harness/deerflow/skills/storage/__init__.py @@ -55,10 +55,12 @@ def get_or_new_skill_storage(**kwargs) -> SkillStorage: from deerflow.config import get_app_config from deerflow.config.skills_config import SkillsConfig - def _make_storage(skills_config: SkillsConfig, *, host_path: str | None = None, **kwargs) -> SkillStorage: + def _make_storage(skills_config: SkillsConfig, *, app_config=None, host_path: str | None = None, **kwargs) -> SkillStorage: from deerflow.reflection import resolve_class cls = resolve_class(skills_config.use, SkillStorage) + if issubclass(cls, LocalSkillStorage): + kwargs["app_config"] = app_config return cls( host_path=host_path if host_path is not None else str(skills_config.get_skills_path()), container_path=skills_config.container_path, @@ -70,7 +72,7 @@ def get_or_new_skill_storage(**kwargs) -> SkillStorage: if skills_path is not None: if app_config is not None: - return _make_storage(app_config.skills, host_path=str(skills_path), **kwargs) + return _make_storage(app_config.skills, app_config=app_config, host_path=str(skills_path), **kwargs) # No app_config: use a default SkillsConfig so we never need to read config.yaml # when the caller has already supplied an explicit host path. from deerflow.config.skills_config import SkillsConfig @@ -78,7 +80,7 @@ def get_or_new_skill_storage(**kwargs) -> SkillStorage: return _make_storage(SkillsConfig(), host_path=str(skills_path), **kwargs) if app_config is not None: - return _make_storage(app_config.skills, **kwargs) + return _make_storage(app_config.skills, app_config=app_config, **kwargs) # If the singleton was manually injected (e.g. in tests) without a config # identity (_default_skill_storage_config is None), skip get_app_config() @@ -96,7 +98,7 @@ def get_or_new_skill_storage(**kwargs) -> SkillStorage: # orphaned instance from a losing racer could not be cleaned up. with _skill_storage_lock: if _default_skill_storage is None or _default_skill_storage_config is not app_config_now: - _default_skill_storage = _make_storage(app_config_now.skills, **kwargs) + _default_skill_storage = _make_storage(app_config_now.skills, app_config=app_config_now, **kwargs) _default_skill_storage_config = app_config_now return _default_skill_storage diff --git a/backend/packages/harness/deerflow/skills/storage/local_skill_storage.py b/backend/packages/harness/deerflow/skills/storage/local_skill_storage.py index cea4477c59..c5ab2a0c45 100644 --- a/backend/packages/harness/deerflow/skills/storage/local_skill_storage.py +++ b/backend/packages/harness/deerflow/skills/storage/local_skill_storage.py @@ -91,12 +91,15 @@ class LocalSkillStorage(SkillStorage): yield category, category_path, Path(current_root) / SKILL_MD_FILE def read_custom_skill(self, name: str) -> str: - if not self.custom_skill_exists(name): - raise FileNotFoundError(f"Custom skill '{name}' not found.") - return (self.get_custom_skill_dir(name) / SKILL_MD_FILE).read_text(encoding="utf-8") + from deerflow.skills.mutations.guard import managed_read + + with managed_read(self): + if not self.custom_skill_exists(name): + raise FileNotFoundError(f"Custom skill '{name}' not found.") + return (self.get_custom_skill_dir(name) / SKILL_MD_FILE).read_text(encoding="utf-8") def write_custom_skill(self, name: str, relative_path: str, content: str) -> None: - with self._skill_projection_mutation(): + with self._skill_projection_mutation(names=(name,)): target = self.validate_relative_path(relative_path, self.get_custom_skill_dir(name)) target.parent.mkdir(parents=True, exist_ok=True) tmp_path = None @@ -112,7 +115,7 @@ class LocalSkillStorage(SkillStorage): def remove_custom_skill_file(self, name: str, relative_path: str) -> str | None: removal = ((SkillCategory.CUSTOM, Path(name)),) - with self._skill_projection_mutation(remove=removal): + with self._skill_projection_mutation(remove=removal, names=(name,)): return super().remove_custom_skill_file(name, relative_path) async def ainstall_skill_from_archive(self, archive_path: str | Path) -> dict: @@ -205,7 +208,7 @@ class LocalSkillStorage(SkillStorage): """Stage and move the validated skill into place (blocking; runs off the event loop).""" from deerflow.skills.installer import _move_staged_skill_into_reserved_target - with self._skill_projection_mutation(): + with self._skill_projection_mutation(names=(skill_name,)): with tempfile.TemporaryDirectory(prefix=f".installing-{skill_name}-", dir=custom_dir) as staging_root: staging_target = Path(staging_root) / skill_name shutil.copytree(skill_dir, staging_target) @@ -229,7 +232,7 @@ class LocalSkillStorage(SkillStorage): e, ) removal = ((SkillCategory.CUSTOM, Path(name)),) - with self._skill_projection_mutation(remove=removal): + with self._skill_projection_mutation(remove=removal, names=(name,)): if target.is_symlink(): # An operator-linked package (see # ``_is_external_skill_directory_symlink``): the skill is the @@ -244,6 +247,7 @@ class LocalSkillStorage(SkillStorage): *, remove: tuple[tuple[SkillCategory, Path], ...] = (), remove_names: tuple[str, ...] = (), + names: tuple[str, ...] = (), ): if getattr(self, "user_id", None) is None: from deerflow.skills.projection import _projection_lock @@ -251,7 +255,7 @@ class LocalSkillStorage(SkillStorage): return _projection_lock(self.get_skills_root_path() / "custom") from deerflow.skills.projection import skill_projection_mutation - return skill_projection_mutation(self, "user", remove=remove, remove_names=remove_names) + return skill_projection_mutation(self, "user", remove=remove, remove_names=remove_names, mutation_names=names) def append_history(self, name: str, record: dict) -> None: self.validate_skill_name(name) diff --git a/backend/packages/harness/deerflow/skills/storage/user_scoped_skill_storage.py b/backend/packages/harness/deerflow/skills/storage/user_scoped_skill_storage.py index 980c32e730..9355a66cdf 100644 --- a/backend/packages/harness/deerflow/skills/storage/user_scoped_skill_storage.py +++ b/backend/packages/harness/deerflow/skills/storage/user_scoped_skill_storage.py @@ -157,7 +157,7 @@ class UserScopedSkillStorage(LocalSkillStorage): def set_skill_enabled_state(self, skill_name: str, enabled: bool) -> None: """Set the enabled state for a custom/legacy skill and persist.""" removal_names = (skill_name,) if not enabled else () - with self._skill_projection_mutation(remove_names=removal_names): + with self._skill_projection_mutation(remove_names=removal_names, names=(skill_name,)): states = self._read_skill_states() states[skill_name] = {"enabled": enabled} self._write_skill_states(states) @@ -185,6 +185,12 @@ class UserScopedSkillStorage(LocalSkillStorage): # ------------------------------------------------------------------ def load_skills(self, *, enabled_only: bool = False) -> list: + from deerflow.skills.mutations.guard import managed_read + + with managed_read(self): + return self._load_skills_unlocked(enabled_only=enabled_only) + + def _load_skills_unlocked(self, *, enabled_only: bool = False) -> list: """Discover all skills and merge enabled state per isolation scope. Delegates skill discovery and PUBLIC enabled-state to @@ -279,30 +285,33 @@ class UserScopedSkillStorage(LocalSkillStorage): diagnostics cannot establish ownership outside the caller's custom root. Root ancestors must not redirect into another user's storage either. """ - root = self._user_custom_root - expected = self._paths.base_dir.resolve() / "users" / self._user_id / "skills" / "custom" - if root.resolve() != expected or not root.is_dir(): - return [] - diagnostics: list[SkillLoadDiagnostic] = [] - for current, dirs, files in walk_skill_directories(root): - package_dir = Path(current) - if not package_dir.resolve().is_relative_to(expected): - dirs.clear() - continue - dirs[:] = sorted(name for name in dirs if not name.startswith(".")) - if SKILL_MD_FILE not in files: - continue - dirs.clear() - try: - skill_file = (package_dir / SKILL_MD_FILE).resolve(strict=True) - if not skill_file.is_relative_to(expected): + from deerflow.skills.mutations.guard import managed_read + + with managed_read(self): + root = self._user_custom_root + expected = self._paths.base_dir.resolve() / "users" / self._user_id / "skills" / "custom" + if root.resolve() != expected or not root.is_dir(): + return [] + diagnostics: list[SkillLoadDiagnostic] = [] + for current, dirs, files in walk_skill_directories(root): + package_dir = Path(current) + if not package_dir.resolve().is_relative_to(expected): + dirs.clear() continue - # Preserve the logical SKILL.md name for owned internal links. - parse_skill_file(package_dir / SKILL_MD_FILE, SkillCategory.CUSTOM, package_dir.relative_to(root), diagnostics=diagnostics) - except (OSError, RuntimeError): - # A concurrently removed/retargeted package is not a YAML error. - continue - return diagnostics + dirs[:] = sorted(name for name in dirs if not name.startswith(".")) + if SKILL_MD_FILE not in files: + continue + dirs.clear() + try: + skill_file = (package_dir / SKILL_MD_FILE).resolve(strict=True) + if not skill_file.is_relative_to(expected): + continue + # Preserve the logical SKILL.md name for owned internal links. + parse_skill_file(package_dir / SKILL_MD_FILE, SkillCategory.CUSTOM, package_dir.relative_to(root), diagnostics=diagnostics) + except (OSError, RuntimeError): + # A concurrently removed/retargeted package is not a YAML error. + continue + return diagnostics def _iter_skill_files(self) -> Iterable[tuple[SkillCategory, Path, Path]]: # 1. Public skills: always from global root @@ -399,7 +408,7 @@ class UserScopedSkillStorage(LocalSkillStorage): # ------------------------------------------------------------------ def write_custom_skill(self, name: str, relative_path: str, content: str) -> None: - with self._skill_projection_mutation(): + with self._skill_projection_mutation(names=(name,)): target = self.validate_relative_path(relative_path, self.get_custom_skill_dir(name)) target.parent.mkdir(parents=True, exist_ok=True) tmp_path = None diff --git a/backend/packages/harness/deerflow/subagents/AGENTS.md b/backend/packages/harness/deerflow/subagents/AGENTS.md index 5800209dae..f9162cc2e1 100644 --- a/backend/packages/harness/deerflow/subagents/AGENTS.md +++ b/backend/packages/harness/deerflow/subagents/AGENTS.md @@ -164,4 +164,5 @@ truncating its tail. Keep `step_events.py` pure and unit-tested. Loop-detection, deferred-tool-promotion, and tool-progress audit records cross this boundary through one narrow exception. `task_tool` captures the parent loop and passes a `_ParentLoopMiddlewareRecorderProxy` to `SubagentExecutor`, which installs separate context keys and schedules `record_middleware` on the journal's owner loop. The proxy also owns the execution-local atomic promotion claim that deduplicates parallel `tool_search` calls. Other middleware consumers never see `__run_journal`. Close the shared proxy once before `task` returns to fence late events and drain accepted appends before completion capture. Only the proxy crosses loops; do not broaden it or call the event store from a subagent loop. Durable batch subagents have no parent journal and do not use this bridge. -**Reverse direction of the loop boundary — deferred cleanup & final usage delivery (#5069)**: when a task-tool poller exits unexpectedly, the registry cleanup is pinned **to** the persistent subagent loop via the public `run_on_isolated_subagent_loop()` (executor) so it survives caller-loop teardown — `asyncio.run()` cancels caller-loop tasks on exit, so a caller-loop `asyncio.create_task` would be cancelled before running. The final usage report crosses the boundary the **other way**: `_schedule_deferred_subagent_cleanup` captures the parent run's loop at unwind time (alive in every path that continues the run — the polling-timeout branch returns normally, and a generic poller error becomes an error `ToolMessage`), and `_deliver_final_usage_report` hands the report back onto that loop with `call_soon_threadsafe`. `record_external_llm_usage_records` must never be invoked from the persistent loop or a worker thread (`to_thread`): the journal's accumulators are unlocked read-modify-write fields and `get_completion_data()` iterates `_tokens_by_model`, so a cross-thread write silently loses token updates or breaks iteration mid-run — calling `_report_subagent_usage` directly inside `_deferred_cleanup_subagent_task` would reintroduce exactly this race. The deferred cleaner captures only the resolved usage recorder (plus ids and the captured report loop) — never the whole `runtime`: the strongly-referenced cleanup task lives for up to the full poll budget, and through `runtime` it would pin the parent run's journal and event store for that entire window. When the captured parent loop is already closed (synchronous `asyncio.run` teardown), the report is dropped on purpose — the run has persisted its completion data and nothing reads the counters back — and logged at info with the execution id and unaccounted record count, because the registry entry is removed right after and those records exist nowhere else. +**Deferred cleanup and final usage delivery (#5069)**: Unexpected task-tool poller exit schedules registry cleanup on the persistent subagent loop via `run_on_isolated_subagent_loop()`, so caller-loop teardown cannot cancel it. Capture the parent report loop during unwind (polling timeout continues the run; generic poller errors become ToolMessages). `_deliver_final_usage_report` sends final usage back with `call_soon_threadsafe`. +Never call `record_external_llm_usage_records` from the persistent loop or `to_thread`: journal counters use unlocked read-modify-write operations, and completion iterates `_tokens_by_model`. Capture only the resolved usage recorder, IDs and report loop, not the whole `runtime`; cleanup can retain them for the full poll budget. If the parent loop has closed after synchronous `asyncio.run`, drop the report because completion is already persisted. Log the execution ID and unaccounted record count at info before removing the registry entry; those records have no other owner. diff --git a/backend/packages/harness/deerflow/tools/skill_manage_tool.py b/backend/packages/harness/deerflow/tools/skill_manage_tool.py index de19b728c2..29cad15b72 100644 --- a/backend/packages/harness/deerflow/tools/skill_manage_tool.py +++ b/backend/packages/harness/deerflow/tools/skill_manage_tool.py @@ -26,7 +26,7 @@ from deerflow.skills.security_static_scanner import ( enforce_static_scan, ) from deerflow.skills.storage import get_or_new_user_skill_storage -from deerflow.skills.storage.skill_storage import SkillStorage, read_text_or_none +from deerflow.skills.storage.skill_storage import SkillStorage from deerflow.skills.types import SKILL_MD_FILE from deerflow.tools.sync import make_sync_tool_wrapper from deerflow.tools.types import Runtime @@ -155,7 +155,10 @@ async def _scan_static_candidate_or_raise(name: str, updates: dict[str, str], sk if skill_storage is None: skill_dir.mkdir(parents=True) else: - shutil.copytree(skill_storage.get_custom_skill_dir(name), skill_dir) + from deerflow.skills.mutations.guard import managed_read + + with managed_read(skill_storage): + shutil.copytree(skill_storage.get_custom_skill_dir(name), skill_dir) for relative_path, content in updates.items(): target = skill_dir / relative_path target.parent.mkdir(parents=True, exist_ok=True) @@ -229,8 +232,7 @@ async def _skill_manage_impl( static_findings = await _scan_static_candidate_or_raise(name, {SKILL_MD_FILE: content}) scan = await _scan_or_raise(content, executable=False, location=f"{name}/{SKILL_MD_FILE}", static_findings=static_findings) scan["static_findings"] = static_findings - skill_file = skill_storage.get_custom_skill_file(name) - prev_content = await _to_thread(skill_file.read_text, encoding="utf-8") + prev_content = await _to_thread(skill_storage.read_custom_skill, name) await _to_thread(skill_storage.write_custom_skill, name, SKILL_MD_FILE, content) await _to_thread( skill_storage.append_history, @@ -244,8 +246,7 @@ async def _skill_manage_impl( await _to_thread(skill_storage.ensure_custom_skill_is_editable, name) if find is None or replace is None: raise ValueError("find and replace are required for patch.") - skill_file = skill_storage.get_custom_skill_file(name) - prev_content = await _to_thread(skill_file.read_text, encoding="utf-8") + prev_content = await _to_thread(skill_storage.read_custom_skill, name) occurrences = prev_content.count(find) if occurrences == 0: raise ValueError("Patch target not found in SKILL.md.") @@ -289,10 +290,9 @@ async def _skill_manage_impl( target = await _to_thread(skill_storage.ensure_safe_support_path, name, path) if await _to_thread(target.is_dir): raise ValueError(f"Supporting file path '{path}' is a directory, not a file.") - exists = await _to_thread(target.exists) - # A binary asset (e.g. from a .skill archive) has no previous *text*; - # the history record takes None rather than aborting the write. - prev_content = await _to_thread(read_text_or_none, target) if exists else None + from deerflow.skills.mutations.guard import read_optional_text + + prev_content = await _to_thread(read_optional_text, skill_storage, target) executable = "scripts/" in path or path.startswith("scripts/") static_findings = await _scan_static_candidate_or_raise(name, {path: content}, skill_storage) scan = await _scan_or_raise(content, executable=executable, location=f"{name}/{path}", static_findings=static_findings) diff --git a/backend/tests/blocking_io/test_skill_mutation_host.py b/backend/tests/blocking_io/test_skill_mutation_host.py new file mode 100644 index 0000000000..f375a4ccf9 --- /dev/null +++ b/backend/tests/blocking_io/test_skill_mutation_host.py @@ -0,0 +1,122 @@ +"""Strict runtime coverage of the durable host, scanner, evidence and admin API.""" + +import asyncio +import os +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest +from fastapi import FastAPI, Request +from sqlalchemy import create_engine +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from sqlalchemy.orm import Session + +from app.gateway.routers.skill_mutations import router +from deerflow.config.app_config import AppConfig +from deerflow.config.paths import Paths +from deerflow.extensions.host_capabilities import HostCapabilities +from deerflow.persistence.base import Base +from deerflow.persistence.run import RunRepository +from deerflow.persistence.run.model import RunRow +from deerflow.persistence.user.model import UserRow +from deerflow.runtime.events.store.db import DbRunEventStore +from deerflow.skills.security_scanner import ScanResult +from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage + +pytestmark = pytest.mark.skipif(os.name != "posix", reason="Skill publication requires POSIX local storage") + +CONTENT = "---\nname: example\ndescription: A sample skill\n---\nOriginal instructions.\n" + + +@pytest.fixture +def durable_host(tmp_path, monkeypatch): + """Only seed data synchronously; all host lifetime work stays under the gate.""" + monkeypatch.setattr("deerflow.config.paths._paths", Paths(base_dir=tmp_path / "home")) + config = AppConfig.model_validate( + { + "sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}, + "database": {"backend": "sqlite", "sqlite_dir": str(tmp_path)}, + "skill_scan": {"enabled": True}, + "plugins": [ + { + "use": "fixture:install", + "name": "fixture", + "host_access": { + "evidence": {"owners": ["owner"]}, + "skill_mutations": {"owners": ["owner"], "operations": ["stage", "check", "commit", "revert"], "trigger_agents": ["lead_agent"], "target_skills": ["example"], "topology": "single_host_local"}, + }, + } + ], + } + ) + monkeypatch.setattr("deerflow.config.get_app_config", lambda: config) + # Replace only the remote moderation boundary, retaining the real scanner, + # package staging/static scan, publication, projection and SQLite paths. + model = AsyncMock(return_value=ScanResult("allow", "safe")) + monkeypatch.setattr("deerflow.skills.mutations.scanner.scan_skill_content", model) + storage = UserScopedSkillStorage("owner", host_path=str(tmp_path / "skills"), app_config=config) + root = storage.get_custom_skill_dir("example") + root.mkdir(parents=True) + (root / "SKILL.md").write_text(CONTENT, encoding="utf-8") + engine = create_engine(config.database.app_sync_sqlalchemy_url) + try: + Base.metadata.create_all(engine) + with Session(engine) as session, session.begin(): + session.add(UserRow(id="owner", email="owner@example.test")) + session.add(RunRow(run_id="run", thread_id="thread", user_id="owner", status="success", evidence_seal_state="sealed", evidence_agent_id="lead_agent", evidence_origin="interactive", evidence_upper_seq=2, evidence_event_count=2)) + finally: + engine.dispose() + return SimpleNamespace(config=config, storage=storage, model=model, root=tmp_path, database_url=config.database.app_sqlalchemy_url, skill_file=root / "SKILL.md") + + +@pytest.mark.asyncio +async def test_host_check_evidence_publication_revert_and_admin_are_off_loop(durable_host): + seed = durable_host + engine = create_async_engine(seed.database_url) + sf = async_sessionmaker(engine, expire_on_commit=False) + events = DbRunEventStore(sf) + host = HostCapabilities(seed.config, RunRepository(sf), events, root=seed.root, storage_factory=lambda _: seed.storage) + try: + await events.put_batch([dict(thread_id="thread", run_id="run", user_id="owner", event_type="test", category="message", content=text) for text in ("first", "second")]) + await host.start() + evidence, mutations = host.bindings["fixture:install"] + assert (await evidence.list_changed_runs()).items[0].run_id == "run" + snapshot = await evidence.get_snapshot(thread_id="thread", run_id="run") + from deerflow_extension_api import EvidenceLimits + + page = await evidence.read_events(snapshot_ref=snapshot.snapshot_ref, limits=EvidenceLimits(max_events=1)) + assert len(page.items) == 1 and page.has_more + next_page = await evidence.read_events(snapshot_ref=snapshot.snapshot_ref, cursor=page.next_cursor, limits=EvidenceLimits(max_events=1)) + assert len(next_page.items) == 1 and not next_page.has_more + assert next_page.items[0].seq > page.items[0].seq + skill = await mutations.read_skill(source_ref=snapshot.snapshot_ref, name="example") + proposal = await mutations.stage(source_refs=(snapshot.snapshot_ref,), target_ref=skill.target_ref, expected_revision=skill.revision, content=CONTENT.replace("Original", "Updated"), idempotency_key="stage") + result = await mutations.check(proposal_id=proposal.proposal_id) + assert result.decision == "allow" + seed.model.assert_awaited_once() + operation = await mutations.commit(proposal_id=proposal.proposal_id, idempotency_key="commit") + assert operation.publication == "APPLIED" and operation.views == "READY" + + app = FastAPI() + app.state.skill_mutation_host = host + + @app.middleware("http") + async def identity(request: Request, call_next): + request.state.user = SimpleNamespace(id="admin", system_role="admin") + return await call_next(request) + + app.include_router(router) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client: + url = f"/api/skill-mutations/operations/{operation.operation_id}" + assert (await client.get("/api/skill-mutations/operations")).status_code == 200 + assert (await client.get(url)).json()["publication"] == "APPLIED" + assert (await client.post(url + "/recover")).json()["views"] == "READY" + assert (await client.post("/api/skill-mutations/owners/owner/recover")).status_code == 200 + reverted = await mutations.revert(operation_id=operation.operation_id, expected_current_revision=operation.after_revision, idempotency_key="revert") + assert reverted.publication == "APPLIED" and reverted.views == "READY" + content = await asyncio.to_thread(seed.skill_file.read_text, encoding="utf-8") + assert content == CONTENT + finally: + await host.close() + await engine.dispose() diff --git a/backend/tests/fixtures/skill_evolution_consumer.py b/backend/tests/fixtures/skill_evolution_consumer.py new file mode 100644 index 0000000000..7da062d4c2 --- /dev/null +++ b/backend/tests/fixtures/skill_evolution_consumer.py @@ -0,0 +1,25 @@ +"""External-consumer contract fixture: imports only the standalone public API.""" + +from deerflow_extension_api import ExtensionRuntimeDeps + + +async def exercise(deps: ExtensionRuntimeDeps, *, thread_id: str, run_id: str, name: str): + evidence, skills = deps.completed_run_evidence, deps.skill_mutations + assert evidence is not None and skills is not None + snapshot = await evidence.get_snapshot(thread_id=thread_id, run_id=run_id) + assert snapshot.seal_state == "sealed" + await evidence.read_events(snapshot_ref=snapshot.snapshot_ref) + skill = await skills.read_skill(source_ref=snapshot.snapshot_ref, name=name) + proposal = await skills.stage(source_refs=(snapshot.snapshot_ref,), target_ref=skill.target_ref, expected_revision=skill.revision, content=skill.content + "\nA fixed test improvement.\n", idempotency_key="fixture-stage") + bundle = await skills.read_proposal_bundle(proposal_id=proposal.proposal_id) + assert bundle.base_revision == skill.revision + result = await skills.check(proposal_id=proposal.proposal_id) + assert result.decision == "allow" + operation = await skills.commit(proposal_id=proposal.proposal_id, idempotency_key="fixture-commit") + assert operation.publication == "APPLIED" + assert await skills.get_operation(operation_id=operation.operation_id) == operation + assert await skills.find_operation(idempotency_key="fixture-commit") == operation + reverted = await skills.revert(operation_id=operation.operation_id, expected_current_revision=operation.after_revision, idempotency_key="fixture-revert") + assert reverted.publication == "APPLIED" + assert reverted.after_revision.mutation_seq > operation.after_revision.mutation_seq + return operation, reverted diff --git a/backend/tests/fixtures/skill_mutation_crash_worker.py b/backend/tests/fixtures/skill_mutation_crash_worker.py new file mode 100644 index 0000000000..01231774d3 --- /dev/null +++ b/backend/tests/fixtures/skill_mutation_crash_worker.py @@ -0,0 +1,80 @@ +"""Fresh process terminated by SIGKILL at a selected publication boundary.""" + +import asyncio +import json +import os +import signal +import stat +import sys +from types import SimpleNamespace + +from deerflow.config import paths +from deerflow.config.paths import Paths +from deerflow.extensions.completed_run_evidence import HostCompletedRunEvidenceReader +from deerflow.extensions.host_access import BoundHostAccess, HostAccess +from deerflow.skills.mutations import publication +from deerflow.skills.mutations import service as service_module +from deerflow.skills.mutations.guard import SkillMutationRuntime, configure_mutation_runtime +from deerflow.skills.mutations.recovery import SkillMutationRecovery +from deerflow.skills.mutations.repository import SkillMutationRepository +from deerflow.skills.mutations.topology import mutation_session_factory +from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage + + +def die(): + os.kill(os.getpid(), signal.SIGKILL) + # Signal delivery is asynchronous: never return to a pre-rename fault + # injection site if the process is still briefly scheduled. + while True: + signal.pause() + + +async def main(args): + paths._paths = Paths(base_dir=args["base"]) + engine, sessions = mutation_session_factory(args["database"]) + storage = UserScopedSkillStorage("owner", host_path=args["skills"]) + runtime = SkillMutationRuntime(SkillMutationRepository(sessions), owners=frozenset({"owner"})) + configure_mutation_runtime(runtime) + recovery = SkillMutationRecovery(runtime, lambda _: storage, rebuild_views=lambda _: None) + binding = BoundHostAccess("plugin", HostAccess.model_validate(args["grant"])) + evidence = SimpleNamespace(scope_digest="scope", revision_for_run=HostCompletedRunEvidenceReader.revision_for_run) + service = service_module.HostSkillMutationService(binding, runtime=runtime, evidence=evidence, storage_factory=lambda _: storage, scanner=SimpleNamespace(policy_version_sync=lambda: "policy-1"), recovery=recovery) + boundary = args["boundary"] + if boundary == "prepared": + service_module.publish_prepared = lambda *_: die() + elif boundary == "written": + write = os.write + + def write_then_die(fd, content): + write(fd, content) + die() + + publication.os.write = write_then_die + elif boundary in {"file_synced", "directory_synced"}: + fsync = os.fsync + + def sync_then_die(fd): + fsync(fd) + mode = os.fstat(fd).st_mode + if (boundary == "file_synced" and stat.S_ISREG(mode)) or (boundary == "directory_synced" and stat.S_ISDIR(mode)): + die() + + publication.os.fsync = sync_then_die + elif boundary == "renamed": + replace = os.replace + + def replace_then_die(*args, **kwargs): + replace(*args, **kwargs) + die() + + publication.os.replace = replace_then_die + elif boundary == "response_lost": + recovery.refresh_views = lambda _: die() + await service.commit(proposal_id=args["proposal"], idempotency_key="crash-commit") + await service.close() + engine.dispose() + raise AssertionError("fault injection did not fire") + + +if __name__ == "__main__": + asyncio.run(main(json.loads(sys.argv[1]))) diff --git a/backend/tests/test_client.py b/backend/tests/test_client.py index 75916600a2..3b8bbd6ebc 100644 --- a/backend/tests/test_client.py +++ b/backend/tests/test_client.py @@ -9,6 +9,7 @@ import stat import tempfile import threading import zipfile +from contextlib import contextmanager from enum import Enum from pathlib import Path from types import SimpleNamespace @@ -31,6 +32,7 @@ from deerflow.agents.thread_state import DeltaThreadState, ThreadState from deerflow.client import DeerFlowClient, StreamEvent from deerflow.config.agents_config import AgentConfig from deerflow.config.authorization_config import AuthorizationConfig, AuthorizationProviderConfig +from deerflow.config.database_config import DatabaseConfig from deerflow.config.extensions_config import ExtensionsConfig, McpServerConfig from deerflow.config.paths import Paths from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig @@ -61,8 +63,7 @@ def mock_app_config(): config.skills.deferred_discovery = False config.skills.container_path = "/mnt/skills" config.tool_search.enabled = False - config.database.checkpoint_channel_mode = "full" - config.database.checkpoint_delta.snapshot_frequency = 10 + config.database = DatabaseConfig(backend="memory") config.authorization = AuthorizationConfig(enabled=False) return config @@ -2786,6 +2787,46 @@ class TestSkillsManagement: finally: tmp_path.unlink() + def test_update_public_skill_enters_managed_write_before_projection(self, client, tmp_path): + from deerflow.skills import projection + from deerflow.skills.mutations import guard + + skill = self._make_skill(enabled=True) + updated_skill = self._make_skill(enabled=False) + config_file = tmp_path / "extensions_config.json" + config_file.write_text('{"skills": {}}', encoding="utf-8") + events = [] + + @contextmanager + def record_managed_write(*_args, **_kwargs): + events.append("managed-enter") + try: + yield + finally: + events.append("managed-exit") + + @contextmanager + def record_projection(*_args, **_kwargs): + events.append("projection-enter") + try: + yield + finally: + events.append("projection-exit") + + with ( + patch( + "deerflow.skills.storage.local_skill_storage.LocalSkillStorage.load_skills", + side_effect=[[skill], [skill], [updated_skill], [updated_skill]], + ), + patch("deerflow.client.ExtensionsConfig.resolve_config_path", return_value=config_file), + patch("deerflow.client.reload_extensions_config"), + patch.object(guard, "managed_global_state_write", record_managed_write), + patch.object(projection, "skill_projection_mutation", record_projection), + ): + client.update_skill("test-skill", enabled=False) + + assert events == ["managed-enter", "projection-enter", "projection-exit", "managed-exit"] + def test_update_skill_persists_state_when_source_omits_skills(self, client): skill = self._make_skill(enabled=True) updated_skill = self._make_skill(enabled=False) @@ -2833,6 +2874,7 @@ class TestSkillsManagement: skill = self._make_skill(enabled=True) skill.category = category storage = MagicMock() + storage._app_config = client._app_config storage.load_skills.side_effect = [[skill], [self._make_skill(enabled=False)]] with ( diff --git a/backend/tests/test_completed_evidence_admission.py b/backend/tests/test_completed_evidence_admission.py new file mode 100644 index 0000000000..e85bd071e3 --- /dev/null +++ b/backend/tests/test_completed_evidence_admission.py @@ -0,0 +1,156 @@ +"""Only host entry points may label the provenance used by automatic publication.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.store.memory import InMemoryStore + +from app.gateway.auth_disabled import AUTH_SOURCE_INTERNAL +from app.gateway.routers.thread_runs import RunCreateRequest +from app.gateway.services import launch_scheduled_thread_run, start_run +from deerflow.config.app_config import AppConfig, reset_app_config, set_app_config +from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore +from deerflow.runtime.events.store.memory import MemoryRunEventStore +from deerflow.runtime.runs.manager import RunManager +from deerflow.runtime.runs.store.memory import MemoryRunStore + + +@pytest.fixture(autouse=True) +def app_config(): + set_app_config(AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}})) + yield + reset_app_config() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "auth_source, host_origin, expected", + [(None, None, "interactive"), (AUTH_SOURCE_INTERNAL, None, "unknown")] + [(AUTH_SOURCE_INTERNAL, origin, origin) for origin in ("interactive", "scheduled", "extension_evaluation", "unknown")], +) +async def test_client_metadata_cannot_forge_completed_evidence_origin(auth_source, host_origin, expected): + manager = RunManager(store=MemoryRunStore()) + request = SimpleNamespace( + headers={}, + state=SimpleNamespace(auth_source=auth_source), + app=SimpleNamespace( + state=SimpleNamespace( + run_manager=manager, + stream_bridge=SimpleNamespace(), + checkpointer=InMemorySaver(), + store=InMemoryStore(), + run_event_store=MemoryRunEventStore(), + run_events_config=None, + thread_store=MemoryThreadMetaStore(InMemoryStore()), + ) + ), + ) + body = RunCreateRequest( + input={"messages": [{"role": "user", "content": "hello"}]}, + metadata={"evidence_origin": "scheduled", "scheduled_task_id": "forged"}, + config={"context": {"evidence_origin": "interactive"}, "configurable": {"evidence_origin": "scheduled"}}, + context={"evidence_origin": "interactive"}, + ) + with ( + patch("app.gateway.services.resolve_agent_factory", return_value=object()), + patch("app.gateway.services.run_agent", new=AsyncMock()), + patch.object(manager, "create_or_reject", wraps=manager.create_or_reject) as admission, + ): + record = await start_run(body, "evidence-origin-test", request, evidence_origin=host_origin) + await record.task + assert admission.call_args.kwargs["evidence_origin"] == expected + + +@pytest.mark.asyncio +async def test_scheduler_passes_origin_outside_request_metadata(): + admitted = AsyncMock(return_value=SimpleNamespace(run_id="r1", thread_id="t1")) + with patch("app.gateway.services.start_run", new=admitted): + await launch_scheduled_thread_run(thread_id="t1", assistant_id="lead_agent", prompt="hello", app=SimpleNamespace(), owner_user_id="owner") + assert admitted.call_args.kwargs["evidence_origin"] == "scheduled" + + +@pytest.mark.asyncio +async def test_evidence_agent_is_effective_canonical_id_not_routing_or_display_name(): + manager = RunManager(store=MemoryRunStore()) + request = SimpleNamespace( + headers={}, + state=SimpleNamespace(), + app=SimpleNamespace( + state=SimpleNamespace( + run_manager=manager, + stream_bridge=SimpleNamespace(), + checkpointer=InMemorySaver(), + store=InMemoryStore(), + run_event_store=MemoryRunEventStore(), + run_events_config=None, + thread_store=MemoryThreadMetaStore(InMemoryStore()), + ) + ), + ) + from deerflow.config.agents_config import AgentConfig + + agent = AgentConfig(name="researcher", display_name="My research helper") + body = RunCreateRequest(assistant_id="lead_agent", input={"messages": [{"role": "user", "content": "hello"}]}, context={"agent_name": "researcher"}, metadata={"evidence_agent_id": "forged"}) + with ( + patch("app.gateway.services.resolve_agent_factory", return_value=object()), + patch("app.gateway.services._load_scope_agent_config", new=AsyncMock(return_value=agent)), + patch("app.gateway.services.run_agent", new=AsyncMock()), + patch.object(manager, "create_or_reject", wraps=manager.create_or_reject) as admission, + ): + record = await start_run(body, "effective-agent-test", request) + await record.task + assert admission.call_args.kwargs["evidence_agent_id"] == "researcher" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("backend", ["memory", "sqlite"]) +async def test_keyed_resume_peer_reuse_preserves_evidence_identity(tmp_path, backend): + from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + + from deerflow.persistence.base import Base + from deerflow.persistence.run.sql import RunRepository + + engine = None + if backend == "sqlite": + engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'admission.db'}") + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + store = RunRepository(async_sessionmaker(engine, expire_on_commit=False)) + else: + store = MemoryRunStore() + identity = {"version": 1, "kind": "resume", "sha256": "a" * 64} + try: + owner = RunManager(store=store) + original = await owner.create_or_reject( + "resume-thread", + "lead_agent", + user_id="alice", + idempotency_key="resume-key", + idempotency_request=identity, + evidence_origin="extension_evaluation", + evidence_agent_id="researcher", + ) + peer = RunManager(store=store) + reused = await peer.create_or_reject( + "resume-thread", + "lead_agent", + user_id="alice", + idempotency_key="resume-key", + idempotency_request=identity, + evidence_origin="interactive", + evidence_agent_id="different-agent", + ) + assert reused.run_id == original.run_id + assert reused.idempotency_reused + assert reused.store_only + assert reused.idempotency_request == identity + assert reused.evidence_origin == "extension_evaluation" + assert reused.evidence_agent_id == "researcher" + persisted = await store.get(original.run_id, user_id="alice") + assert persisted["idempotency_request"] == identity + assert persisted["evidence_origin"] == "extension_evaluation" + assert persisted["evidence_agent_id"] == "researcher" + finally: + if engine is not None: + await engine.dispose() diff --git a/backend/tests/test_completed_run_evidence.py b/backend/tests/test_completed_run_evidence.py new file mode 100644 index 0000000000..802085d54c --- /dev/null +++ b/backend/tests/test_completed_run_evidence.py @@ -0,0 +1,895 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import FrozenInstanceError, asdict +from datetime import UTC, datetime, timedelta + +import pytest +import pytest_asyncio +from sqlalchemy import event, select, update +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from deerflow.persistence.base import Base +from deerflow.persistence.models.run_event import RunEventRow +from deerflow.persistence.run.model import CompletedRunSnapshotRow, RunRow +from deerflow.persistence.run.sql import RunRepository +from deerflow.runtime.events.store.db import DbRunEventStore +from deerflow.runtime.events.store.memory import MemoryRunEventStore +from deerflow.runtime.journal import RunJournal + + +@pytest_asyncio.fixture +async def evidence(tmp_path): + from deerflow.extensions.completed_run_evidence import HostCompletedRunEvidenceReader + + engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'evidence.db'}") + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + sf = async_sessionmaker(engine, expire_on_commit=False) + events = DbRunEventStore(sf) + runs = RunRepository(sf) + reader = HostCompletedRunEvidenceReader(runs, events, plugin_id="test", owner_ids=frozenset({"alice"})) + yield sf, events, runs, reader + await engine.dispose() + + +async def put_run(runs, run_id="r", *, owner="alice", status="success", **kwargs): + await runs.put(run_id, thread_id=f"t-{run_id}", user_id=owner, assistant_id="routing-alias", evidence_agent_id="stable-agent", status=status, **kwargs) + + +async def put_event(events, run_id="r", content="hello", owner="alice"): + return await events.put_batch([dict(thread_id=f"t-{run_id}", run_id=run_id, user_id=owner, event_type="test", category="message", content=content)]) + + +@pytest.mark.asyncio +async def test_source_fence_metadata_has_explicit_accessors(evidence): + sf, _events, runs, reader = evidence + await put_run(runs) + snapshot = await reader.get_snapshot(thread_id="t-r", run_id="r") + async with sf() as session: + run = await session.scalar(select(RunRow).where(RunRow.run_id == "r")) + assert len(reader.scope_digest) == 64 + assert reader.revision_for_run(run) == snapshot.evidence_revision + + +@pytest.mark.asyncio +async def test_new_giant_digest_has_constant_read_queries(evidence): + sf, events, runs, reader = evidence + await put_run(runs) + content = "x" * (16 * 1024 * 1024) + await put_event(events, content=content) + snap = await reader.get_snapshot(thread_id="t-r", run_id="r") + queries = [] + engine = sf.kw["bind"].sync_engine + + def track(_conn, _cursor, statement, _parameters, _context, _many): + queries.append(statement) + + event.listen(engine, "before_cursor_execute", track) + try: + page = await reader.read_events(snapshot_ref=snap.snapshot_ref) + finally: + event.remove(engine, "before_cursor_execute", track) + assert page.items[0].content_sha256 == hashlib.sha256(content.encode()).hexdigest() + assert len(queries) <= 6 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sizes", [[1024 * 1024 + 1], [600 * 1024, 600 * 1024]]) +async def test_legacy_digest_budget_is_cumulative_and_never_trusts_metadata(evidence, sizes): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + sf, events, runs, reader = evidence + await put_run(runs) + async with sf() as session: + session.add_all([RunEventRow(thread_id="t-r", run_id="r", user_id="alice", event_type="test", category="message", content="x" * size, event_metadata={"content_sha256": "fake"}, seq=i + 1) for i, size in enumerate(sizes)]) + await session.commit() + snap = await reader.get_snapshot(thread_id="t-r", run_id="r") + queries = [] + engine = sf.kw["bind"].sync_engine + + def track(_conn, _cursor, statement, _parameters, _context, _many): + queries.append(statement) + + event.listen(engine, "before_cursor_execute", track) + try: + with pytest.raises(HostCapabilityError, match="LIMIT_EXCEEDED"): + await reader.read_events(snapshot_ref=snap.snapshot_ref) + finally: + event.remove(engine, "before_cursor_execute", track) + assert len(queries) <= 72 + + +@pytest.mark.asyncio +async def test_legacy_digest_within_budget_is_exact(evidence): + sf, events, runs, reader = evidence + await put_run(runs) + content = "中\x00" * 10000 + async with sf() as session: + session.add(RunEventRow(thread_id="t-r", run_id="r", user_id="alice", event_type="test", category="message", content=content, seq=1)) + await session.commit() + snap = await reader.get_snapshot(thread_id="t-r", run_id="r") + from deerflow_extension_api.completed_run_evidence import EvidenceLimits + + page = await reader.read_events(snapshot_ref=snap.snapshot_ref, limits=EvidenceLimits(max_bytes=1024)) + assert page.items[0].content_sha256 == hashlib.sha256(content.encode()).hexdigest() + + +@pytest.mark.asyncio +async def test_legacy_digest_deadline_interrupts_query_wait(evidence, monkeypatch): + import asyncio + + from deerflow_extension_api.host_capabilities import HostCapabilityError + from sqlalchemy.ext.asyncio import AsyncSession + + from deerflow.extensions import completed_run_evidence as module + + sf, events, runs, reader = evidence + await put_run(runs) + async with sf() as session: + session.add(RunEventRow(thread_id="t-r", run_id="r", user_id="alice", event_type="test", category="message", content="x" * 200000, seq=1)) + await session.commit() + snap = await reader.get_snapshot(thread_id="t-r", run_id="r") + clock = iter([0, 1.999]) + monkeypatch.setattr(module, "monotonic", lambda: next(clock)) + original = AsyncSession.scalar + + async def delayed(session, statement, *args, **kwargs): + if "substr(" in str(statement): + await asyncio.sleep(0.02) + return await original(session, statement, *args, **kwargs) + + monkeypatch.setattr(AsyncSession, "scalar", delayed) + with pytest.raises(HostCapabilityError, match="LIMIT_EXCEEDED.*deadline"): + await reader.read_events(snapshot_ref=snap.snapshot_ref) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["put", "put_batch", "put_if_absent"]) +async def test_ingestion_digest_is_of_persisted_truncated_serialization(evidence, method): + sf, events, runs, reader = evidence + payload = dict(thread_id="t-r", run_id="r", event_type="test", category="trace", content={"text": "中" * 20000}) + if method == "put_batch": + await events.put_batch([payload]) + else: + await getattr(events, method)(**payload) + async with sf() as session: + row = await session.scalar(select(RunEventRow)) + assert getattr(row, "content_sha256", None) == hashlib.sha256(row.content.encode("utf-8")).hexdigest() + + +@pytest.mark.asyncio +async def test_thread_snapshot_cleanup_locks_runs_in_prior_statement(evidence): + sf, events, runs, reader = evidence + await put_run(runs) + await reader.get_snapshot(thread_id="t-r", run_id="r") + statements = [] + engine = sf.kw["bind"].sync_engine + + def track(_conn, _cursor, statement, _parameters, _context, _many): + statements.append(statement) + + event.listen(engine, "before_cursor_execute", track) + try: + await runs.delete_by_thread("t-r", user_id="alice") + finally: + event.remove(engine, "before_cursor_execute", track) + lock_indices = [i for i, statement in enumerate(statements) if statement.startswith("SELECT") and "runs.run_id" in statement] + assert lock_indices, "run locks must be acquired in their own statement" + cleanup_index = next(i for i, statement in enumerate(statements) if statement.startswith("DELETE FROM completed_run_snapshots")) + assert lock_indices[0] < cleanup_index + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", ["run", "thread", "run_events", "thread_events"]) +async def test_snapshot_cleanup_is_owner_scoped(evidence, operation): + from deerflow.extensions.completed_run_evidence import HostCompletedRunEvidenceReader + + sf, events, runs, reader = evidence + await put_run(runs) + await runs.put("b", thread_id="t-r", user_id="bob", status="success") + bob = HostCompletedRunEvidenceReader(runs, events, plugin_id="test", owner_ids={"bob"}) + alice_snapshot = await reader.get_snapshot(thread_id="t-r", run_id="r") + bob_snapshot = await bob.get_snapshot(thread_id="t-r", run_id="b") + + async def remove(owner): + if operation == "run": + await runs.delete("r", user_id=owner) + elif operation == "thread": + await runs.delete_by_thread("t-r", user_id=owner) + elif operation == "run_events": + await events.delete_by_run("t-r", "r", user_id=owner) + else: + await events.delete_by_thread("t-r", user_id=owner) + + await remove("mallory") + async with sf() as session: + assert set(await session.scalars(select(CompletedRunSnapshotRow.snapshot_ref))) == {alice_snapshot.snapshot_ref, bob_snapshot.snapshot_ref} + await remove("alice") + async with sf() as session: + assert set(await session.scalars(select(CompletedRunSnapshotRow.snapshot_ref))) == {bob_snapshot.snapshot_ref} + + +def test_new_dtos_detach_mutable_constructor_sequences(): + from deerflow_extension_api.completed_run_evidence import CompletedRunEventPage, CompletedRunSnapshot + + notes = ["limited"] + items = [] + snapshot = CompletedRunSnapshot(coverage_limits=notes, skill_observations=notes) + page = CompletedRunEventPage(items=items) + notes.append("changed") + items.append("changed") + assert snapshot.coverage_limits == ("limited",) + assert snapshot.skill_observations == ("limited",) + assert page.items == () + + +@pytest.mark.asyncio +async def test_scope_filters_before_pagination_and_cursor_binds_plugin(evidence): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + from deerflow.extensions.completed_run_evidence import HostCompletedRunEvidenceReader + + sf, events, runs, reader = evidence + await put_run(runs, "other", owner="bob") + await put_run(runs, "a") + await put_run(runs, "b") + first = await reader.list_changed_runs(cursor=None, limit=1) + assert [r.run_id for r in first.items] == ["a"] + assert first.has_more + await runs.update_model_name("a", "changed") + second = await reader.list_changed_runs(cursor=first.next_cursor, limit=10) + assert [r.run_id for r in second.items] == ["b", "a"] + peer = HostCompletedRunEvidenceReader(runs, events, plugin_id="other", owner_ids={"alice"}) + with pytest.raises(HostCapabilityError, match="INVALID_CURSOR"): + await peer.list_changed_runs(cursor=first.next_cursor, limit=1) + with pytest.raises(HostCapabilityError, match="NOT_FOUND"): + await reader.get_snapshot(thread_id="t-other", run_id="other") + + +@pytest.mark.parametrize("owners", [None, set(), {None}, {"*"}, {""}]) +def test_explicit_owner_scope_is_required(owners): + from deerflow.extensions.completed_run_evidence import HostCompletedRunEvidenceReader + + with pytest.raises(ValueError): + HostCompletedRunEvidenceReader(None, MemoryRunEventStore(), plugin_id="test", owner_ids=owners) + + +@pytest.mark.asyncio +async def test_unsupported_is_explicit(): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + from deerflow.extensions.completed_run_evidence import HostCompletedRunEvidenceReader + + reader = HostCompletedRunEvidenceReader(None, MemoryRunEventStore(), plugin_id="test", owner_ids={"alice"}) + with pytest.raises(HostCapabilityError, match="UNSUPPORTED"): + await reader.list_changed_runs(cursor=None, limit=1) + + +@pytest.mark.asyncio +async def test_snapshot_fixed_boundaries_and_retention_invalidation(evidence): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + sf, events, runs, reader = evidence + await put_run(runs) + await put_event(events, content={"text": "first"}) + snap = await reader.get_snapshot(thread_id="t-r", run_id="r") + assert snap.owner_id == "alice" + assert snap.agent_id == "stable-agent" + assert snap.origin == "unknown" + assert snap.status == "success" + assert snap.seal_state == "partial" + assert snap.coverage == "lead-journal-v1" + assert snap.coverage_limits + assert snap.event_count == 1 + assert (await reader.get_snapshot(thread_id="t-r", run_id="r")).snapshot_ref == snap.snapshot_ref + late = await put_event(events, content="late") + page = await reader.read_events(snapshot_ref=snap.snapshot_ref, cursor=None) + assert len(page.items) == 1 + assert not page.has_more + assert json.loads(page.items[0].content_json) == {"text": "first"} + with pytest.raises(FrozenInstanceError): + snap.owner_id = "bob" + await events.delete_by_run("t-r", "r", user_id="alice") + # Reopen the store to ensure deletion preserved the durable sequence floor. + replacement = await put_event(DbRunEventStore(sf), content="replacement") + assert replacement[0]["seq"] > late[0]["seq"] + with pytest.raises(HostCapabilityError, match="NOT_FOUND"): + await reader.read_events(snapshot_ref=snap.snapshot_ref, cursor=None) + + +@pytest.mark.asyncio +async def test_every_page_reauthorizes_current_owner(evidence): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + sf, events, runs, reader = evidence + await put_run(runs) + await put_event(events) + snap = await reader.get_snapshot(thread_id="t-r", run_id="r") + async with sf() as session: + await session.execute(update(RunRow).where(RunRow.run_id == "r").values(user_id="bob")) + await session.commit() + with pytest.raises(HostCapabilityError, match="NOT_FOUND"): + await reader.read_events(snapshot_ref=snap.snapshot_ref, cursor=None) + + +@pytest.mark.asyncio +async def test_internal_resolver_rechecks_current_retention_without_reading_events(evidence): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + sf, events, runs, reader = evidence + await put_run(runs) + snap = await reader.get_snapshot(thread_id="t-r", run_id="r") + assert await reader.resolve_snapshot(snap.snapshot_ref) == snap + await events.delete_by_run("t-r", "r", user_id="alice") + with pytest.raises(HostCapabilityError, match="NOT_FOUND"): + await reader.resolve_snapshot(snap.snapshot_ref) + + +@pytest.mark.asyncio +async def test_giant_event_preview_fits_envelope_and_advances(evidence): + from deerflow_extension_api.completed_run_evidence import EvidenceLimits + + sf, events, runs, reader = evidence + await put_run(runs) + await put_event(events, content="巨大" * 100000) + await put_event(events, content="last") + snap = await reader.get_snapshot(thread_id="t-r", run_id="r") + limits = EvidenceLimits(max_events=1, max_bytes=4096) + first = await reader.read_events(snapshot_ref=snap.snapshot_ref, cursor=None, limits=limits) + assert first.has_more + assert first.items[0].truncated + assert first.items[0].content_bytes == 600000 + assert len(first.items[0].content_sha256) == 64 + assert len(json.dumps(asdict(first), ensure_ascii=False, separators=(",", ":")).encode()) <= 4096 + second = await reader.read_events(snapshot_ref=snap.snapshot_ref, cursor=first.next_cursor, limits=limits) + assert json.loads(second.items[0].content_json) == "last" + assert not second.has_more + + +@pytest.mark.asyncio +async def test_content_hash_and_body_preserve_embedded_null(evidence): + import hashlib + + sf, events, runs, reader = evidence + await put_run(runs) + content = "before\x00after" + await put_event(events, content=content) + snap = await reader.get_snapshot(thread_id="t-r", run_id="r") + page = await reader.read_events(snapshot_ref=snap.snapshot_ref) + assert json.loads(page.items[0].content_json) == content + assert page.items[0].content_sha256 == hashlib.sha256(content.encode()).hexdigest() + + +@pytest.mark.asyncio +async def test_seal_requires_drain_and_current_owner(evidence): + sf, events, runs, reader = evidence + await put_run(runs, owner_worker_id="worker", lease_expires_at=(datetime.now(UTC) + timedelta(minutes=5)).isoformat()) + journal = RunJournal("r", "t-r", events, user_id="alice") + journal.record_middleware("test", name="test", hook="test", action="test", changes={}) + receipt = await journal.finish_evidence() + assert not journal._closed + journal.record_middleware("late", name="late", hook="late", action="late", changes={}) + assert await runs.seal_evidence("r", owner_worker_id="wrong", receipt=receipt) is False + assert await runs.seal_evidence("r", owner_worker_id="worker", receipt=receipt) is True + snap = await reader.get_snapshot(thread_id="t-r", run_id="r") + assert snap.seal_state == "sealed" + assert snap.event_count == 1 + assert snap.upper_event_seq == 1 + await journal.close() + + +@pytest.mark.asyncio +async def test_proven_not_committed_flush_never_issues_seal_and_close_can_retry(): + from deerflow.runtime.events.store.base import RunEventStoreNotCommittedError + + class FailingStore(MemoryRunEventStore): + fail = True + + async def put_batch(self, events): + if self.fail: + raise RunEventStoreNotCommittedError("write failed") + return await super().put_batch(events) + + store = FailingStore() + journal = RunJournal("r", "t", store) + journal.record_middleware("test", name="test", hook="test", action="test", changes={}) + with pytest.raises(RuntimeError, match="write failed"): + await journal.finish_evidence() + assert not journal._closed + assert journal._buffer + store.fail = False + await journal.close() + assert journal._closed + assert len(await store.list_events("t", "r")) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", ["success", "error", "interrupted", "timeout"]) +async def test_legacy_terminal_run_is_partial_even_with_delivery(evidence, status): + sf, events, runs, reader = evidence + await put_run(runs, status=status) + async with sf() as session: + await session.execute(update(RunRow).where(RunRow.run_id == "r").values(evidence_seal_state=None)) + await session.commit() + await events.put(thread_id="t-r", run_id="r", event_type="run.delivery", category="outputs", content={}) + snap = await reader.get_snapshot(thread_id="t-r", run_id="r") + assert snap.status == status + assert snap.seal_state == "partial" + + +@pytest.mark.asyncio +async def test_admission_origin_survives_hydration_and_ignores_metadata(evidence): + from deerflow.runtime.runs.manager import RunManager + + sf, events, runs, reader = evidence + manager = RunManager(store=runs) + first = await manager.create_or_reject("t", "stable", user_id="alice", evidence_origin="interactive", metadata={"evidence_origin": "scheduled"}) + row = await runs.get(first.run_id, user_id="alice") + assert row["evidence_origin"] == "interactive" + assert manager._record_from_store(row).evidence_origin == "interactive" + with pytest.raises(ValueError, match="origin"): + await manager.create("bad", user_id="alice", evidence_origin="untrusted") + + +@pytest.mark.asyncio +async def test_expired_lease_cannot_seal(evidence): + sf, events, runs, reader = evidence + await put_run(runs, owner_worker_id="worker", lease_expires_at=(datetime.now(UTC) - timedelta(seconds=1)).isoformat()) + journal = RunJournal("r", "t-r", events) + receipt = await journal.finish_evidence() + assert not await runs.seal_evidence("r", owner_worker_id="worker", receipt=receipt) + assert (await reader.get_snapshot(thread_id="t-r", run_id="r")).seal_state != "sealed" + await journal.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", ["success", "error", "interrupted", "timeout"]) +async def test_terminal_unsealed_crash_recovery_advances_discovery(evidence, status): + from deerflow.runtime.runs.manager import RunManager + + sf, events, runs, reader = evidence + expired = (datetime.now(UTC) - timedelta(minutes=5)).isoformat() + await put_run(runs, "crash", status=status, owner_worker_id="dead", lease_expires_at=expired) + await put_run(runs, "live", owner_worker_id="live", lease_expires_at=(datetime.now(UTC) + timedelta(minutes=5)).isoformat()) + before = await reader.list_changed_runs() + await RunManager(store=runs).reconcile_orphaned_inflight_runs(error="owner died") + row = await runs.get("crash", user_id="alice") + assert row["status"] == status + assert row["evidence_seal_state"] == "partial" + assert row["evidence_seal_error"] + assert (await runs.get("live", user_id="alice"))["evidence_seal_state"] == "pending" + changes = await reader.list_changed_runs(cursor=before.next_cursor) + assert [item.run_id for item in changes.items] == ["crash"] + + +@pytest.mark.asyncio +async def test_startup_recovers_more_than_one_evidence_batch(evidence): + from deerflow.runtime.runs.manager import RunManager + + sf, events, runs, reader = evidence + async with sf() as session: + session.add_all([RunRow(run_id=f"crash-{i}", thread_id=f"t-{i}", user_id="alice", status="success", evidence_seal_state="pending") for i in range(201)]) + await session.commit() + await RunManager(store=runs).reconcile_orphaned_inflight_runs(error="owner died") + changes = await reader.list_changed_runs(limit=500) + assert len(changes.items) == 201 + async with sf() as session: + from sqlalchemy import func, select + + assert await session.scalar(select(func.count()).select_from(RunRow).where(RunRow.evidence_seal_state == "pending")) == 0 + + +@pytest.mark.asyncio +async def test_terminal_unsealed_recovery_keeps_live_local_finalization(evidence): + from deerflow.runtime.runs.manager import RunManager + from deerflow.runtime.runs.schemas import RunStatus + + sf, events, runs, reader = evidence + manager = RunManager(store=runs) + record = await manager.create("t-local", user_id="alice") + await manager.set_status(record.run_id, RunStatus.success) + await manager.set_finalizing(record.run_id, True) + await put_run(runs, "crashed-without-lease") + await manager.reconcile_orphaned_inflight_runs(error="owner died") + assert (await runs.get(record.run_id, user_id="alice"))["evidence_seal_state"] == "pending" + assert (await runs.get("crashed-without-lease", user_id="alice"))["evidence_seal_state"] == "partial" + + +@pytest.mark.asyncio +async def test_thread_delete_invalidates_even_empty_snapshot(evidence): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + sf, events, runs, reader = evidence + await put_run(runs) + snap = await reader.get_snapshot(thread_id="t-r", run_id="r") + await events.delete_by_thread("t-r", user_id="alice") + with pytest.raises(HostCapabilityError, match="NOT_FOUND"): + await reader.read_events(snapshot_ref=snap.snapshot_ref, cursor=None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["success", "error", "interrupted"]) +async def test_worker_seals_after_completion_hook_before_end(evidence, outcome): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from deerflow.runtime.runs.manager import RunManager + from deerflow.runtime.runs.worker import RunContext, run_agent + + sf, events, runs, reader = evidence + manager = RunManager(store=runs, event_store=events) + record = await manager.create("worker-thread", "stable-agent", user_id="alice") + captured = [] + + class Agent: + async def astream(self, graph_input, config=None, **kwargs): + captured.append(config["context"]["__run_journal"]) + if outcome == "error": + raise RuntimeError("agent failure") + if outcome == "interrupted": + record.abort_event.set() + yield {"messages": []} + + async def completion_hook(record): + captured[0].record_middleware("completion", name="completion", hook="stop", action="done", changes={}) + + async def end(run_id): + snap = await reader.get_snapshot(thread_id=record.thread_id, run_id=run_id) + assert snap.seal_state == "sealed" + page = await reader.read_events(snapshot_ref=snap.snapshot_ref, cursor=None) + assert "middleware:completion" in {e.event_type for e in page.items} + + bridge = SimpleNamespace(publish=AsyncMock(), publish_end=end, cleanup=AsyncMock()) + await run_agent(bridge, manager, record, ctx=RunContext(checkpointer=None, event_store=events, on_run_completed=completion_hook), agent_factory=lambda **kwargs: Agent(), graph_input={}, config={}) + assert record.status.value == outcome + assert captured[0]._closed + + +@pytest.mark.asyncio +async def test_seal_failure_does_not_change_business_success(evidence, monkeypatch): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from deerflow.runtime.runs.manager import RunManager + from deerflow.runtime.runs.worker import RunContext, run_agent + + sf, events, runs, reader = evidence + manager = RunManager(store=runs, event_store=events) + record = await manager.create("worker-thread", "stable-agent", user_id="alice") + failure = AsyncMock(side_effect=RuntimeError("seal database unavailable")) + monkeypatch.setattr(runs, "seal_evidence", failure) + + class Agent: + async def astream(self, *args, **kwargs): + yield {"messages": []} + + bridge = SimpleNamespace(publish=AsyncMock(), publish_end=AsyncMock(), cleanup=AsyncMock()) + await run_agent(bridge, manager, record, ctx=RunContext(checkpointer=None, event_store=events), agent_factory=lambda **kwargs: Agent(), graph_input={}, config={}) + failure.assert_awaited_once() + assert record.status.value == "success" + bridge.publish_end.assert_awaited_once() + row = await runs.get(record.run_id, user_id="alice") + assert row["evidence_seal_state"] == "partial" + assert row["evidence_seal_error"] + + +@pytest.mark.asyncio +async def test_seal_cancellation_is_deferred_until_after_end(evidence, monkeypatch): + import asyncio + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from deerflow.runtime.runs.manager import RunManager + from deerflow.runtime.runs.worker import RunContext, run_agent + + sf, events, runs, reader = evidence + manager = RunManager(store=runs, event_store=events) + record = await manager.create("cancel-seal", user_id="alice") + + class Agent: + async def astream(self, *args, **kwargs): + yield {"messages": []} + + monkeypatch.setattr(RunJournal, "finish_evidence", AsyncMock(side_effect=asyncio.CancelledError())) + bridge = SimpleNamespace(publish=AsyncMock(), publish_end=AsyncMock(), cleanup=AsyncMock()) + with pytest.raises(asyncio.CancelledError): + await run_agent(bridge, manager, record, ctx=RunContext(checkpointer=None, event_store=events), agent_factory=lambda **kwargs: Agent(), graph_input={}, config={}) + assert record.status.value == "success" + assert not record.finalizing + bridge.publish_end.assert_awaited_once() + row = await runs.get(record.run_id, user_id="alice") + assert row["evidence_seal_state"] == "partial" + + +@pytest.mark.asyncio +async def test_host_cancellation_drains_durable_evidence_seal(evidence, monkeypatch): + import asyncio + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from deerflow.runtime.runs.manager import RunManager + from deerflow.runtime.runs.worker import RunContext, run_agent + + _sf, events, runs, reader = evidence + manager = RunManager(store=runs, event_store=events) + record = await manager.create("cancel-durable-seal", user_id="alice") + entered, release = asyncio.Event(), asyncio.Event() + original_seal = runs.seal_evidence + journals = [] + + async def gated_seal(*args, **kwargs): + entered.set() + await release.wait() + return await original_seal(*args, **kwargs) + + seal = AsyncMock(side_effect=gated_seal) + monkeypatch.setattr(runs, "seal_evidence", seal) + + class Agent: + async def astream(self, graph_input, config=None, **kwargs): + journals.append(config["context"]["__run_journal"]) + yield {"messages": []} + + async def aget_state(self, _config): + return SimpleNamespace(values={"messages": []}) + + bridge = SimpleNamespace(publish=AsyncMock(), publish_end=AsyncMock(), cleanup=AsyncMock()) + worker = asyncio.create_task(run_agent(bridge, manager, record, ctx=RunContext(checkpointer=None, event_store=events), agent_factory=lambda **kwargs: Agent(), graph_input={}, config={})) + record.task = worker + try: + await asyncio.wait_for(entered.wait(), 5) + worker.cancel("first host cancellation") + await asyncio.sleep(0) + for _ in range(3): + worker.cancel("repeated host cancellation") + await asyncio.sleep(0) + assert not worker.done() + assert not journals[0]._closed + bridge.publish_end.assert_not_awaited() + release.set() + done, _pending = await asyncio.wait({worker}, timeout=5) + assert done, "worker did not finish after the seal write was released" + with pytest.raises(asyncio.CancelledError, match="first host cancellation"): + await worker + seal.assert_awaited_once() + snapshot = await reader.get_snapshot(thread_id=record.thread_id, run_id=record.run_id) + assert snapshot.seal_state == "sealed" + assert snapshot.status == "success" + assert journals[0]._closed + assert not record.finalizing + bridge.publish_end.assert_awaited_once() + finally: + release.set() + await asyncio.gather(worker, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_graph_assembly_failure_has_partial_coverage(evidence): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from deerflow.runtime.runs.manager import RunManager + from deerflow.runtime.runs.worker import RunContext, run_agent + + sf, events, runs, reader = evidence + manager = RunManager(store=runs, event_store=events) + record = await manager.create("preflight", user_id="alice") + + def fail(**kwargs): + raise RuntimeError("assembly failed") + + bridge = SimpleNamespace(publish=AsyncMock(), publish_end=AsyncMock(), cleanup=AsyncMock()) + await run_agent(bridge, manager, record, ctx=RunContext(checkpointer=None, event_store=events), agent_factory=fail, graph_input={}, config={}) + snap = await reader.get_snapshot(thread_id=record.thread_id, run_id=record.run_id) + assert snap.status == "error" + assert snap.seal_state == "partial" + assert "execution" in snap.seal_error + + +def test_migration_is_additive_current_chain(): + from alembic.script import ScriptDirectory + + from deerflow.persistence.bootstrap import _MIGRATIONS_DIR + + script = ScriptDirectory(str(_MIGRATIONS_DIR)) + rev = script.get_revision("0045_completed_run_evidence") + assert rev.down_revision == "0044_channel_runtime_configs" + + +@pytest.mark.asyncio +async def test_cancelled_background_flush_cannot_silently_seal_an_empty_tail(evidence): + import asyncio + + sf, events, runs, reader = evidence + await put_run(runs, owner_worker_id="worker") + original = events.put_batch + entered = asyncio.Event() + + async def blocked(batch): + entered.set() + await asyncio.Event().wait() + + events.put_batch = blocked + journal = RunJournal("r", "t-r", events, user_id="alice", flush_threshold=1) + journal.record_middleware("test", name="test", hook="test", action="test", changes={}) + await entered.wait() + pending = tuple(journal._owned_writes) + for task in pending: + task.cancel() + await asyncio.gather(*pending, return_exceptions=True) + events.put_batch = original + with pytest.raises(RuntimeError, match="unknown durable outcome"): + await journal.finish_evidence() + assert (await reader.get_snapshot(thread_id="t-r", run_id="r")).seal_state != "sealed" + assert not await events.list_events("t-r", "r", user_id="alice") + await journal.close(flush=False) + + +@pytest.mark.asyncio +async def test_cancelled_flush_before_first_execution_cannot_seal_an_empty_tail(evidence): + import asyncio + + sf, events, runs, reader = evidence + await put_run(runs) + journal = RunJournal("r", "t-r", events, user_id="alice", flush_threshold=1) + journal.record_middleware("test", name="test", hook="test", action="test", changes={}) + pending = tuple(journal._owned_writes) + for task in pending: + task.cancel() + await asyncio.gather(*pending, return_exceptions=True) + with pytest.raises(RuntimeError, match="unknown durable outcome"): + await journal.finish_evidence() + assert (await reader.get_snapshot(thread_id="t-r", run_id="r")).seal_state != "sealed" + assert not await events.list_events("t-r", "r", user_id="alice") + await journal.close(flush=False) + + +@pytest.mark.asyncio +async def test_already_admitted_cross_thread_callback_survives_stop(evidence): + import threading + + sf, events, runs, reader = evidence + await put_run(runs) + journal = RunJournal("r", "t-r", events, user_id="alice") + thread = threading.Thread(target=lambda: journal.record_middleware("accepted", name="accepted", hook="test", action="test", changes={})) + thread.start() + thread.join() + receipt = await journal.finish_evidence() + assert receipt.event_count == 1 + assert not journal._closed + journal.record_middleware("rejected", name="rejected", hook="test", action="test", changes={}) + await journal.close() + assert len(await events.list_events("t-r", "r", user_id="alice")) == 1 + + +@pytest.mark.asyncio +async def test_lifecycle_transition_invalidates_pending_snapshot(evidence): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + sf, events, runs, reader = evidence + await put_run(runs, status="running") + pending = await reader.get_snapshot(thread_id="t-r", run_id="r") + await runs.update_status("r", "error") + terminal = await reader.get_snapshot(thread_id="t-r", run_id="r") + assert terminal.status == "error" + assert terminal.seal_state == "partial" + assert terminal.evidence_revision != pending.evidence_revision + with pytest.raises(HostCapabilityError, match="EVIDENCE_EXPIRED"): + await reader.read_events(snapshot_ref=pending.snapshot_ref) + + +def test_migration_upgrades_real_sqlite_and_preserves_legacy_partial(tmp_path): + from alembic import command + from sqlalchemy import create_engine, inspect, text + + from deerflow.persistence.bootstrap import _get_alembic_config + + engine = create_engine(f"sqlite:///{tmp_path / 'migration.db'}") + with engine.begin() as conn: + Base.metadata.create_all(conn) + conn.execute(RunRow.__table__.insert().values(run_id="old", thread_id="thread", user_id="alice", status="success")) + conn.execute(text("INSERT INTO channel_runtime_configs VALUES ('telegram', 'synthetic-ciphertext', 0, 7, 1.0)")) + conn.execute(text("DROP TABLE completed_run_snapshots")) + conn.execute(RunEventRow.__table__.insert().values(thread_id="thread", run_id="old", user_id="alice", event_type="test", category="message", content="legacy", seq=1)) + conn.execute(text("ALTER TABLE run_events DROP COLUMN content_sha256")) + for name in ("evidence_origin", "evidence_agent_id", "evidence_seal_state", "evidence_seal_error", "evidence_revision", "evidence_upper_seq", "evidence_event_count", "evidence_retention_revision"): + conn.execute(text(f"ALTER TABLE runs DROP COLUMN {name}")) + config = _get_alembic_config(engine) + config.set_main_option("sqlalchemy.url", f"sqlite+aiosqlite:///{tmp_path / 'migration.db'}") + command.stamp(config, "0044_channel_runtime_configs") + command.upgrade(config, "0045_completed_run_evidence") + with engine.connect() as conn: + row = conn.execute(text("SELECT evidence_seal_state, evidence_revision, evidence_retention_revision FROM runs WHERE run_id='old'")).one() + assert tuple(row) == (None, None, 0) + assert conn.scalar(text("SELECT content_sha256 FROM run_events WHERE run_id='old'")) is None + assert "completed_run_snapshots" in inspect(conn).get_table_names() + assert conn.execute(text("SELECT config_ciphertext, version FROM channel_runtime_configs WHERE provider='telegram'")).one() == ("synthetic-ciphertext", 7) + engine.dispose() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("limit", [0, -1, 2001, True, 1.5]) +async def test_changed_run_page_rejects_invalid_limits(evidence, limit): + _sf, _events, runs, reader = evidence + await put_run(runs) + with pytest.raises(ValueError, match="max_events"): + await reader.list_changed_runs(limit=limit) + + +@pytest.mark.asyncio +async def test_changed_run_page_uses_validated_limit(evidence, monkeypatch): + from deerflow_extension_api import EvidenceLimits + + _sf, _events, runs, reader = evidence + for run_id in ("a", "b", "c"): + await put_run(runs, run_id) + + def bounded_limits(*, max_events): + return EvidenceLimits(max_events=min(max_events, 1)) + + # Exercise a tighter host policy without changing the public hard limit. + monkeypatch.setattr("deerflow.extensions.completed_run_evidence.EvidenceLimits", bounded_limits) + first = await reader.list_changed_runs(limit=100) + assert [row.run_id for row in first.items] == ["a"] + assert first.has_more + second = await reader.list_changed_runs(cursor=first.next_cursor, limit=100) + assert [row.run_id for row in second.items] == ["b"] + + +@pytest.mark.asyncio +async def test_normal_close_stops_evidence_producers_before_draining(evidence): + import asyncio + + sf, events, runs, reader = evidence + await put_run(runs) + entered, release = asyncio.Event(), asyncio.Event() + original = events.put_batch + + async def blocked(batch): + entered.set() + await release.wait() + return await original(batch) + + events.put_batch = blocked + journal = RunJournal("r", "t-r", events, user_id="alice") + journal.record_middleware("accepted", name="accepted", hook="test", action="test", changes={}) + closing = asyncio.create_task(journal.close()) + try: + await entered.wait() + journal.record_middleware("late", name="late", hook="test", action="test", changes={}) + finally: + release.set() + await closing + rows = await events.list_events("t-r", "r", user_id="alice") + assert len(rows) == 1 + assert rows[0]["event_type"] == "middleware:accepted" + + +@pytest.mark.asyncio +async def test_commit_then_error_never_replays_or_issues_evidence_receipt(evidence): + sf, events, runs, reader = evidence + await put_run(runs) + original = events.put_batch + calls = 0 + + async def commit_then_error(batch): + nonlocal calls + calls += 1 + await original(batch) + raise OSError("acknowledgement lost") + + events.put_batch = commit_then_error + journal = RunJournal("r", "t-r", events, user_id="alice") + journal.record_middleware("accepted", name="accepted", hook="test", action="test", changes={}) + try: + with pytest.raises(OSError, match="acknowledgement lost"): + await journal.finish_evidence() + with pytest.raises(RuntimeError, match="unknown durable outcome"): + await journal.finish_evidence() + assert calls == 1 + assert len(await events.list_events("t-r", "r", user_id="alice")) == 1 + assert (await reader.get_snapshot(thread_id="t-r", run_id="r")).seal_state != "sealed" + finally: + await journal.close(flush=False) diff --git a/backend/tests/test_db_event_store_lock_lifecycle.py b/backend/tests/test_db_event_store_lock_lifecycle.py index db12462c60..cf6846ef3b 100644 --- a/backend/tests/test_db_event_store_lock_lifecycle.py +++ b/backend/tests/test_db_event_store_lock_lifecycle.py @@ -1,5 +1,6 @@ import asyncio import weakref +from types import SimpleNamespace import pytest @@ -7,17 +8,17 @@ from deerflow.runtime.events.store.db import DbRunEventStore class _PausedDeleteSession: - """Fake session that pauses the deletion on its first aggregate read. + """Pause deletion on its first scalar (the retention change clock). - The fixed deletion path opens the session, takes the thread mutation fence - (a no-op on this fake's dialect) and only then reads the expected count, so - pausing in ``scalar`` proves the caller already owns the thread's mutation - critical section. + The in-process thread lock must already be held before any database work. + The database advisory fence is a no-op for this fake's SQLite dialect. """ def __init__(self, scalar_started: asyncio.Event, allow_scalar: asyncio.Event) -> None: self._scalar_started = scalar_started self._allow_scalar = allow_scalar + # Deletion now allocates the evidence-retention change sequence too. + self.bind = SimpleNamespace(dialect=SimpleNamespace(name="sqlite")) async def __aenter__(self): return self @@ -94,7 +95,7 @@ async def test_delete_waiter_handoff_keeps_one_write_lock_generation(): # Delete now owns the same generation, and the queued writer is still # waiting behind it. - await scalar_started.wait() + await asyncio.wait_for(scalar_started.wait(), timeout=5) assert not waiter_entered.is_set() allow_scalar.set() diff --git a/backend/tests/test_extension_host_access.py b/backend/tests/test_extension_host_access.py new file mode 100644 index 0000000000..4127b7aeec --- /dev/null +++ b/backend/tests/test_extension_host_access.py @@ -0,0 +1,166 @@ +from types import SimpleNamespace + +import pytest +from pydantic import ValidationError + +from deerflow.extensions.loader import ExtensionSpec, load_extensions + + +def granted(**overrides): + return ExtensionSpec( + use="example:install", + name="evolution", + host_access={ + "evidence": {"owners": ["owner"]}, + "skill_mutations": {"owners": ["owner"], "operations": ["stage", "check"], "trigger_agents": ["agent"], "target_skills": ["example"], **overrides}, + }, + ) + + +def test_host_access_is_default_deny_and_not_private_config(): + plain = ExtensionSpec(use="example:install") + assert plain.host_access.evidence.owners == () + assert plain.host_access.skill_mutations.operations == () + spec = granted() + assert spec.config == {} + assert spec.host_access.skill_mutations.operations == ("stage", "check") + + +@pytest.mark.parametrize("key,value", [("owners", ["*"]), ("operations", ["create"]), ("trigger_agents", [""]), ("target_skills", ["../bad"]), ("topology", "multi_node")]) +def test_invalid_grants_rejected(key, value): + with pytest.raises(ValidationError): + granted(**{key: value}) + + +def test_host_access_requires_stable_name(): + with pytest.raises(ValidationError): + ExtensionSpec(use="example:install", host_access={"evidence": {"owners": ["owner"]}}) + + +def test_granted_source_cannot_be_ambiguous_even_with_other_ungranted_entry(monkeypatch): + calls = [] + monkeypatch.setattr("deerflow.extensions.loader.resolve_variable", lambda _: calls.append(1)) + with pytest.raises(ValueError, match="unique"): + load_extensions([granted(), ExtensionSpec(use="example:install", name="other")]) + assert calls == [] + + +def test_granted_name_cannot_be_ambiguous(monkeypatch): + monkeypatch.setattr("deerflow.extensions.loader.resolve_variable", lambda _: None) + with pytest.raises(ValueError, match="unique"): + load_extensions([granted(), ExtensionSpec(use="other:install", name="evolution")]) + + +def test_binding_changes_with_entry_point_and_empty_scope_means_none(): + from deerflow.extensions.host_access import bind_host_access + + first = bind_host_access([granted()])["example:install"] + replacement = granted().model_copy(update={"use": "replacement:install"}) + second = bind_host_access([replacement])["replacement:install"] + assert first.plugin_id != second.plugin_id + assert not first.allows("commit", "owner", "example") + assert first.allows("stage", "owner", "example") + assert not first.allows("stage", "another", "example") + empty = granted(owners=[]) + assert not bind_host_access([empty])["example:install"].allows("stage", "owner", "example") + + +@pytest.mark.asyncio +async def test_services_receive_independently_bound_capabilities(): + from deerflow.extensions.gateway import start_services + from deerflow.extensions.registry import ExtensionRegistry + + received = [] + + class Service: + async def start(self, deps): + received.append(deps) + + registry = ExtensionRegistry() + for source in ("first", "second"): + with registry.attributed_to(source): + registry.service(Service()) + first, second = object(), object() + await start_services(registry.build(), SimpleNamespace(), None, host_capabilities={"first": (first, None), "second": (None, second)}) + assert received[0].completed_run_evidence is first + assert received[0].skill_mutations is None + assert received[1].completed_run_evidence is None + assert received[1].skill_mutations is second + + +def test_new_host_contracts_are_available_without_harness_imports(): + import deerflow_extension_api as api + + for name in ( + "HostCapabilityError", + "CompletedRunSnapshot", + "CompletedRunEvent", + "CompletedRunEventPage", + "CompletedRunEvidenceReader", + "EvidenceLimits", + "AssetRevision", + "SkillRevisionView", + "MutationCapabilities", + "Proposal", + "ProposalBundle", + "BundleFile", + "HostCheckResult", + "AssessmentRef", + "Operation", + "SkillMutationService", + ): + assert name in api.__all__ + assert getattr(api, name).__module__.startswith("deerflow_extension_api.") + + +def test_mutation_bundles_and_capabilities_detach_mutable_constructor_values(): + from deerflow_extension_api import AssetRevision, BundleFile, MutationCapabilities, Proposal, ProposalBundle + + content = bytearray(b"original") + files = [BundleFile("SKILL.md", content, "digest", False)] + revision = AssetRevision("incarnation", 1, "digest") + bundle = ProposalBundle("p", files, files, revision, "hash") + ops = ["stage"] + caps = MutationCapabilities(ops) + sources = ["s"] + proposal = Proposal("p", "o", "t", "skill", revision, "hash", "STAGED", "expiry", sources) + content[:] = b"tampered" + files.clear() + ops.append("commit") + sources.clear() + assert bundle.baseline[0].content == b"original" + assert caps.operations == ("stage",) + assert proposal.source_refs == ("s",) + + +@pytest.mark.asyncio +async def test_model_and_evolution_grants_compose_in_same_service(monkeypatch): + from deerflow.extensions.gateway import start_services, stop_services + + received = [] + + class Service: + async def start(self, deps): + received.append(deps) + + async def stop(self): + pass + + def install(registry, config): + registry.service(Service()) + + monkeypatch.setattr("deerflow.extensions.loader.resolve_variable", lambda _: install) + access = granted().host_access.model_dump() + access["model_invocation"] = {"roles": {"default": "host-model"}} + spec = ExtensionSpec(use="example:install", name="evolution", host_access=access) + loaded, diagnostics = load_extensions([spec]) + assert not diagnostics + evidence, mutations = object(), object() + try: + diagnostics = await start_services(loaded, SimpleNamespace(), None, host_capabilities={spec.use: (evidence, mutations)}) + assert not diagnostics + assert received[0].model_invoker is not None + assert received[0].completed_run_evidence is evidence + assert received[0].skill_mutations is mutations + finally: + await stop_services(loaded) diff --git a/backend/tests/test_gateway_extension_service_lifecycle.py b/backend/tests/test_gateway_extension_service_lifecycle.py index 83f3c79b12..1149d00a94 100644 --- a/backend/tests/test_gateway_extension_service_lifecycle.py +++ b/backend/tests/test_gateway_extension_service_lifecycle.py @@ -52,6 +52,46 @@ def _patch_runtime_resources(monkeypatch, events: list[str]) -> None: monkeypatch.setattr("deerflow.persistence.engine.get_session_factory", lambda: None) +@pytest.mark.asyncio +async def test_host_capabilities_outlive_extension_stop_and_close_before_database(monkeypatch): + from app.gateway.deps import langgraph_runtime + + events = [] + _patch_runtime_resources(monkeypatch, events) + evidence, mutations = object(), object() + + class Host: + def __init__(self, *args): + self.bindings = {"example:install": (evidence, mutations)} + + async def start(self): + events.append("host_start") + + async def close(self): + events.append("host_close") + + class Service: + async def start(self, deps): + assert deps.completed_run_evidence is evidence + assert deps.skill_mutations is mutations + events.append("service_start") + + async def stop(self): + events.append("service_stop") + + monkeypatch.setattr("deerflow.extensions.host_capabilities.HostCapabilities", Host) + monkeypatch.setattr("deerflow.persistence.thread_meta.make_thread_store", lambda *_: (_ for _ in ()).throw(RuntimeError("later failure"))) + registry = ExtensionRegistry() + with registry.attributed_to("example:install"): + registry.service(Service()) + app = FastAPI() + app.state.extensions = registry.build() + with pytest.raises(RuntimeError, match="later failure"): + async with langgraph_runtime(app, SimpleNamespace(database=_database_config())): + pytest.fail("must not yield") + assert events == ["host_start", "service_start", "service_stop", "host_close", "engine_close"] + + @pytest.mark.asyncio async def test_runtime_owns_engine_cleanup_before_initialization(monkeypatch): from app.gateway.deps import langgraph_runtime diff --git a/backend/tests/test_gateway_run_recovery.py b/backend/tests/test_gateway_run_recovery.py index e793ff21b9..e80824887e 100644 --- a/backend/tests/test_gateway_run_recovery.py +++ b/backend/tests/test_gateway_run_recovery.py @@ -276,6 +276,9 @@ async def test_sql_runtime_shares_run_repository_with_scheduler(monkeypatch): async def noop(*_args, **_kwargs): return None + # This test asserts repository identity, not SQL startup/recovery. Its + # session factory is intentionally an opaque sentinel, not a usable DB. + monkeypatch.setattr("deerflow.extensions.host_capabilities.HostCapabilities.start", noop) monkeypatch.setattr(engine_module, "init_engine_from_config", noop) monkeypatch.setattr(engine_module, "get_session_factory", lambda: session_factory) monkeypatch.setattr(engine_module, "close_engine", noop) diff --git a/backend/tests/test_gateway_services.py b/backend/tests/test_gateway_services.py index fd471d5ae9..b723f6344e 100644 --- a/backend/tests/test_gateway_services.py +++ b/backend/tests/test_gateway_services.py @@ -3643,7 +3643,7 @@ def test_launch_scheduled_thread_run_marks_context_non_interactive(_stub_app_con async def _scenario(): captured: dict[str, object] = {} - async def fake_start_run(body, thread_id, request, *, idempotency_key=None, scheduled_task_runtime=None): + async def fake_start_run(body, thread_id, request, *, idempotency_key=None, evidence_origin=None, scheduled_task_runtime=None): captured["body"] = body captured["thread_id"] = thread_id captured["context"] = body.context @@ -3705,7 +3705,7 @@ def test_launch_scheduled_thread_run_uses_configured_recursion_limit(_stub_app_c async def _scenario(): captured: dict[str, object] = {} - async def fake_start_run(body, thread_id, request, *, idempotency_key=None, scheduled_task_runtime=None): + async def fake_start_run(body, thread_id, request, *, idempotency_key=None, evidence_origin=None, scheduled_task_runtime=None): assert idempotency_key is None captured["config"] = body.config return SimpleNamespace(run_id="run-1", thread_id=thread_id) @@ -3747,7 +3747,7 @@ def test_launch_scheduled_thread_run_recursion_limit_is_clamped_to_ceiling(_stub async def _scenario(): captured: dict[str, object] = {} - async def fake_start_run(body, thread_id, request, *, idempotency_key=None, scheduled_task_runtime=None): + async def fake_start_run(body, thread_id, request, *, idempotency_key=None, evidence_origin=None, scheduled_task_runtime=None): assert idempotency_key is None captured["config"] = body.config return SimpleNamespace(run_id="run-1", thread_id=thread_id) @@ -3780,7 +3780,7 @@ def test_launch_scheduled_thread_run_falls_back_when_config_unloadable(_stub_app async def _scenario(): captured: dict[str, object] = {} - async def fake_start_run(body, thread_id, request, *, idempotency_key=None, scheduled_task_runtime=None): + async def fake_start_run(body, thread_id, request, *, idempotency_key=None, evidence_origin=None, scheduled_task_runtime=None): assert idempotency_key is None captured["config"] = body.config return SimpleNamespace(run_id="run-1", thread_id=thread_id) @@ -5416,7 +5416,7 @@ async def _capture_scheduled_launch(**kwargs): captured: dict[str, object] = {} - async def fake_start_run(body, thread_id, request, *, idempotency_key=None, scheduled_task_runtime=None): + async def fake_start_run(body, thread_id, request, *, idempotency_key=None, evidence_origin=None, scheduled_task_runtime=None): captured["body"] = body captured["idempotency_key"] = idempotency_key return SimpleNamespace(run_id="run-1", thread_id=thread_id) diff --git a/backend/tests/test_mcp_lifecycle_writers.py b/backend/tests/test_mcp_lifecycle_writers.py index b6ae4e5007..967472cb4c 100644 --- a/backend/tests/test_mcp_lifecycle_writers.py +++ b/backend/tests/test_mcp_lifecycle_writers.py @@ -189,7 +189,7 @@ def test_skills_writer_preserves_the_lifecycle_ledger( monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(cfg)) monkeypatch.setattr("app.gateway.routers.skills.reload_extensions_config", lambda *a, **k: None) - skills_router._write_extensions_skill_state(MagicMock(), "demo", False, rebuild_public_projection=False) + skills_router._write_extensions_skill_state(MagicMock(_app_config=None), "demo", False, rebuild_public_projection=False) written = json.loads(cfg.read_text(encoding="utf-8")) assert written["mcpLifecycle"] == ledger # preserved byte-for-value, never recomputed diff --git a/backend/tests/test_migration_0026_mcp_task_lease_tokens.py b/backend/tests/test_migration_0026_mcp_task_lease_tokens.py index 0499b69668..3c5541c21b 100644 --- a/backend/tests/test_migration_0026_mcp_task_lease_tokens.py +++ b/backend/tests/test_migration_0026_mcp_task_lease_tokens.py @@ -2,7 +2,7 @@ Adds the nullable per-claim token columns ``McpTaskRepository`` uses to fence poll, cancel, and notification mutations to the exact claim generation. The -chain-head pin moved on to ``test_migration_0027_0028_notification_outbox``. +chain-head pin lives in ``test_migration_0046_skill_mutations``. """ from __future__ import annotations diff --git a/backend/tests/test_migration_0027_0028_notification_outbox.py b/backend/tests/test_migration_0027_0028_notification_outbox.py index ac434b2e5f..ca52699338 100644 --- a/backend/tests/test_migration_0027_0028_notification_outbox.py +++ b/backend/tests/test_migration_0027_0028_notification_outbox.py @@ -23,7 +23,7 @@ OUTBOX = "0027_notification_deliveries" PARKED = "0028_parked_attempts" SCHEDULER = "0029_scheduler_agent_tasks" CLAIM_TOKENS = "0030_notification_claim_tokens" -HEAD = "0044_channel_runtime_configs" +HEAD = bootstrap._get_head_revision() PREVIOUS = "0026_mcp_task_lease_tokens" TABLE = "notification_deliveries" diff --git a/backend/tests/test_migration_0029_scheduler_agent_tasks.py b/backend/tests/test_migration_0029_scheduler_agent_tasks.py index 4edafee0b7..0870b1ca09 100644 --- a/backend/tests/test_migration_0029_scheduler_agent_tasks.py +++ b/backend/tests/test_migration_0029_scheduler_agent_tasks.py @@ -23,7 +23,6 @@ from deerflow.persistence.postgres_schema import build_asyncpg_connect_args from deerflow.persistence.scheduled_tasks import ScheduledTaskRepository REVISION = "0029_scheduler_agent_tasks" -CURRENT_HEAD = "0044_channel_runtime_configs" NEXT = "0030_notification_claim_tokens" PREVIOUS = "0028_parked_attempts" TASK_FIELDS = {"origin_thread_id", "goal_objective", "max_runs", "end_at", "standing_notes"} @@ -33,7 +32,8 @@ pytestmark = pytest.mark.asyncio async def test_0029_remains_in_the_single_migration_chain(): script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR)) - assert script.get_heads() == [CURRENT_HEAD] + assert len(script.get_heads()) == 1 + assert REVISION in {revision.revision for revision in script.walk_revisions()} assert script.get_revision(REVISION).down_revision == PREVIOUS assert script.get_revision(NEXT).down_revision == REVISION assert len(REVISION) <= 32 @@ -75,7 +75,7 @@ async def test_0029_preserves_legacy_fields_and_downgrades_unmet(tmp_path, migra assert all(columns[table][field]["nullable"] and columns[table][field]["default"] is None for field in fields) # Compare with the ORM at the current head: later revisions add # scheduled_tasks columns (0031) that the model already declares. - await asyncio.to_thread(command.upgrade, cfg, CURRENT_HEAD) + await asyncio.to_thread(command.upgrade, cfg, "head") async with engine.connect() as conn: def scheduler_schema_diff(sync): @@ -130,7 +130,7 @@ async def test_0029_preserves_legacy_fields_and_downgrades_unmet(tmp_path, migra assert (await conn.execute(sa.text("SELECT run_count FROM scheduled_tasks WHERE id='legacy'"))).scalar_one() == 7 # The ORM model carries later scheduled_tasks columns (0031), so read # it back at the current head. - await asyncio.to_thread(command.upgrade, cfg, CURRENT_HEAD) + await asyncio.to_thread(command.upgrade, cfg, "head") assert (await repo.get("new", user_id="owner"))["origin_thread_id"] is None finally: if schema: diff --git a/backend/tests/test_migration_0032_activity_and_task_events.py b/backend/tests/test_migration_0032_activity_and_task_events.py index e519ead02f..4536b7df8b 100644 --- a/backend/tests/test_migration_0032_activity_and_task_events.py +++ b/backend/tests/test_migration_0032_activity_and_task_events.py @@ -24,7 +24,6 @@ IDEMPOTENCY = "0036_run_idempotency_request" DOCUMENT_SUMMARIES = "0037_project_document_summaries" CHANNEL_BINDINGS = "0038_channel_thread_bindings" USER_DISABLED = "0039_user_disabled" -CURRENT_HEAD = "0044_channel_runtime_configs" PREVIOUS = "0031_scheduled_streak_boundary" TABLES = {"thread_read_markers", "thread_read_versions", "scheduled_task_events"} INDEXES = { @@ -37,7 +36,8 @@ pytestmark = pytest.mark.asyncio async def test_0032_remains_in_the_single_migration_chain(): script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR)) - assert script.get_heads() == [CURRENT_HEAD] + assert len(script.get_heads()) == 1 + assert REVISION in {revision.revision for revision in script.walk_revisions()} assert script.get_revision(REVISION).down_revision == PREVIOUS assert script.get_revision(NEXT).down_revision == REVISION assert script.get_revision(AFTER_NEXT).down_revision == NEXT diff --git a/backend/tests/test_migration_0034_run_event_seq_watermark.py b/backend/tests/test_migration_0034_run_event_seq_watermark.py index 2276ba81f9..be82b88c7b 100644 --- a/backend/tests/test_migration_0034_run_event_seq_watermark.py +++ b/backend/tests/test_migration_0034_run_event_seq_watermark.py @@ -24,14 +24,14 @@ IDEMPOTENCY = "0036_run_idempotency_request" DOCUMENT_SUMMARIES = "0037_project_document_summaries" CHANNEL_BINDINGS = "0038_channel_thread_bindings" USER_DISABLED = "0039_user_disabled" -CURRENT_HEAD = "0044_channel_runtime_configs" TABLE = "run_event_thread_seq" pytestmark = pytest.mark.asyncio async def test_0034_remains_in_the_single_migration_chain(): script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR)) - assert script.get_heads() == [CURRENT_HEAD] + assert len(script.get_heads()) == 1 + assert REVISION in {revision.revision for revision in script.walk_revisions()} assert script.get_revision(REVISION).down_revision == PREVIOUS assert script.get_revision(LOGIN_THROTTLE).down_revision == REVISION assert script.get_revision(IDEMPOTENCY).down_revision == LOGIN_THROTTLE diff --git a/backend/tests/test_migration_0035_login_throttle.py b/backend/tests/test_migration_0035_login_throttle.py index 959a203305..b47a0fcf66 100644 --- a/backend/tests/test_migration_0035_login_throttle.py +++ b/backend/tests/test_migration_0035_login_throttle.py @@ -25,7 +25,6 @@ IDEMPOTENCY = "0036_run_idempotency_request" DOCUMENT_SUMMARIES = "0037_project_document_summaries" CHANNEL_BINDINGS = "0038_channel_thread_bindings" USER_DISABLED = "0039_user_disabled" -CURRENT_HEAD = "0044_channel_runtime_configs" PREVIOUS = "0034_run_event_seq_watermark" TABLE = "login_throttle" COLUMNS = {"ip", "fail_count", "locked_at", "lock_duration_seconds", "updated_at"} @@ -33,9 +32,10 @@ INDEX = "ix_login_throttle_updated_at" pytestmark = pytest.mark.asyncio -async def test_0035_remains_in_the_single_head_chain_after_0034(): +async def test_0035_remains_in_the_single_migration_chain(): script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR)) - assert script.get_heads() == [CURRENT_HEAD] + assert len(script.get_heads()) == 1 + assert REVISION in {revision.revision for revision in script.walk_revisions()} assert script.get_revision(REVISION).down_revision == PREVIOUS assert script.get_revision(IDEMPOTENCY).down_revision == REVISION assert script.get_revision(DOCUMENT_SUMMARIES).down_revision == IDEMPOTENCY @@ -186,6 +186,6 @@ async def test_bootstrap_provisions_the_table_on_an_empty_database_and_is_idempo assert set(shape["columns"]) == COLUMNS assert INDEX in shape["indexes"] async with engine.connect() as conn: - assert (await conn.execute(sa.text("SELECT version_num FROM alembic_version"))).scalar_one() == CURRENT_HEAD + assert (await conn.execute(sa.text("SELECT version_num FROM alembic_version"))).scalar_one() == bootstrap._get_head_revision() finally: await engine.dispose() diff --git a/backend/tests/test_migration_0037_project_document_summaries.py b/backend/tests/test_migration_0037_project_document_summaries.py index 0eb68fb1b8..890425c95a 100644 --- a/backend/tests/test_migration_0037_project_document_summaries.py +++ b/backend/tests/test_migration_0037_project_document_summaries.py @@ -24,20 +24,21 @@ GROUP_NOTICE = "0040_scheduled_group_notice" IM_NOTICE_PREFERENCES = "0041_im_notice_preferences" CHANNEL_LEASES = "0042_channel_leases" RUNTIME_IM_NOTICES = "0043_runtime_im_notices" -CURRENT_HEAD = "0044_channel_runtime_configs" +CHANNEL_RUNTIME_CONFIGS = "0044_channel_runtime_configs" pytestmark = pytest.mark.asyncio -async def test_0037_chains_after_0036_and_is_single_head(): +async def test_0037_chains_after_0036_in_single_migration_chain(): script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR)) - assert script.get_heads() == [CURRENT_HEAD] + assert len(script.get_heads()) == 1 + assert REVISION in {revision.revision for revision in script.walk_revisions()} assert script.get_revision(CHANNEL_BINDINGS).down_revision == REVISION assert script.get_revision(USER_DISABLED).down_revision == CHANNEL_BINDINGS assert script.get_revision(GROUP_NOTICE).down_revision == USER_DISABLED assert script.get_revision(IM_NOTICE_PREFERENCES).down_revision == GROUP_NOTICE assert script.get_revision(CHANNEL_LEASES).down_revision == IM_NOTICE_PREFERENCES assert script.get_revision(RUNTIME_IM_NOTICES).down_revision == CHANNEL_LEASES - assert script.get_revision(CURRENT_HEAD).down_revision == RUNTIME_IM_NOTICES + assert script.get_revision(CHANNEL_RUNTIME_CONFIGS).down_revision == RUNTIME_IM_NOTICES assert script.get_revision(REVISION).down_revision == PREVIOUS # alembic_version.version_num is VARCHAR(32). assert len(REVISION) <= 32 diff --git a/backend/tests/test_migration_0038_channel_thread_bindings.py b/backend/tests/test_migration_0038_channel_thread_bindings.py index d137cc0895..8c99c5e2c1 100644 --- a/backend/tests/test_migration_0038_channel_thread_bindings.py +++ b/backend/tests/test_migration_0038_channel_thread_bindings.py @@ -22,7 +22,6 @@ from deerflow.persistence.postgres_schema import build_asyncpg_connect_args REVISION = "0038_channel_thread_bindings" PREVIOUS = "0037_project_document_summaries" -CURRENT_HEAD = "0044_channel_runtime_configs" TABLE = "channel_thread_bindings" COLUMNS = {"key", "channel_name", "chat_id", "topic_id", "thread_id", "user_id", "created_at", "updated_at"} INDEX = "ix_channel_thread_bindings_channel_chat" @@ -31,7 +30,8 @@ pytestmark = pytest.mark.asyncio async def test_0038_remains_in_the_single_migration_chain(): script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR)) - assert script.get_heads() == [CURRENT_HEAD] + assert len(script.get_heads()) == 1 + assert REVISION in {revision.revision for revision in script.walk_revisions()} assert script.get_revision(REVISION).down_revision == PREVIOUS # alembic_version.version_num is VARCHAR(32). assert len(REVISION) <= 32 @@ -185,6 +185,6 @@ async def test_bootstrap_provisions_the_table_on_an_empty_database_and_is_idempo async with engine.connect() as conn: # Bootstrap runs the chain to its head, which this test no longer # owns once later revisions chain after 0038. - assert (await conn.execute(sa.text("SELECT version_num FROM alembic_version"))).scalar_one() == CURRENT_HEAD + assert (await conn.execute(sa.text("SELECT version_num FROM alembic_version"))).scalar_one() == bootstrap._get_head_revision() finally: await engine.dispose() diff --git a/backend/tests/test_migration_0040_scheduled_group_notice.py b/backend/tests/test_migration_0040_scheduled_group_notice.py index 4e595dbb1d..cde5bccec5 100644 --- a/backend/tests/test_migration_0040_scheduled_group_notice.py +++ b/backend/tests/test_migration_0040_scheduled_group_notice.py @@ -16,13 +16,13 @@ from deerflow.persistence.postgres_schema import build_asyncpg_connect_args REVISION = "0040_scheduled_group_notice" PREVIOUS = "0039_user_disabled" -CURRENT_HEAD = "0044_channel_runtime_configs" COLUMN = "notification_group_connection_id" -def test_group_notification_revision_has_one_head_after_0039(): +def test_group_notification_revision_is_in_single_chain_after_0039(): script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR)) - assert script.get_heads() == [CURRENT_HEAD] + assert len(script.get_heads()) == 1 + assert REVISION in {revision.revision for revision in script.walk_revisions()} assert script.get_revision("0041_im_notice_preferences").down_revision == REVISION assert script.get_revision(REVISION).down_revision == PREVIOUS assert len(REVISION) <= 32 diff --git a/backend/tests/test_migration_0042_channel_leases.py b/backend/tests/test_migration_0042_channel_leases.py index 1d6b518671..e114c6ac54 100644 --- a/backend/tests/test_migration_0042_channel_leases.py +++ b/backend/tests/test_migration_0042_channel_leases.py @@ -23,7 +23,6 @@ from deerflow.persistence.postgres_schema import build_asyncpg_connect_args REVISION = "0042_channel_leases" PREVIOUS = "0041_im_notice_preferences" NEXT = "0043_runtime_im_notices" -CURRENT_HEAD = "0044_channel_runtime_configs" TABLE = "channel_leases" COLUMNS = {"provider", "owner_id", "epoch", "lease_expires_at", "transport_running", "updated_at"} pytestmark = pytest.mark.asyncio @@ -31,8 +30,9 @@ pytestmark = pytest.mark.asyncio async def test_0042_remains_in_the_single_migration_chain(): script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR)) - assert script.get_heads() == [CURRENT_HEAD] - assert script.get_revision(CURRENT_HEAD).down_revision == NEXT + assert len(script.get_heads()) == 1 + assert REVISION in {revision.revision for revision in script.walk_revisions()} + assert script.get_revision("0044_channel_runtime_configs").down_revision == NEXT assert script.get_revision(NEXT).down_revision == REVISION assert script.get_revision(REVISION).down_revision == PREVIOUS # alembic_version.version_num is VARCHAR(32). @@ -158,6 +158,6 @@ async def test_bootstrap_provisions_the_table_on_an_empty_database(tmp_path): await bootstrap.bootstrap_schema(engine, backend="sqlite") assert set((await _shape(engine))["columns"]) == COLUMNS async with engine.connect() as conn: - assert (await conn.execute(sa.text("SELECT version_num FROM alembic_version"))).scalar_one() == CURRENT_HEAD + assert (await conn.execute(sa.text("SELECT version_num FROM alembic_version"))).scalar_one() == bootstrap._get_head_revision() finally: await engine.dispose() diff --git a/backend/tests/test_migration_0044_channel_runtime_configs.py b/backend/tests/test_migration_0044_channel_runtime_configs.py index 68228d311b..d081ffc96c 100644 --- a/backend/tests/test_migration_0044_channel_runtime_configs.py +++ b/backend/tests/test_migration_0044_channel_runtime_configs.py @@ -22,7 +22,7 @@ from deerflow.persistence.postgres_schema import build_asyncpg_connect_args REVISION = "0044_channel_runtime_configs" PREVIOUS = "0043_runtime_im_notices" -CURRENT_HEAD = REVISION +CURRENT_HEAD = bootstrap._get_head_revision() TABLE = "channel_runtime_configs" COLUMNS = {"provider", "config_ciphertext", "disconnected", "version", "updated_at"} pytestmark = pytest.mark.asyncio diff --git a/backend/tests/test_migration_0046_skill_mutations.py b/backend/tests/test_migration_0046_skill_mutations.py new file mode 100644 index 0000000000..3d8a9f8870 --- /dev/null +++ b/backend/tests/test_migration_0046_skill_mutations.py @@ -0,0 +1,56 @@ +"""Additive Skill host schema and current migration-chain head.""" + +import asyncio + +import pytest +import sqlalchemy as sa +from alembic import command +from alembic.script import ScriptDirectory +from sqlalchemy.ext.asyncio import create_async_engine + +from deerflow.persistence import bootstrap +from deerflow.persistence.run.model import RunRow + +REVISION = "0046_skill_mutations" +TABLES = {"skill_mutation_assets", "skill_mutation_owners", "skill_mutation_proposals", "skill_mutation_operations", "skill_mutation_scan_attempts"} + + +def test_0046_is_the_chain_head(): + assert bootstrap._get_head_revision() == REVISION + script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR)) + assert script.get_revision(REVISION).down_revision == "0045_completed_run_evidence" + assert script.get_revision("0045_completed_run_evidence").down_revision == "0044_channel_runtime_configs" + assert script.get_revision("0044_channel_runtime_configs").down_revision == "0043_runtime_im_notices" + + +@pytest.mark.asyncio +async def test_0046_additive_upgrade_and_nullable_recovery_metadata(tmp_path): + engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'skill-host.db'}") + config = bootstrap._get_alembic_config(engine) + try: + await asyncio.to_thread(bootstrap._upgrade, config, "0045_completed_run_evidence") + # The historical create_all bootstrap uses today's metadata. Remove + # exactly the new, empty tables to represent a real 0045 installation. + async with engine.begin() as connection: + for table in sorted(TABLES): + await connection.execute(sa.text(f'DROP TABLE IF EXISTS "{table}"')) + await connection.execute(RunRow.__table__.insert().values(run_id="legacy", thread_id="thread", status="success")) + await connection.execute(sa.text("INSERT INTO channel_runtime_configs VALUES ('telegram', 'synthetic-ciphertext', 0, 7, 1.0)")) + await asyncio.to_thread(bootstrap._upgrade, config, REVISION) + async with engine.connect() as connection: + names = await connection.run_sync(lambda sync: sa.inspect(sync).get_table_names()) + assert TABLES <= set(names) + columns = await connection.run_sync(lambda sync: sa.inspect(sync).get_columns("skill_mutation_operations")) + nullable = {column["name"]: column["nullable"] for column in columns} + assert nullable["before_operation_id"] + assert nullable["superseded_by_generation"] + assert await connection.scalar(sa.text("SELECT status FROM runs WHERE run_id='legacy'")) == "success" + assert (await connection.execute(sa.text("SELECT config_ciphertext, version FROM channel_runtime_configs WHERE provider='telegram'"))).one() == ("synthetic-ciphertext", 7) + await asyncio.to_thread(command.downgrade, config, "0045_completed_run_evidence") + async with engine.connect() as connection: + names = await connection.run_sync(lambda sync: sa.inspect(sync).get_table_names()) + assert not TABLES.intersection(names) + assert await connection.scalar(sa.text("SELECT status FROM runs WHERE run_id='legacy'")) == "success" + assert (await connection.execute(sa.text("SELECT config_ciphertext, version FROM channel_runtime_configs WHERE provider='telegram'"))).one() == ("synthetic-ciphertext", 7) + finally: + await engine.dispose() diff --git a/backend/tests/test_run_event_store.py b/backend/tests/test_run_event_store.py index dcb4d660e9..893800e9a4 100644 --- a/backend/tests/test_run_event_store.py +++ b/backend/tests/test_run_event_store.py @@ -646,6 +646,7 @@ class TestDbRunEventStore: class FakeSession: def __init__(self): self.dialect = postgresql.dialect() + self.bind = self self.execute_calls = [] def get_bind(self): @@ -675,8 +676,13 @@ class TestDbRunEventStore: assert count == 3 assert session.execute_calls - assert "pg_advisory_xact_lock" in str(session.execute_calls[0][0]) - assert session.execute_calls[0][1] == {"thread_id": "thread-1"} + # The shared changed-run clock precedes every run-row mutation. + assert "run_change_clock" in str(session.execute_calls[0][0]) + assert "pg_advisory_xact_lock" in str(session.execute_calls[1][0]) + assert session.execute_calls[1][1] == {"thread_id": "thread-1"} + assert "UPDATE runs" in str(session.execute_calls[2][0]) + assert "DELETE FROM completed_run_snapshots" in str(session.execute_calls[3][0]) + assert "DELETE FROM run_events" in str(session.execute_calls[4][0]) @pytest.mark.anyio async def test_delete_by_run_takes_postgres_advisory_lock(self): @@ -688,6 +694,7 @@ class TestDbRunEventStore: class FakeSession: def __init__(self): self.dialect = postgresql.dialect() + self.bind = self self.execute_calls = [] def get_bind(self): @@ -723,8 +730,12 @@ class TestDbRunEventStore: assert count == 2 assert session.execute_calls - assert "pg_advisory_xact_lock" in str(session.execute_calls[0][0]) - assert session.execute_calls[0][1] == {"thread_id": "thread-1"} + assert "run_change_clock" in str(session.execute_calls[0][0]) + assert "pg_advisory_xact_lock" in str(session.execute_calls[1][0]) + assert session.execute_calls[1][1] == {"thread_id": "thread-1"} + assert "UPDATE runs" in str(session.execute_calls[2][0]) + assert "DELETE FROM completed_run_snapshots" in str(session.execute_calls[3][0]) + assert "DELETE FROM run_events" in str(session.execute_calls[4][0]) assert sum("pg_advisory_xact_lock" in str(call[0]) for call in session.execute_calls) == 1 @pytest.mark.anyio diff --git a/backend/tests/test_run_worker_rollback.py b/backend/tests/test_run_worker_rollback.py index aa5be987be..64c30a342f 100644 --- a/backend/tests/test_run_worker_rollback.py +++ b/backend/tests/test_run_worker_rollback.py @@ -1451,7 +1451,8 @@ async def test_terminal_cycle_collection_is_coalesced_contextless_and_off_loop(m @pytest.mark.anyio -async def test_run_agent_releases_terminal_runtime_callbacks(caplog): +@pytest.mark.parametrize("fail_checkpoint_inspection", [False, True]) +async def test_run_agent_releases_terminal_runtime_callbacks(caplog, fail_checkpoint_inspection): from deerflow.runtime.journal import RunJournal caplog.set_level(logging.DEBUG, logger="deerflow.runtime.runs.worker") @@ -1473,8 +1474,10 @@ async def test_run_agent_releases_terminal_runtime_callbacks(caplog): yield {"messages": []} async def aget_state(self, _config): - # Finalization reads checkpoint messages. A missing-method traceback - # retained by DEBUG log capture would itself keep the journal alive. + # A real checkpoint failure can retain the worker frame through a + # diagnostic traceback, just as a missing mock method can. + if fail_checkpoint_inspection: + raise RuntimeError("checkpoint inspection unavailable") return SimpleNamespace(values={"messages": []}) await run_agent( @@ -1492,7 +1495,10 @@ async def test_run_agent_releases_terminal_runtime_callbacks(caplog): stream_config = captured["config"] journal = captured["journal"] - assert "Failed to inspect trailing human_input_request" not in caplog.text + if fail_checkpoint_inspection: + assert any(record.exc_info and "Failed to inspect trailing human_input_request" in record.message for record in caplog.records) + else: + assert "Failed to inspect trailing human_input_request" not in caplog.text assert "__pregel_runtime" not in stream_config["configurable"] assert "__run_journal" not in stream_config["context"] assert journal not in (stream_config.get("callbacks") or []) diff --git a/backend/tests/test_skill_asset_versions.py b/backend/tests/test_skill_asset_versions.py new file mode 100644 index 0000000000..264d98a4a0 --- /dev/null +++ b/backend/tests/test_skill_asset_versions.py @@ -0,0 +1,401 @@ +"""Existing storage writers participate in the same version chain as plugins.""" + +import threading +from contextlib import contextmanager + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +from deerflow.config.paths import Paths +from deerflow.persistence.base import Base +from deerflow.skills.mutations.guard import SkillMutationRuntime, configure_mutation_runtime +from deerflow.skills.mutations.repository import SkillMutationRepository +from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage + +CONTENT = "---\nname: example\ndescription: Before\n---\nA\n" + + +@pytest.fixture +def assets(tmp_path, monkeypatch): + from deerflow.persistence.user.model import UserRow + + monkeypatch.setattr("deerflow.config.paths._paths", Paths(base_dir=tmp_path / "home")) + storage = UserScopedSkillStorage("owner", host_path=str(tmp_path / "skills")) + root = storage.get_custom_skill_dir("example") + root.mkdir(parents=True) + (root / "SKILL.md").write_text(CONTENT, encoding="utf-8") + engine = create_engine(f"sqlite:///{tmp_path / 'mutations.db'}") + Base.metadata.create_all(engine) + sessions = sessionmaker(engine, expire_on_commit=False) + with sessions.begin() as session: + session.add(UserRow(id="owner", email="owner@example.test")) + repository = SkillMutationRepository(sessions) + runtime = SkillMutationRuntime(repository, owners=frozenset({"owner"})) + configure_mutation_runtime(runtime) + # This suite checks canonical versions; view semantics have their own suite. + monkeypatch.setattr("deerflow.skills.projection._rebuild_user_locked", lambda *_: None) + yield storage, runtime, repository + configure_mutation_runtime(None) + engine.dispose() + + +def test_managed_aba_conflicts_even_when_digest_matches(assets): + storage, runtime, _ = assets + first = runtime.read_revision(storage, "example") + storage.write_custom_skill("example", "SKILL.md", CONTENT.replace("\nA\n", "\nB\n")) + storage.write_custom_skill("example", "SKILL.md", CONTENT) + last = runtime.read_revision(storage, "example") + assert last.content_digest == first.content_digest + assert last.incarnation_id == first.incarnation_id + assert last.mutation_seq == first.mutation_seq + 2 + + +def test_no_change_does_not_increment_revision(assets): + storage, runtime, _ = assets + first = runtime.read_revision(storage, "example") + storage.write_custom_skill("example", "SKILL.md", CONTENT) + assert runtime.read_revision(storage, "example") == first + + +def test_delete_recreate_changes_incarnation_and_retains_sequence(assets): + storage, runtime, _ = assets + first = runtime.read_revision(storage, "example") + storage.delete_custom_skill("example") + storage.write_custom_skill("example", "SKILL.md", CONTENT) + last = runtime.read_revision(storage, "example") + assert last.incarnation_id != first.incarnation_id + assert last.mutation_seq > first.mutation_seq + + +def test_support_file_and_toggle_changes_invalidate_prior_revision(assets): + storage, runtime, _ = assets + first = runtime.read_revision(storage, "example") + storage.write_custom_skill("example", "references/help.txt", "help") + second = runtime.read_revision(storage, "example") + assert second.mutation_seq == first.mutation_seq + 1 + storage.set_skill_enabled_state("example", False) + third = runtime.read_revision(storage, "example") + assert third.mutation_seq == second.mutation_seq + 1 + assert third.content_digest == second.content_digest + + +def test_other_skill_does_not_conflict(assets): + storage, runtime, _ = assets + first = runtime.read_revision(storage, "example") + storage.write_custom_skill("other", "SKILL.md", CONTENT.replace("example", "other")) + assert runtime.read_revision(storage, "example") == first + + +def test_observed_external_edit_advances_revision(assets): + storage, runtime, _ = assets + first = runtime.read_revision(storage, "example") + storage.get_custom_skill_file("example").write_text(CONTENT + "external", encoding="utf-8") + assert runtime.read_revision(storage, "example").mutation_seq == first.mutation_seq + 1 + + +def test_catalog_read_uses_owner_guard(assets, monkeypatch): + from contextlib import contextmanager + + storage, runtime, _ = assets + entered = [] + + @contextmanager + def guard(_storage, **_kwargs): + entered.append(True) + yield + + monkeypatch.setattr("deerflow.skills.projection.skill_projection_read_lock", guard) + storage.load_skills() + assert entered + + +def test_owner_prompt_cache_never_hides_other_worker_change(assets): + from deerflow.agents.lead_agent.prompt import get_enabled_skills_for_config + from deerflow.config.app_config import AppConfig + + storage, _, _ = assets + config = AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}, "skills": {"path": str(storage.get_skills_root_path())}}) + first = get_enabled_skills_for_config(config, "owner") + storage.write_custom_skill("example", "SKILL.md", CONTENT.replace("Before", "After")) + second = get_enabled_skills_for_config(config, "owner") + assert next(skill.description for skill in first if skill.name == "example") == "Before" + assert next(skill.description for skill in second if skill.name == "example") == "After" + + +def test_global_disable_participates_in_effective_revision(assets, monkeypatch): + from deerflow.config.extensions_config import ExtensionsConfig + + storage, runtime, _ = assets + first = runtime.read_revision(storage, "example") + monkeypatch.setattr(ExtensionsConfig, "from_file", lambda *a, **kw: ExtensionsConfig(skills={"example": {"enabled": False}})) + assert runtime.state(storage, "example")[1] is False + assert runtime.read_revision(storage, "example").mutation_seq > first.mutation_seq + + +def test_deleted_owner_cannot_read_catalog(assets): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + from deerflow.persistence.user.model import UserRow + + storage, _, repository = assets + with repository.sessions.begin() as session: + session.delete(session.get(UserRow, "owner")) + with pytest.raises(HostCapabilityError, match="NOT_FOUND_OR_FORBIDDEN"): + storage.load_skills() + + +def test_uncertain_legacy_write_blocks_catalog_until_revalidated(assets): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + storage, runtime, repository = assets + digest, enabled, exists = runtime.state(storage, "example") + repository.observe("owner", "example", digest, enabled, exists=exists, mark_mutating=True) + with pytest.raises(HostCapabilityError, match="NEEDS_REPAIR"): + storage.load_skills() + runtime.read_revision(storage, "example") + assert storage.load_skills() + + +@pytest.mark.parametrize("entry", ["shared", "thread", "rebuild", "slash"]) +def test_every_new_asset_acquire_rejects_unresolved_owner(assets, monkeypatch, entry): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + from deerflow.skills.projection import ensure_skill_projections, ensure_thread_skill_projection, rebuild_skill_projections + + storage, _, repository = assets + + def blocked(_): + raise HostCapabilityError("NEEDS_REPAIR") + + monkeypatch.setattr(repository, "check_readable", blocked) + # Even an otherwise fresh thread projection must honor the DB barrier. + monkeypatch.setattr("deerflow.skills.projection._thread_projection_is_fresh", lambda *_: True) + with pytest.raises(HostCapabilityError, match="NEEDS_REPAIR"): + if entry == "shared": + ensure_skill_projections(storage) + elif entry == "thread": + ensure_thread_skill_projection(storage, "thread", {"example"}) + elif entry == "rebuild": + rebuild_skill_projections(storage, include_public=False) + else: + from deerflow.agents.middlewares.skill_activation_middleware import SkillActivationMiddleware + + SkillActivationMiddleware._read_skill_content(storage.get_custom_skill_file("example"), storage.get_skills_root_path(), storage=storage) + + +def test_uncertain_owner_blocks_other_managed_writer_before_touching_source(assets): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + storage, runtime, repository = assets + digest, enabled, exists = runtime.state(storage, "example") + repository.observe("owner", "example", digest, enabled, exists=exists, mark_mutating=True) + with pytest.raises(HostCapabilityError, match="NEEDS_REPAIR"): + storage.write_custom_skill("other", "SKILL.md", CONTENT.replace("example", "other")) + assert not storage.get_custom_skill_file("other").exists() + + +def test_global_toggle_aba_invalidates_same_named_custom_asset(assets, monkeypatch, tmp_path): + from app.gateway.routers.skills import _write_extensions_skill_state + from deerflow.config.extensions_config import reload_extensions_config, reset_extensions_config + + storage, runtime, _ = assets + config_file = tmp_path / "extensions.json" + config_file.write_text('{"skills": {}}', encoding="utf-8") + monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(config_file)) + reload_extensions_config() + first = runtime.read_revision(storage, "example") + _write_extensions_skill_state(storage, "example", False, rebuild_public_projection=False) + _write_extensions_skill_state(storage, "example", True, rebuild_public_projection=False) + last = runtime.read_revision(storage, "example") + assert last.content_digest == first.content_digest + assert last.mutation_seq == first.mutation_seq + 2 + reset_extensions_config() + + +def test_global_toggle_does_not_lock_unrelated_owner(assets, monkeypatch, tmp_path): + from app.gateway.routers.skills import _write_extensions_skill_state + from deerflow.config.extensions_config import ExtensionsConfig, reload_extensions_config, reset_extensions_config + from deerflow.persistence.user.model import UserRow + from deerflow.skills.projection import skill_projection_read_lock + + storage, runtime, repository = assets + with repository.sessions.begin() as session: + session.add(UserRow(id="unrelated", email="unrelated@example.test")) + runtime.owners = frozenset({"owner", "unrelated"}) + config_file = tmp_path / "extensions.json" + config_file.write_text('{"skills": {}}', encoding="utf-8") + monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(config_file)) + reload_extensions_config() + + @contextmanager + def reject_unrelated(scoped, *args, **kwargs): + if scoped.user_id == "unrelated": + raise TimeoutError("unrelated owner is busy") + with skill_projection_read_lock(scoped, *args, **kwargs): + yield + + monkeypatch.setattr("deerflow.skills.projection.skill_projection_read_lock", reject_unrelated) + _write_extensions_skill_state(storage, "example", False, rebuild_public_projection=False) + assert ExtensionsConfig.from_file().is_skill_enabled("example", "public") is False + reset_extensions_config() + + +def test_global_toggle_acquires_name_fence_before_public_projection(assets, monkeypatch): + from deerflow.skills import projection + from deerflow.skills.mutations import guard + + storage, _runtime, _repository = assets + events = [] + + @contextmanager + def record_name_fence(*_args, **_kwargs): + events.append("name-enter") + try: + yield + finally: + events.append("name-exit") + + @contextmanager + def record_projection_lock(*_args, **_kwargs): + events.append("projection-enter") + try: + yield + finally: + events.append("projection-exit") + + monkeypatch.setattr(guard, "managed_name_writes", record_name_fence) + monkeypatch.setattr(projection, "_projection_lock", record_projection_lock) + + with guard.managed_global_state_write(storage, "missing"): + pass + + assert events == ["name-enter", "projection-enter", "projection-exit", "name-exit"] + + +def test_gateway_global_toggle_enters_managed_write_before_projection(assets, monkeypatch, tmp_path): + from app.gateway.routers.skills import _write_extensions_skill_state + from deerflow.config.extensions_config import reload_extensions_config, reset_extensions_config + from deerflow.skills import projection + from deerflow.skills.mutations import guard + + storage, _runtime, _repository = assets + config_file = tmp_path / "extensions.json" + config_file.write_text('{"skills": {}}', encoding="utf-8") + monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(config_file)) + reload_extensions_config() + events = [] + + @contextmanager + def record_managed_write(*_args, **_kwargs): + events.append("managed-enter") + try: + yield + finally: + events.append("managed-exit") + + @contextmanager + def record_projection(*_args, **_kwargs): + events.append("projection-enter") + try: + yield + finally: + events.append("projection-exit") + + monkeypatch.setattr(guard, "managed_global_state_write", record_managed_write) + monkeypatch.setattr(projection, "skill_projection_mutation", record_projection) + + try: + _write_extensions_skill_state(storage, "example", False, rebuild_public_projection=True) + finally: + reset_extensions_config() + + assert events == ["managed-enter", "projection-enter", "projection-exit", "managed-exit"] + + +def test_global_toggle_serializes_same_name_creation(assets, monkeypatch, tmp_path): + from app.gateway.routers import skills as skills_router + from deerflow.config.extensions_config import reload_extensions_config, reset_extensions_config + from deerflow.persistence.skill_mutations.model import SkillAssetRow + from deerflow.persistence.user.model import UserRow + + storage, runtime, repository = assets + with repository.sessions.begin() as session: + session.add(UserRow(id="new-owner", email="new-owner@example.test")) + runtime.owners = frozenset({"owner", "new-owner"}) + other = UserScopedSkillStorage("new-owner", host_path=str(storage.get_skills_root_path())) + config_file = tmp_path / "extensions.json" + config_file.write_text('{"skills": {}}', encoding="utf-8") + monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(config_file)) + reload_extensions_config() + + entered, release, writer_done = threading.Event(), threading.Event(), threading.Event() + errors = [] + write_config = skills_router.atomic_write_extensions_config + + def pause_config_write(path, raw): + entered.set() + if not release.wait(2): + raise TimeoutError("test did not release config write") + write_config(path, raw) + + def toggle(): + try: + skills_router._write_extensions_skill_state(storage, "example", False, rebuild_public_projection=False) + except BaseException as exc: + errors.append(exc) + + def create_same_name(): + try: + other.write_custom_skill("example", "SKILL.md", CONTENT) + except BaseException as exc: + errors.append(exc) + finally: + writer_done.set() + + monkeypatch.setattr(skills_router, "atomic_write_extensions_config", pause_config_write) + toggle_thread = threading.Thread(target=toggle) + writer_thread = threading.Thread(target=create_same_name) + toggle_thread.start() + assert entered.wait(2) + writer_thread.start() + try: + serialized = not writer_done.wait(0.2) + finally: + release.set() + toggle_thread.join(2) + writer_thread.join(2) + + assert not toggle_thread.is_alive() + assert not writer_thread.is_alive() + assert serialized + assert not errors + with repository.sessions() as session: + asset = session.scalar(select(SkillAssetRow).where(SkillAssetRow.owner_id == "new-owner", SkillAssetRow.name == "example")) + assert asset.enabled is False + reset_extensions_config() + + +def test_export_refuses_uncertain_canonical_state(assets): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + from deerflow.skills.export import export_manifest + + storage, runtime, repository = assets + digest, enabled, exists = runtime.state(storage, "example") + repository.observe("owner", "example", digest, enabled, exists=exists, mark_mutating=True) + with pytest.raises(HostCapabilityError, match="NEEDS_REPAIR"): + export_manifest(storage, "example") + + +@pytest.mark.asyncio +async def test_tool_candidate_copy_refuses_uncertain_canonical_state(assets): + from deerflow_extension_api.host_capabilities import HostCapabilityError + + from deerflow.tools.skill_manage_tool import _scan_static_candidate_or_raise + + storage, runtime, repository = assets + digest, enabled, exists = runtime.state(storage, "example") + repository.observe("owner", "example", digest, enabled, exists=exists, mark_mutating=True) + with pytest.raises(HostCapabilityError, match="NEEDS_REPAIR"): + await _scan_static_candidate_or_raise("example", {"help.txt": "help"}, storage) diff --git a/backend/tests/test_skill_evolution_host_integration.py b/backend/tests/test_skill_evolution_host_integration.py new file mode 100644 index 0000000000..52b3ecc15a --- /dev/null +++ b/backend/tests/test_skill_evolution_host_integration.py @@ -0,0 +1,83 @@ +"""Real SQLite, host assembly, evidence, projection and public-only consumer.""" + +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest +from deerflow_extension_api import ExtensionData, ExtensionRuntimeDeps, HostPolicySnapshot +from sqlalchemy import create_engine +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from deerflow.config.database_config import DatabaseConfig +from deerflow.config.paths import Paths +from deerflow.extensions.host_capabilities import HostCapabilities +from deerflow.extensions.loader import ExtensionSpec +from deerflow.persistence.base import Base +from deerflow.persistence.run import RunRepository +from deerflow.persistence.run.model import RunRow +from deerflow.persistence.user.model import UserRow +from deerflow.runtime.events.store.db import DbRunEventStore +from deerflow.skills.mutations.scanner import CandidateScanner, ScanVerdict +from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage + + +@pytest.mark.asyncio +async def test_public_consumer_end_to_end_and_plugin_removed_restart(tmp_path, monkeypatch): + monkeypatch.setattr("deerflow.config.paths._paths", Paths(base_dir=tmp_path / "home")) + storage = UserScopedSkillStorage("owner", host_path=str(tmp_path / "skills")) + root = storage.get_custom_skill_dir("example") + root.mkdir(parents=True) + content = "---\nname: example\ndescription: A sample skill\n---\nOriginal body.\n" + (root / "SKILL.md").write_text(content, encoding="utf-8") + database = DatabaseConfig(backend="sqlite", sqlite_dir=str(tmp_path)) + engine = create_engine(database.app_sync_sqlalchemy_url) + Base.metadata.create_all(engine) + engine.dispose() + async_engine = create_async_engine(database.app_sqlalchemy_url) + sf = async_sessionmaker(async_engine, expire_on_commit=False) + async with sf.begin() as session: + session.add(UserRow(id="owner", email="owner@example.test")) + session.add(RunRow(run_id="run", thread_id="thread", user_id="owner", status="success", evidence_seal_state="sealed", evidence_agent_id="lead_agent", evidence_origin="interactive", evidence_upper_seq=0, evidence_event_count=0)) + + async def scan(self, package, name): + assert name == "example" + assert package.main_content.startswith("---") + return ScanVerdict("allow", "CHECK_PASSED", "test-policy") + + async def policy(self): + return "test-policy" + + monkeypatch.setattr(CandidateScanner, "scan", scan) + monkeypatch.setattr(CandidateScanner, "policy_version", policy) + monkeypatch.setattr(CandidateScanner, "policy_version_sync", lambda _: "test-policy") + spec = ExtensionSpec( + use="fixture:install", + name="fixture", + host_access={ + "evidence": {"owners": ["owner"]}, + "skill_mutations": {"owners": ["owner"], "operations": ["stage", "check", "commit", "revert"], "trigger_agents": ["lead_agent"], "target_skills": ["example"], "topology": "single_host_local"}, + }, + ) + config = SimpleNamespace(database=database, plugins=[spec]) + host = HostCapabilities(config, RunRepository(sf), DbRunEventStore(sf), root=tmp_path, storage_factory=lambda _: storage) + try: + await host.start() + evidence, mutations = host.bindings[spec.use] + deps = ExtensionRuntimeDeps(app_store=ExtensionData("fixture"), policy=HostPolicySnapshot(), completed_run_evidence=evidence, skill_mutations=mutations) + module_spec = importlib.util.spec_from_file_location("external_consumer", Path(__file__).parent / "fixtures" / "skill_evolution_consumer.py") + consumer = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(consumer) + operation, reverted = await consumer.exercise(deps, thread_id="thread", run_id="run", name="example") + assert (operation.views, reverted.views) == ("READY", "READY") + assert (root / "SKILL.md").read_text(encoding="utf-8") == content + await host.close() + config.plugins = [] + host = HostCapabilities(config, RunRepository(sf), DbRunEventStore(sf), root=tmp_path, storage_factory=lambda _: storage) + await host.start() + assert host.bindings == {} + assert "owner" in host.runtime.owners + assert host.recovery.get_operation(reverted.operation_id).publication == "APPLIED" + finally: + await host.close() + await async_engine.dispose() diff --git a/backend/tests/test_skill_export.py b/backend/tests/test_skill_export.py index cddd378c48..2e852a8b8a 100644 --- a/backend/tests/test_skill_export.py +++ b/backend/tests/test_skill_export.py @@ -319,7 +319,7 @@ def test_temp_creation_and_cleanup_within_mutation(tmp_path, monkeypatch, user_s active = False @contextmanager - def locked(): + def locked(*, names=()): nonlocal active assert not root.exists() active = True diff --git a/backend/tests/test_skill_mutation_admin.py b/backend/tests/test_skill_mutation_admin.py new file mode 100644 index 0000000000..6dab4e5af3 --- /dev/null +++ b/backend/tests/test_skill_mutation_admin.py @@ -0,0 +1,144 @@ +"""Recovery is an authenticated host operation, never a plugin route.""" + +from dataclasses import dataclass +from types import SimpleNamespace + +import httpx +import pytest +from fastapi import FastAPI, Request + +from deerflow.skills.mutations.workers import MutationWorkers + + +@pytest.mark.asyncio +@pytest.mark.parametrize("role,source,status", [("user", None, 403), ("admin", "pat", 403), ("admin", None, 200)]) +async def test_admin_recovery_auth_and_off_loop(role, source, status): + import threading + + from app.gateway.routers.skill_mutations import router + + loop_thread = threading.get_ident() + calls = [] + + @dataclass + class Result: + operation_id: str + publication: str = "APPLIED" + + class Recovery: + def list_operations(self, *, limit, after_id): + assert threading.get_ident() != loop_thread + assert limit == 51 and after_id is None + calls.append("list") + return (Result("a" * 32),) + + def get_operation(self, operation_id): + assert threading.get_ident() != loop_thread + calls.append("query") + return Result(operation_id) + + def recover_operation(self, operation_id): + assert threading.get_ident() != loop_thread + calls.append("recover") + return Result(operation_id) + + app = FastAPI() + workers = MutationWorkers() + app.state.skill_mutation_host = SimpleNamespace(workers=workers, recovery=Recovery()) + + @app.middleware("http") + async def identity(request: Request, call_next): + request.state.user = SimpleNamespace(id="admin", system_role=role) + request.state.auth_source = source + return await call_next(request) + + app.include_router(router) + try: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client: + base = "/api/skill-mutations/operations/" + "a" * 32 + listed = await client.get("/api/skill-mutations/operations") + assert listed.status_code == status + if status == 200: + assert listed.json()["items"][0]["operation_id"] == "a" * 32 + assert (await client.get(base)).status_code == status + assert (await client.post(base + "/recover")).status_code == status + assert calls == (["list", "query", "recover"] if status == 200 else []) + finally: + await workers.close() + + +@pytest.mark.asyncio +async def test_admin_errors_do_not_return_database_or_candidate_content(): + from deerflow_extension_api import HostCapabilityError + + from app.gateway.routers.skill_mutations import router + + class Recovery: + def get_operation(self, operation_id): + raise HostCapabilityError("NOT_FOUND_OR_FORBIDDEN", "candidate content must not leak") + + app = FastAPI() + workers = MutationWorkers() + app.state.skill_mutation_host = SimpleNamespace(workers=workers, recovery=Recovery()) + + @app.middleware("http") + async def identity(request: Request, call_next): + request.state.user = SimpleNamespace(id="admin", system_role="admin") + return await call_next(request) + + app.include_router(router) + try: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client: + response = await client.get("/api/skill-mutations/operations/" + "a" * 32) + assert response.status_code == 404 + assert "candidate content" not in response.text + finally: + await workers.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("count,limit", [(0, 2), (1, 2), (2, 2), (3, 2), (3, 1)]) +async def test_operation_pagination_terminates_with_null_cursor(count, limit): + from app.gateway.routers.skill_mutations import router + + @dataclass + class Result: + operation_id: str + + operations = [Result(f"{number:032x}") for number in range(1, count + 1)] + + class Recovery: + def list_operations(self, *, limit, after_id): + return tuple(row for row in operations if after_id is None or row.operation_id > after_id)[:limit] + + app = FastAPI() + workers = MutationWorkers() + app.state.skill_mutation_host = SimpleNamespace(workers=workers, recovery=Recovery()) + + @app.middleware("http") + async def identity(request: Request, call_next): + request.state.user = SimpleNamespace(id="admin", system_role="admin") + return await call_next(request) + + app.include_router(router) + try: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client: + params = {"limit": limit} + seen = [] + for _ in range(count + 1): + response = await client.get("/api/skill-mutations/operations", params=params) + assert response.status_code == 200 + page = response.json() + seen.extend(row["operation_id"] for row in page["items"]) + assert page["has_more"] == (page["next_cursor"] is not None) + if page["next_cursor"] is None: + break + params["after_id"] = page["next_cursor"] + else: + pytest.fail("Pagination failed to terminate") + assert seen == [row.operation_id for row in operations] + # A stale end cursor must not echo itself back forever. + empty = await client.get("/api/skill-mutations/operations", params={"after_id": "f" * 32, "limit": limit}) + assert empty.json() == {"items": [], "has_more": False, "next_cursor": None} + finally: + await workers.close() diff --git a/backend/tests/test_skill_mutation_async_reads.py b/backend/tests/test_skill_mutation_async_reads.py new file mode 100644 index 0000000000..928071a358 --- /dev/null +++ b/backend/tests/test_skill_mutation_async_reads.py @@ -0,0 +1,20 @@ +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + + +@pytest.mark.asyncio +async def test_gateway_catalog_never_takes_managed_guard_on_event_loop(monkeypatch): + from app.gateway.routers import skills + + loop_thread = threading.get_ident() + + def load_skills(**kwargs): + assert threading.get_ident() != loop_thread + return [] + + monkeypatch.setattr(skills, "_get_user_skill_storage", lambda _: SimpleNamespace(load_skills=load_skills)) + monkeypatch.setattr(skills, "_filter_visible_skills", AsyncMock(return_value=[])) + assert (await skills.list_skills(SimpleNamespace(), SimpleNamespace())).skills == [] diff --git a/backend/tests/test_skill_mutation_guard.py b/backend/tests/test_skill_mutation_guard.py new file mode 100644 index 0000000000..422420a01f --- /dev/null +++ b/backend/tests/test_skill_mutation_guard.py @@ -0,0 +1,66 @@ +"""The same owner lock protects exports, old writers and new publication.""" + +import os +import subprocess +import sys +import threading + +import pytest + +from deerflow.skills.projection import _projection_lock, skill_projection_read_lock +from deerflow.skills.storage.local_skill_storage import LocalSkillStorage + + +def test_read_guard_is_reentrant_inside_existing_writer(tmp_path): + storage = LocalSkillStorage(host_path=str(tmp_path)) + with _projection_lock(tmp_path / "custom"): + with skill_projection_read_lock(storage, timeout=0.05): + pass + + +def test_guard_wait_is_bounded_across_threads(tmp_path): + storage = LocalSkillStorage(host_path=str(tmp_path)) + held, release = threading.Event(), threading.Event() + + def holder(): + with _projection_lock(tmp_path / "custom"): + held.set() + release.wait(5) + + thread = threading.Thread(target=holder) + thread.start() + try: + assert held.wait(2) + with pytest.raises(TimeoutError): + with skill_projection_read_lock(storage, timeout=0.05): + pytest.fail("Concurrent writer lock was bypassed") + finally: + release.set() + thread.join(2) + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX publication locks") +def test_another_process_cannot_bypass_owner_guard_and_kill_releases_it(tmp_path): + script = """ +import sys +from pathlib import Path +from deerflow.skills.projection import _projection_lock +with _projection_lock(Path(sys.argv[1]) / 'custom'): + print('locked', flush=True) + sys.stdin.read() +""" + child = subprocess.Popen([sys.executable, "-c", script, str(tmp_path)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + try: + assert child.stdout.readline().strip() == "locked" + storage = LocalSkillStorage(host_path=str(tmp_path)) + with pytest.raises(TimeoutError): + with skill_projection_read_lock(storage, timeout=0.05): + pytest.fail("process bypassed owner lock") + child.kill() + child.wait(timeout=5) + with skill_projection_read_lock(storage, timeout=0.5): + pass + finally: + if child.poll() is None: + child.kill() + child.communicate(timeout=5) diff --git a/backend/tests/test_skill_mutation_host_lifecycle.py b/backend/tests/test_skill_mutation_host_lifecycle.py new file mode 100644 index 0000000000..ad5a4ccc39 --- /dev/null +++ b/backend/tests/test_skill_mutation_host_lifecycle.py @@ -0,0 +1,312 @@ +"""The host owns recovery even when every extension is disabled.""" + +import asyncio +import threading +from contextlib import contextmanager +from types import SimpleNamespace + +import pytest +import pytest_asyncio +from sqlalchemy import create_engine +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from sqlalchemy.orm import Session + +from deerflow.config.database_config import DatabaseConfig +from deerflow.config.run_events_config import RunEventsConfig +from deerflow.extensions.host_access import HostAccess +from deerflow.extensions.loader import ExtensionSpec +from deerflow.persistence.base import Base +from deerflow.persistence.run import RunRepository +from deerflow.persistence.skill_mutations.model import SkillOwnerRow, SkillProposalRow +from deerflow.persistence.user.model import UserRow +from deerflow.runtime.events.store.db import DbRunEventStore + + +@pytest_asyncio.fixture +async def durable(tmp_path): + database = DatabaseConfig(backend="sqlite", sqlite_dir=str(tmp_path)) + sync = create_engine(database.app_sync_sqlalchemy_url) + Base.metadata.create_all(sync) + with Session(sync) as session, session.begin(): + session.add(UserRow(id="owner", email="owner@example.test")) + sync.dispose() + engine = create_async_engine(database.app_sqlalchemy_url) + sf = async_sessionmaker(engine, expire_on_commit=False) + yield database, sf + await engine.dispose() + + +def config(database, plugins=()): + return SimpleNamespace(database=database, run_events=RunEventsConfig(backend="db"), plugins=plugins) + + +def granted(): + return ExtensionSpec( + use="example:install", + name="evolution", + host_access=HostAccess.model_validate( + { + "evidence": {"owners": ["owner"]}, + "skill_mutations": {"owners": ["owner"], "operations": ["stage", "check", "commit", "revert"], "trigger_agents": ["lead_agent"], "target_skills": ["example"], "topology": "single_host_local"}, + } + ), + ) + + +@pytest.mark.asyncio +async def test_default_has_no_mutation_runtime_or_worker_pool(durable, tmp_path): + from deerflow.extensions.host_capabilities import HostCapabilities + + database, sf = durable + host = HostCapabilities(config(database), RunRepository(sf), DbRunEventStore(sf), root=tmp_path) + await host.start() + try: + assert host.bindings == {} + assert host.runtime is None + assert host.recovery is None + finally: + await host.close() + + +@pytest.mark.asyncio +async def test_disabled_plugin_still_recovers_enrolled_owners(durable, tmp_path): + from deerflow.extensions.host_capabilities import HostCapabilities + + database, sf = durable + async with sf.begin() as session: + session.add(SkillOwnerRow(owner_id="owner", generation=4, deleting=False)) + events = [] + + class Recovery: + def __init__(self, runtime, storage_factory): + assert runtime.owners == frozenset({"owner"}) + events.append("constructed") + + def recover_all(self): + events.append("recovered") + + def collect_garbage(self): + events.append("collected") + + host = HostCapabilities(config(database), RunRepository(sf), DbRunEventStore(sf), root=tmp_path, recovery_factory=Recovery) + await host.start() + try: + assert host.bindings == {} + assert events == ["constructed", "recovered", "collected"] + assert host.runtime.owners == frozenset({"owner"}) + finally: + await host.close() + assert host.workers._closed + + +@pytest.mark.asyncio +async def test_source_binding_and_independent_evidence_scope(durable, tmp_path): + from deerflow.extensions.host_capabilities import HostCapabilities + + database, sf = durable + seen = [] + + class Recovery: + def __init__(self, *args): + pass + + def recover_all(self): + pass + + def collect_garbage(self): + pass + + def service(binding, **kwargs): + seen.append((binding, kwargs)) + return object() + + host = HostCapabilities(config(database, [granted()]), RunRepository(sf), DbRunEventStore(sf), root=tmp_path, recovery_factory=Recovery, service_factory=service) + await host.start() + try: + reader, mutations = host.bindings["example:install"] + assert reader is seen[0][1]["evidence"] + assert reader._owners == frozenset({"owner"}) + assert mutations is not None + assert seen[0][0].plugin_id != "evolution" + finally: + await host.close() + + +@pytest.mark.asyncio +async def test_missing_topology_refuses_mutation_startup(durable, tmp_path): + from deerflow_extension_api import HostCapabilityError + + from deerflow.extensions.host_capabilities import HostCapabilities + + database, sf = durable + spec = granted() + spec = spec.model_copy(update={"host_access": spec.host_access.model_copy(update={"skill_mutations": spec.host_access.skill_mutations.model_copy(update={"topology": None})})}) + host = HostCapabilities(config(database, [spec]), RunRepository(sf), DbRunEventStore(sf), root=tmp_path) + try: + with pytest.raises(HostCapabilityError, match="UNSUPPORTED_TOPOLOGY"): + await host.start() + finally: + await host.close() + + +@pytest.mark.asyncio +async def test_cancelled_startup_drains_worker_before_releasing_topology(durable, tmp_path): + from deerflow.extensions.host_capabilities import HostCapabilities + from deerflow.skills.mutations.topology import PublicationTopology + + database, sf = durable + entered, release = threading.Event(), threading.Event() + + class Recovery: + def __init__(self, *args): + pass + + def recover_all(self): + entered.set() + assert release.wait(5) + + def collect_garbage(self): + pass + + host = HostCapabilities(config(database, [granted()]), RunRepository(sf), DbRunEventStore(sf), root=tmp_path, recovery_factory=Recovery) + task = asyncio.create_task(host.start()) + assert await asyncio.to_thread(entered.wait, 5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + closing = asyncio.create_task(host.close()) + await asyncio.sleep(0) + assert not closing.done() + assert host._topology._fd is not None + release.set() + await closing + next_host = PublicationTopology("sqlite", tmp_path) + next_host.acquire() + next_host.close() + + +@pytest.mark.asyncio +async def test_read_only_evidence_reports_unsupported_without_mutation_pool(tmp_path): + from deerflow_extension_api import HostCapabilityError + + from deerflow.extensions.host_capabilities import HostCapabilities + from deerflow.runtime.events.store.memory import MemoryRunEventStore + from deerflow.runtime.runs.store.memory import MemoryRunStore + + spec = ExtensionSpec(use="example:install", name="reader", host_access=HostAccess.model_validate({"evidence": {"owners": ["owner"]}})) + host = HostCapabilities(config(DatabaseConfig(), [spec]), MemoryRunStore(), MemoryRunEventStore(), root=tmp_path) + await host.start() + try: + reader, mutations = host.bindings["example:install"] + assert mutations is None + assert host.workers is None + with pytest.raises(HostCapabilityError, match="UNSUPPORTED"): + await reader.list_changed_runs() + finally: + await host.close() + + +@pytest.mark.asyncio +async def test_mutation_owner_without_evidence_grant_is_rejected(durable, tmp_path): + from deerflow_extension_api import HostCapabilityError + + from deerflow.extensions.host_capabilities import HostCapabilities + + database, sf = durable + spec = granted() + spec.host_access = spec.host_access.model_copy(update={"evidence": HostAccess().evidence}) + host = HostCapabilities(config(database, [spec]), RunRepository(sf), DbRunEventStore(sf), root=tmp_path) + try: + with pytest.raises(HostCapabilityError, match="INVALID_GRANT"): + await host.start() + assert host.workers is None + finally: + await host.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("phase", ["startup", "maintenance"]) +@pytest.mark.parametrize("failure", ["lock-directory", "lock-timeout"]) +async def test_owner_gc_failure_does_not_block_host_or_healthy_owner(durable, tmp_path, monkeypatch, phase, failure): + from deerflow.config.paths import Paths + from deerflow.extensions.host_capabilities import HostCapabilities + from deerflow.skills.mutations import recovery as recovery_module + from deerflow.skills.projection import get_skill_projection_paths + from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage + + database, sf = durable + monkeypatch.setattr("deerflow.config.paths._paths", Paths(base_dir=tmp_path / "home")) + owners = ("a-broken", "z-healthy") + async with sf.begin() as session: + for owner_id in owners: + session.add(UserRow(id=owner_id, email=f"{owner_id}@example.test")) + session.add(SkillOwnerRow(owner_id=owner_id, generation=0, deleting=False)) + + def storage_factory(owner_id): + return UserScopedSkillStorage(owner_id, host_path=str(tmp_path / "skills")) + + async def expire_proposals(): + async with sf.begin() as session: + for owner_id in owners: + session.add( + SkillProposalRow( + proposal_id=owner_id, + plugin_id="removed-plugin", + owner_id=owner_id, + target_ref=owner_id, + name="example", + base_revision={}, + sources=[], + candidate_hash="unused", + package_blob=b"baseline", + candidate_blob=b"candidate", + state="PENDING", + expires_at=0, + idempotency_key=owner_id, + request_hash="unused", + ) + ) + + def break_owner_lock(): + if failure == "lock-directory": + root = get_skill_projection_paths(storage_factory(owners[0])).custom.parent + lock_path = root.parent / f".{root.name}.projection.lock" + # During maintenance an earlier successful acquire made a file. + if lock_path.is_file(): + lock_path.unlink() + lock_path.mkdir(parents=True) + else: + original_lock = recovery_module.skill_projection_read_lock + + @contextmanager + def blocked_lock(storage, **kwargs): + if storage.user_id == owners[0]: + raise TimeoutError("owner lock unavailable") + with original_lock(storage, **kwargs): + yield + + monkeypatch.setattr(recovery_module, "skill_projection_read_lock", blocked_lock) + + # No enabled plugin: persisted host state alone must remain recoverable. + host = HostCapabilities(config(database), RunRepository(sf), DbRunEventStore(sf), root=tmp_path, storage_factory=storage_factory) + try: + if phase == "startup": + await expire_proposals() + break_owner_lock() + await host.start() + if phase == "maintenance": + await expire_proposals() + break_owner_lock() + await host.workers.run(lambda _: (host.recovery.recover_all(), host.recovery.collect_garbage())) + assert host.bindings == {} + assert host.recovery is not None + assert not host._maintenance.done() + with host.runtime.repository.sessions() as session: + broken = session.get(SkillProposalRow, owners[0]) + healthy = session.get(SkillProposalRow, owners[1]) + assert broken.state == "PENDING" + assert broken.package_blob == b"baseline" and broken.candidate_blob == b"candidate" + assert healthy.state == "EXPIRED" + assert healthy.package_blob is None and healthy.candidate_blob is None + finally: + await host.close() diff --git a/backend/tests/test_skill_mutation_package_cost.py b/backend/tests/test_skill_mutation_package_cost.py new file mode 100644 index 0000000000..7e17c2c248 --- /dev/null +++ b/backend/tests/test_skill_mutation_package_cost.py @@ -0,0 +1,24 @@ +"""Reproducible local capture measurements; not a model/provider benchmark.""" + +import statistics +import time + +import pytest + +from deerflow.skills.mutations.assets import capture_package + + +@pytest.mark.parametrize("files,size", [(4, 1024), (256, 65535)]) +def test_capture_cost_at_typical_and_package_limit(tmp_path, files, size): + (tmp_path / "SKILL.md").write_text("---\nname: example\ndescription: Example\n---\nBody\n", encoding="utf-8") + for index in range(files - 1): + (tmp_path / f"resource-{index:03}.bin").write_bytes(b"x" * size) + timings, digests = [], set() + for _ in range(5): + start = time.perf_counter() + package = capture_package(tmp_path) + digests.add(package.digest) + timings.append((time.perf_counter() - start) * 1000) + assert len(digests) == 1 + assert len(package.files) == files + print(f"capture+digest files={files} bytes={sum(len(item.content) for item in package.files)} median_ms={statistics.median(timings):.2f} max_ms={max(timings):.2f}") diff --git a/backend/tests/test_skill_mutation_postgres.py b/backend/tests/test_skill_mutation_postgres.py new file mode 100644 index 0000000000..1f9a8fa49c --- /dev/null +++ b/backend/tests/test_skill_mutation_postgres.py @@ -0,0 +1,69 @@ +"""Opt-in live PostgreSQL fence test; never substitute SQLite for PG locking.""" + +import asyncio +import os +import threading +import uuid + +import pytest +from sqlalchemy import create_engine, select, text +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from deerflow.config.database_config import DatabaseConfig +from deerflow.persistence.base import Base +from deerflow.persistence.run import RunRepository +from deerflow.persistence.run.model import CompletedRunSnapshotRow, RunRow +from deerflow.skills.mutations.topology import mutation_session_factory + +pytestmark = pytest.mark.skipif(not os.environ.get("DEERFLOW_TEST_POSTGRES_URL"), reason="requires an explicit live DEERFLOW_TEST_POSTGRES_URL") + + +@pytest.mark.asyncio +async def test_run_deletion_waits_for_publication_source_share_lock(): + schema = "skill_evolution_test_" + uuid.uuid4().hex + config = DatabaseConfig(backend="postgres", postgres_url=os.environ["DEERFLOW_TEST_POSTGRES_URL"], postgres_schema=schema) + admin = create_engine(config.app_sync_sqlalchemy_url) + with admin.begin() as connection: + connection.execute(text(f'CREATE SCHEMA "{schema}"')) + sync_engine, sessions = mutation_session_factory(config.app_sync_sqlalchemy_url, postgres_schema=schema) + async_engine = create_async_engine(config.app_sqlalchemy_url, connect_args={"server_settings": {"search_path": schema}}) + sf = async_sessionmaker(async_engine, expire_on_commit=False) + release, acquired = threading.Event(), threading.Event() + worker = None + deleting = None + try: + Base.metadata.create_all(sync_engine) + with sessions.begin() as session: + session.add(RunRow(run_id="run", thread_id="thread", user_id="owner", status="success")) + session.flush() + session.add(CompletedRunSnapshotRow(snapshot_ref="snapshot", run_id="run", scope_digest="scope", evidence_revision="revision", retention_revision=0, snapshot_json={})) + + def publication_fence(): + with sessions.begin() as session: + session.scalar(select(RunRow).where(RunRow.run_id == "run").with_for_update(read=True)) + acquired.set() + assert release.wait(5) + + worker = asyncio.create_task(asyncio.to_thread(publication_fence)) + assert await asyncio.to_thread(acquired.wait, 5) + deleting = asyncio.create_task(RunRepository(sf).delete("run", user_id="owner")) + await asyncio.sleep(0.2) + assert not deleting.done(), "source deletion must wait for the accepted publication fence" + release.set() + await asyncio.wait_for(deleting, 5) + await worker + async with sf() as session: + assert await session.get(RunRow, "run") is None + assert await session.get(CompletedRunSnapshotRow, "snapshot") is None + finally: + release.set() + if worker is not None: + await worker + if deleting is not None: + await asyncio.gather(deleting, return_exceptions=True) + await async_engine.dispose() + sync_engine.dispose() + # Exact, randomly-created test schema only; never a configured schema. + with admin.begin() as connection: + connection.execute(text(f'DROP SCHEMA "{schema}" CASCADE')) + admin.dispose() diff --git a/backend/tests/test_skill_mutation_process_crash.py b/backend/tests/test_skill_mutation_process_crash.py new file mode 100644 index 0000000000..73e307ca9c --- /dev/null +++ b/backend/tests/test_skill_mutation_process_crash.py @@ -0,0 +1,50 @@ +"""Real SIGKILL verifies journal/file ordering, not exception-only simulation.""" + +import asyncio +import json +import os +import signal +import sys +from pathlib import Path + +import pytest +from test_skill_mutations import CONTENT, NEW, host, stage # noqa: F401 -- shared real-SQLite fixture + + +@pytest.mark.skipif(os.name != "posix", reason="P0 publication supports POSIX only") +@pytest.mark.asyncio +@pytest.mark.parametrize( + "boundary,expected", + [ + ("prepared", "ABORTED"), + ("written", "ABORTED"), + ("file_synced", "ABORTED"), + ("renamed", "APPLIED"), + ("directory_synced", "APPLIED"), + ("response_lost", "APPLIED"), + ], +) +async def test_sigkill_recovers_without_replay_or_double_version(host, boundary, expected): # noqa: F811 -- injected fixture + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + args = { + "base": str(host.storage._paths.base_dir), + "skills": str(host.storage.get_skills_root_path()), + "database": str(host.sessions.kw["bind"].url), + "grant": host.binding.access.model_dump(mode="json"), + "proposal": proposal.proposal_id, + "boundary": boundary, + } + child = await asyncio.create_subprocess_exec(sys.executable, str(Path(__file__).parent / "fixtures" / "skill_mutation_crash_worker.py"), json.dumps(args), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + _, stderr = await asyncio.wait_for(child.communicate(), timeout=30) + assert child.returncode == -signal.SIGKILL, stderr.decode() + await asyncio.to_thread(host.recovery.recover_owner, "owner") + operation = await host.service.find_operation(idempotency_key="crash-commit") + assert operation.publication == expected + expected_content = NEW if expected == "APPLIED" else CONTENT + assert host.storage.get_custom_skill_file("example").read_text(encoding="utf-8") == expected_content + first_revision = host.runtime.read_revision(host.storage, "example") + await asyncio.to_thread(host.recovery.recover_owner, "owner") + assert host.runtime.read_revision(host.storage, "example") == first_revision + assert first_revision == (operation.after_revision if expected == "APPLIED" else operation.before_revision) + assert not list(host.storage.get_custom_skill_dir("example").parent.glob(".host-mutation-*")) diff --git a/backend/tests/test_skill_mutation_scanner.py b/backend/tests/test_skill_mutation_scanner.py new file mode 100644 index 0000000000..5ca480bc65 --- /dev/null +++ b/backend/tests/test_skill_mutation_scanner.py @@ -0,0 +1,72 @@ +from unittest.mock import AsyncMock + +import pytest + +from deerflow.config.app_config import AppConfig +from deerflow.skills.mutations.assets import PackageFile, PackageSnapshot +from deerflow.skills.security_scanner import ScanResult + +CONTENT = b"---\nname: example\ndescription: Fine\n---\nHelpful instructions.\n" + + +def configuration(enabled=True): + return AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}, "skill_scan": {"enabled": enabled}, "skill_evolution": {"security_fail_closed": False}}) + + +@pytest.mark.asyncio +async def test_host_check_never_treats_disabled_scanner_as_allow(monkeypatch): + from deerflow.skills.mutations.scanner import CandidateScanner + + model = AsyncMock(return_value=ScanResult("allow", "fine")) + monkeypatch.setattr("deerflow.skills.mutations.scanner.scan_skill_content", model) + result = await CandidateScanner(lambda: configuration(False)).scan(PackageSnapshot((PackageFile("SKILL.md", CONTENT, False),)), "example") + assert result.decision == "unavailable" + model.assert_not_called() + + +@pytest.mark.asyncio +async def test_host_scan_covers_support_files_and_forces_fail_closed(monkeypatch): + from deerflow.skills.mutations.scanner import CandidateScanner + + model = AsyncMock(return_value=ScanResult("allow", "fine")) + monkeypatch.setattr("deerflow.skills.mutations.scanner.scan_skill_content", model) + scanner = CandidateScanner(configuration) + package = PackageSnapshot((PackageFile("SKILL.md", CONTENT, False), PackageFile("references/help.md", b"Supporting advice", False))) + result = await scanner.scan(package, "example") + assert result.decision == "allow" + assert result.policy_version == await scanner.policy_version() + assert model.await_count == 2 + assert all(call.kwargs["app_config"].skill_evolution.security_fail_closed for call in model.await_args_list) + + +@pytest.mark.asyncio +async def test_static_block_prevents_model_scan(monkeypatch): + from deerflow.skills.mutations.scanner import CandidateScanner + + model = AsyncMock(return_value=ScanResult("allow", "fine")) + monkeypatch.setattr("deerflow.skills.mutations.scanner.scan_skill_content", model) + package = PackageSnapshot((PackageFile("SKILL.md", CONTENT, False), PackageFile("scripts/run.py", b"eval(input())", True))) + result = await CandidateScanner(configuration).scan(package, "example") + assert result.decision == "reject" + model.assert_not_called() + + +@pytest.mark.asyncio +async def test_scanner_resource_limit_is_not_partial_approval(monkeypatch): + from deerflow.skills.mutations.scanner import CandidateScanner + + model = AsyncMock(return_value=ScanResult("allow", "fine")) + monkeypatch.setattr("deerflow.skills.mutations.scanner.scan_skill_content", model) + package = PackageSnapshot((PackageFile("SKILL.md", CONTENT, False), PackageFile("notes.txt", b"x" * (128 * 1024 + 1), False))) + result = await CandidateScanner(configuration).scan(package, "example") + assert result.decision == "unavailable" + model.assert_not_called() + + +@pytest.mark.asyncio +async def test_host_scan_timeout_propagates_without_authoritative_verdict(monkeypatch): + from deerflow.skills.mutations.scanner import CandidateScanner + + monkeypatch.setattr("deerflow.skills.mutations.scanner.CHECK_TIMEOUT_SECONDS", 0) + with pytest.raises(TimeoutError): + await CandidateScanner(configuration).scan(PackageSnapshot((PackageFile("SKILL.md", CONTENT, False),)), "example") diff --git a/backend/tests/test_skill_mutation_topology.py b/backend/tests/test_skill_mutation_topology.py new file mode 100644 index 0000000000..7093316edc --- /dev/null +++ b/backend/tests/test_skill_mutation_topology.py @@ -0,0 +1,42 @@ +import os + +import pytest +from deerflow_extension_api.host_capabilities import HostCapabilityError + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX publication contract") +def test_sqlite_single_gateway_lease_is_exclusive_and_recoverable(tmp_path): + from deerflow.skills.mutations.topology import PublicationTopology + + first = PublicationTopology("sqlite", tmp_path) + first.acquire() + try: + with pytest.raises(HostCapabilityError, match="UNSUPPORTED_TOPOLOGY"): + PublicationTopology("sqlite", tmp_path).acquire() + finally: + first.close() + second = PublicationTopology("sqlite", tmp_path) + second.acquire() + second.close() + + +@pytest.mark.parametrize("backend", ["memory", "other"]) +def test_unsupported_backend_fails_closed(tmp_path, backend): + from deerflow.skills.mutations.topology import PublicationTopology + + with pytest.raises(HostCapabilityError, match="UNSUPPORTED_TOPOLOGY"): + PublicationTopology(backend, tmp_path).acquire() + + +def test_sync_journal_uses_full_durability(tmp_path): + from deerflow.skills.mutations.topology import mutation_session_factory + + engine, sessions = mutation_session_factory(f"sqlite:///{tmp_path / 'journal.db'}") + try: + with sessions() as session: + from sqlalchemy import text + + assert session.scalar(text("PRAGMA synchronous")) == 2 + assert session.scalar(text("PRAGMA foreign_keys")) == 1 + finally: + engine.dispose() diff --git a/backend/tests/test_skill_mutation_validation.py b/backend/tests/test_skill_mutation_validation.py new file mode 100644 index 0000000000..5f30babce2 --- /dev/null +++ b/backend/tests/test_skill_mutation_validation.py @@ -0,0 +1,117 @@ +"""P0 changes preserve authority declarations and capture a bounded exact package.""" + +import os + +import pytest + +from deerflow.skills.mutations.assets import capture_package +from deerflow.skills.mutations.validation import validate_candidate + +BASE = "---\nname: example\ndescription: Before\nallowed-tools: [Read]\nsecrets-autonomous: false\nmetadata: {custom: 1}\nfuture-field: {enabled: true}\n---\nBefore body\n" + + +def test_only_body_and_description_may_change(): + validate_candidate(BASE, BASE.replace("Before", "After"), "example") + + +@pytest.mark.parametrize( + "old,new", + [ + ("allowed-tools: [Read]\n", ""), + ("allowed-tools: [Read]", "allowed-tools: []"), + ("secrets-autonomous: false", "secrets-autonomous: true"), + ("custom: 1", "custom: true"), + ("future-field: {enabled: true}\n", ""), + ("name: example", "name: renamed"), + ], +) +def test_behavior_and_unknown_fields_preserve_values_and_types(old, new): + with pytest.raises(ValueError, match="FRONTMATTER_CHANGED"): + validate_candidate(BASE, BASE.replace(old, new), "example") + + +@pytest.mark.parametrize("extra", ["description: Duplicate\n", "metadata: &x {a: 1}\nfuture: *x\n", "metadata: {a: 1, a: 2}\n", "1: value\n"]) +def test_ambiguous_yaml_is_rejected(extra): + candidate = "---\nname: example\ndescription: Valid\n" + extra + "---\nBody" + with pytest.raises(ValueError, match="INVALID_FRONTMATTER"): + validate_candidate(candidate, candidate, "example") + + +@pytest.mark.parametrize("description", ["", "[]", "null", "''"]) +def test_description_remains_runtime_valid(description): + with pytest.raises(ValueError, match="INVALID_FRONTMATTER"): + validate_candidate(BASE, BASE.replace("description: Before", f"description: {description}"), "example") + + +def test_main_file_size_is_bounded(): + with pytest.raises(ValueError, match="QUOTA_EXCEEDED"): + validate_candidate(BASE, BASE + "x" * (128 * 1024), "example") + + +def test_non_finite_frontmatter_has_stable_error_code(): + content = BASE.replace("custom: 1", "custom: .nan") + with pytest.raises(ValueError, match="INVALID_FRONTMATTER"): + validate_candidate(content, content, "example") + + +def test_package_snapshot_covers_support_bytes_and_execute_flags(tmp_path): + (tmp_path / "SKILL.md").write_text(BASE, encoding="utf-8") + support = tmp_path / "script.sh" + support.write_bytes(b"echo before\n") + first = capture_package(tmp_path) + assert first.main_content == BASE + assert first == capture_package(tmp_path) + support.write_bytes(b"echo after\n") + assert first.digest != capture_package(tmp_path).digest + before_mode = capture_package(tmp_path) + support.chmod(0o755) + assert before_mode.digest != capture_package(tmp_path).digest + + +def test_snapshot_detached_from_live_files(tmp_path): + main = tmp_path / "SKILL.md" + main.write_text(BASE, encoding="utf-8") + captured = capture_package(tmp_path) + main.write_text("changed", encoding="utf-8") + assert captured.main_content == BASE + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX publication contract") +def test_capture_refuses_symlinked_ancestor(tmp_path): + real = tmp_path / "real" / "example" + real.mkdir(parents=True) + (real / "SKILL.md").write_text(BASE, encoding="utf-8") + (tmp_path / "alias").symlink_to(tmp_path / "real", target_is_directory=True) + with pytest.raises(ValueError, match="UNSUPPORTED_ASSET"): + capture_package(tmp_path / "alias" / "example") + + +@pytest.mark.parametrize("kind", ["root-link", "main-link", "support-link", "directory-link", "fifo"]) +@pytest.mark.skipif(os.name != "posix", reason="POSIX publication contract") +def test_automatic_capture_rejects_links_and_special_files(tmp_path, kind): + root = tmp_path / "skill" + root.mkdir() + (root / "SKILL.md").write_text(BASE, encoding="utf-8") + if kind == "root-link": + alias = tmp_path / "alias" + alias.symlink_to(root, target_is_directory=True) + root = alias + elif kind == "main-link": + (root / "SKILL.md").unlink() + (root / "SKILL.md").symlink_to(tmp_path / "missing") + elif kind == "support-link": + (root / "support").symlink_to(tmp_path / "missing") + elif kind == "directory-link": + (root / "refs").symlink_to(tmp_path, target_is_directory=True) + else: + os.mkfifo(root / "pipe") + with pytest.raises(ValueError, match="UNSUPPORTED_ASSET"): + capture_package(root) + + +def test_snapshot_never_returns_a_partially_captured_package(tmp_path): + (tmp_path / "SKILL.md").write_text(BASE, encoding="utf-8") + for index in range(256): + (tmp_path / f"support-{index}").touch() + with pytest.raises(ValueError, match="QUOTA_EXCEEDED"): + capture_package(tmp_path) diff --git a/backend/tests/test_skill_mutation_workers.py b/backend/tests/test_skill_mutation_workers.py new file mode 100644 index 0000000000..73dbc5bf51 --- /dev/null +++ b/backend/tests/test_skill_mutation_workers.py @@ -0,0 +1,83 @@ +import asyncio +import threading + +import pytest +from deerflow_extension_api.host_capabilities import HostCapabilityError + +from deerflow.skills.mutations.workers import MutationWorkers + + +@pytest.mark.asyncio +async def test_publication_work_does_not_use_the_shared_default_executor(monkeypatch): + workers = MutationWorkers(limit=1) + loop = asyncio.get_running_loop() + original = loop.run_in_executor + + def require_dedicated(executor, func, *args): + assert executor is not None, "publication must not queue behind unrelated default-executor work" + return original(executor, func, *args) + + monkeypatch.setattr(loop, "run_in_executor", require_dedicated) + try: + assert await workers.run(lambda _: 42) == 42 + finally: + await workers.close() + + +@pytest.mark.asyncio +async def test_cancelled_caller_does_not_release_running_worker_slot(): + pool = MutationWorkers(limit=1) + entered, release = threading.Event(), threading.Event() + + def work(check): + entered.set() + release.wait(2) + return 1 + + caller = asyncio.create_task(pool.run(work)) + await asyncio.to_thread(entered.wait, 2) + caller.cancel() + with pytest.raises(asyncio.CancelledError): + await caller + with pytest.raises(HostCapabilityError, match="BUSY"): + await pool.run(lambda check: 2) + release.set() + await pool.close() + + +@pytest.mark.asyncio +async def test_waiting_worker_gets_cancellation_checkpoint(): + from deerflow.skills.mutations.workers import MutationWorkers + + pool = MutationWorkers(limit=1) + entered, release, cancelled = threading.Event(), threading.Event(), threading.Event() + + def work(check): + entered.set() + release.wait(2) + try: + check() + except HostCapabilityError as exc: + if exc.code == "CANCELLED": + cancelled.set() + raise + + caller = asyncio.create_task(pool.run(work)) + await asyncio.to_thread(entered.wait, 2) + caller.cancel() + with pytest.raises(asyncio.CancelledError): + await caller + release.set() + await pool.close() + assert cancelled.is_set() + + +@pytest.mark.asyncio +async def test_pool_closes_to_new_work(): + from deerflow.skills.mutations.workers import MutationWorkers + + pool = MutationWorkers() + assert await pool.run(lambda check: 42) == 42 + await pool.close() + with pytest.raises(HostCapabilityError, match="UNAVAILABLE"): + await pool.run(lambda check: 1) diff --git a/backend/tests/test_skill_mutations.py b/backend/tests/test_skill_mutations.py new file mode 100644 index 0000000000..cf9a540672 --- /dev/null +++ b/backend/tests/test_skill_mutations.py @@ -0,0 +1,734 @@ +"""Durable publication through the source-bound host contract.""" + +import asyncio +import importlib.util +from dataclasses import asdict, dataclass +from types import SimpleNamespace + +import pytest +import pytest_asyncio +from deerflow_extension_api.completed_run_evidence import CompletedRunSnapshot +from deerflow_extension_api.host_capabilities import HostCapabilityError +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +from deerflow.config.paths import Paths +from deerflow.extensions.completed_run_evidence import HostCompletedRunEvidenceReader +from deerflow.extensions.host_access import BoundHostAccess, HostAccess +from deerflow.persistence.base import Base +from deerflow.persistence.run.model import CompletedRunSnapshotRow, RunRow +from deerflow.persistence.skill_mutations.model import SkillAssetRow, SkillOperationRow, SkillProposalRow +from deerflow.persistence.user.model import UserRow +from deerflow.skills.mutations.guard import SkillMutationRuntime, configure_mutation_runtime +from deerflow.skills.mutations.repository import SkillMutationRepository +from deerflow.skills.mutations.scanner import ScanVerdict +from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage + +CONTENT = "---\nname: example\ndescription: Before\n---\nA\n" +NEW = CONTENT.replace("\nA\n", "\nB\n") + + +@pytest_asyncio.fixture +async def host(tmp_path, monkeypatch): + assert importlib.util.find_spec("deerflow.skills.mutations.service") is not None, "bound mutation service is missing" + from deerflow.skills.mutations.recovery import SkillMutationRecovery + from deerflow.skills.mutations.service import HostSkillMutationService + + monkeypatch.setattr("deerflow.config.paths._paths", Paths(base_dir=tmp_path / "home")) + storage = UserScopedSkillStorage("owner", host_path=str(tmp_path / "skills")) + root = storage.get_custom_skill_dir("example") + root.mkdir(parents=True) + (root / "SKILL.md").write_text(CONTENT, encoding="utf-8") + (root / "help.bin").write_bytes(b"\x00\xff") + engine = create_engine(f"sqlite:///{tmp_path / 'mutations.db'}", connect_args={"check_same_thread": False}) + Base.metadata.create_all(engine) + sessions = sessionmaker(engine, expire_on_commit=False) + with sessions.begin() as session: + session.add(UserRow(id="owner", email="owner@example.test")) + run = RunRow(run_id="run", thread_id="thread", user_id="owner", status="success", evidence_seal_state="sealed", evidence_agent_id="trainer", evidence_origin="interactive", evidence_retention_revision=0) + session.add(run) + session.flush() + snap = CompletedRunSnapshot( + snapshot_ref="source", run_id="run", thread_id="thread", owner_id="owner", agent_id="trainer", origin="interactive", status="success", seal_state="sealed", evidence_revision=HostCompletedRunEvidenceReader.revision_for_run(run) + ) + session.add(CompletedRunSnapshotRow(snapshot_ref="source", run_id="run", scope_digest="scope", evidence_revision=snap.evidence_revision, retention_revision=0, snapshot_json=asdict(snap))) + + class Evidence: + scope_digest = "scope" + + @staticmethod + def revision_for_run(run): + return HostCompletedRunEvidenceReader.revision_for_run(run) + + async def resolve_snapshot(self, ref): + if ref != "source": + raise HostCapabilityError("NOT_FOUND") + return snap + + class Scanner: + calls = 0 + policy = "policy-1" + decision = "allow" + + def policy_version_sync(self): + return self.policy + + async def policy_version(self): + return self.policy + + async def scan(self, package, name): + self.calls += 1 + await asyncio.sleep(0) + return ScanVerdict(self.decision, "CHECK_PASSED", self.policy) + + binding = BoundHostAccess( + "plugin", + HostAccess.model_validate( + {"evidence": {"owners": ["owner"]}, "skill_mutations": {"owners": ["owner"], "operations": ["stage", "check", "commit", "revert"], "trigger_agents": ["trainer"], "target_skills": ["example"], "topology": "single_host_local"}} + ), + ) + runtime = SkillMutationRuntime(SkillMutationRepository(sessions), owners=frozenset({"owner"})) + configure_mutation_runtime(runtime) + + def factory(owner): + return storage + + recovery = SkillMutationRecovery(runtime, factory, rebuild_views=lambda _: None) + scanner = Scanner() + service = HostSkillMutationService(binding, runtime=runtime, evidence=Evidence(), storage_factory=factory, scanner=scanner, recovery=recovery) + yield SimpleNamespace(service=service, recovery=recovery, scanner=scanner, runtime=runtime, storage=storage, sessions=sessions, binding=binding, snapshot=snap) + await service.close() + configure_mutation_runtime(None) + engine.dispose() + + +async def stage(host, *, key="stage-1", content=NEW): + skill = await host.service.read_skill(source_ref="source", name="example") + return await host.service.stage(source_refs=("source",), target_ref=skill.target_ref, expected_revision=skill.revision, content=content, idempotency_key=key) + + +async def publish(host): + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + operation = await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="commit-1") + return proposal, operation + + +@pytest.mark.asyncio +async def test_bundle_is_exact_detached_and_stage_is_idempotent(host): + proposal = await stage(host) + repeated = await stage(host) + assert repeated == proposal + bundle = await host.service.read_proposal_bundle(proposal_id=proposal.proposal_id) + assert {f.path: f.content for f in bundle.baseline} == {"SKILL.md": CONTENT.encode(), "help.bin": b"\x00\xff"} + assert {f.path: f.content for f in bundle.candidate} == {"SKILL.md": NEW.encode(), "help.bin": b"\x00\xff"} + assert host.storage.get_custom_skill_file("example").read_text() == CONTENT + + +@pytest.mark.asyncio +async def test_commit_is_durable_idempotent_and_revert_moves_forward(host): + proposal, operation = await publish(host) + assert operation.publication == "APPLIED" + assert operation.views == "READY" + assert host.storage.get_custom_skill_file("example").read_text() == NEW + assert await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="commit-1") == operation + with pytest.raises(HostCapabilityError, match="PROPOSAL_ALREADY_COMMITTED"): + await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="other-key") + assert await host.service.find_operation(idempotency_key="other-key") is None + reverted = await host.service.revert(operation_id=operation.operation_id, expected_current_revision=operation.after_revision, idempotency_key="revert-1") + assert reverted.publication == "APPLIED" + assert reverted.after_revision.mutation_seq == operation.after_revision.mutation_seq + 1 + assert host.storage.get_custom_skill_file("example").read_text() == CONTENT + + +@pytest.mark.asyncio +async def test_source_deletion_prevents_commit_but_not_query_or_revert(host): + proposal, operation = await publish(host) + with host.sessions.begin() as session: + session.delete(session.get(RunRow, "run")) + assert (await host.service.get_proposal(proposal_id=proposal.proposal_id)).operation_id == operation.operation_id + assert await host.service.get_operation(operation_id=operation.operation_id) == operation + reverted = await host.service.revert(operation_id=operation.operation_id, expected_current_revision=operation.after_revision, idempotency_key="revert-1") + assert reverted.publication == "APPLIED" + + +@pytest.mark.asyncio +async def test_fresh_source_fence_and_stale_policy_fail_closed(host): + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + host.scanner.policy = "policy-2" + with pytest.raises(HostCapabilityError, match="CHECK_REQUIRED"): + await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="commit-1") + await host.service.check(proposal_id=proposal.proposal_id) + with host.sessions.begin() as session: + session.get(RunRow, "run").evidence_retention_revision += 1 + with pytest.raises(HostCapabilityError, match="SOURCE_STALE_OR_GONE"): + await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="commit-1") + assert host.storage.get_custom_skill_file("example").read_text() == CONTENT + + +@pytest.mark.asyncio +async def test_commit_maps_malformed_assessment_to_contract_error(host): + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + + @dataclass(frozen=True) + class DriftedAssessment: + evaluator: str + evaluator_version: str + + for index, assessment in enumerate(({"evaluator": "plugin"}, DriftedAssessment("plugin", "1"))): + with pytest.raises(HostCapabilityError, match="INVALID_ASSESSMENT"): + await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key=f"bad-assessment-{index}", assessment_ref=assessment) + + +@pytest.mark.asyncio +async def test_commit_bounds_serialized_assessment_bytes(host): + from deerflow_extension_api.skill_mutations import AssessmentRef + + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + assessment = AssessmentRef(proposal.candidate_hash, proposal.base_revision, "é" * 300, "é" * 300, "é" * 300) + with pytest.raises(HostCapabilityError, match="INVALID_ASSESSMENT"): + await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="oversized-assessment", assessment_ref=assessment) + + +@pytest.mark.asyncio +async def test_commit_persists_valid_assessment(host): + from deerflow_extension_api.skill_mutations import AssessmentRef + + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + assessment = AssessmentRef(proposal.candidate_hash, proposal.base_revision, "skill-evolution", "1", "report-1") + operation = await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="assessed", assessment_ref=assessment) + with host.sessions() as session: + assert session.get(SkillOperationRow, operation.operation_id).assessment == asdict(assessment) + + +@pytest.mark.asyncio +async def test_no_change_has_persisted_result_without_version_increment(host): + proposal = await stage(host, content=CONTENT) + await host.service.check(proposal_id=proposal.proposal_id) + op = await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="same") + assert op.publication == "NO_CHANGE" + assert op.before_revision == op.after_revision == proposal.base_revision + assert (await host.service.find_operation(idempotency_key="same")).operation_id == op.operation_id + + +@pytest.mark.asyncio +async def test_revert_refuses_even_caller_supplied_new_head(host): + _, operation = await publish(host) + host.storage.get_custom_skill_file("example").write_text(NEW + "manual", encoding="utf-8") + current = host.runtime.read_revision(host.storage, "example") + with pytest.raises(HostCapabilityError, match="REVISION_CONFLICT"): + await host.service.revert(operation_id=operation.operation_id, expected_current_revision=current, idempotency_key="revert-1") + + +@pytest.mark.asyncio +async def test_check_reuses_valid_result_and_rejection_cannot_publish(host): + proposal = await stage(host) + host.scanner.decision = "reject" + result = await host.service.check(proposal_id=proposal.proposal_id) + assert await host.service.check(proposal_id=proposal.proposal_id) == result + assert host.scanner.calls == 1 + with pytest.raises(HostCapabilityError, match="CHECK_REQUIRED"): + await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="commit-1") + + +@pytest.mark.asyncio +async def test_discard_and_expiry_do_not_destroy_idempotence(host): + proposal = await stage(host) + assert (await host.service.discard(proposal_id=proposal.proposal_id)).state == "DISCARDED" + host.recovery.collect_garbage() + assert (await stage(host)).proposal_id == proposal.proposal_id + with pytest.raises(HostCapabilityError, match="PROPOSAL_UNAVAILABLE"): + await host.service.read_proposal_bundle(proposal_id=proposal.proposal_id) + + +@pytest.mark.asyncio +async def test_view_failure_keeps_applied_publication(host): + def fail(_): + raise RuntimeError("view failure") + + host.recovery.rebuild_views = fail + _, operation = await publish(host) + assert (operation.publication, operation.views) == ("APPLIED", "ERROR") + assert host.storage.get_custom_skill_file("example").read_text() == NEW + + +@pytest.mark.asyncio +async def test_idempotency_key_cannot_change_request(host): + await stage(host) + with pytest.raises(HostCapabilityError, match="IDEMPOTENCY_CONFLICT"): + await stage(host, content=NEW + "different") + + +@pytest.mark.asyncio +async def test_pending_quota_is_durable(host): + for number in range(20): + await stage(host, key=f"stage-{number}") + with pytest.raises(HostCapabilityError, match="QUOTA_EXCEEDED"): + await stage(host, key="overflow") + with host.sessions() as session: + assert len(session.scalars(select(SkillProposalRow)).all()) == 20 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("disk,expected", [(CONTENT, "ABORTED"), (NEW, "APPLIED"), (NEW + "external", "NEEDS_REPAIR")]) +async def test_recovery_classifies_disk_without_replaying(host, monkeypatch, disk, expected): + def interrupted(*_): + raise RuntimeError("simulated process exit after PREPARED") + + monkeypatch.setattr("deerflow.skills.mutations.service.publish_prepared", interrupted) + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + response = await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="crash") + assert response.publication == "PREPARED" + operation = await host.service.find_operation(idempotency_key="crash") + assert operation.publication == "PREPARED" + host.storage.get_custom_skill_file("example").write_text(disk, encoding="utf-8") + if expected == "NEEDS_REPAIR": + with pytest.raises(HostCapabilityError, match="NEEDS_REPAIR"): + host.recovery.recover_operation(operation.operation_id) + recovered = host.recovery.get_operation(operation.operation_id) + else: + recovered = host.recovery.recover_operation(operation.operation_id) + assert host.recovery.recover_operation(operation.operation_id) == recovered + assert recovered.publication == expected + assert host.storage.get_custom_skill_file("example").read_text() == disk + if expected == "NEEDS_REPAIR": + with pytest.raises(HostCapabilityError, match="NEEDS_REPAIR"): + host.recovery.recover_owner("owner") + host.recovery.recover_all() # A broken owner must not take down startup. + + +@pytest.mark.asyncio +async def test_gc_keeps_live_recovery_blobs_and_terminal_metadata(host, monkeypatch): + def interrupted(*_): + raise RuntimeError("simulated exit") + + monkeypatch.setattr("deerflow.skills.mutations.service.publish_prepared", interrupted) + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + response = await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="crash") + assert response.publication == "PREPARED" + operation = await host.service.find_operation(idempotency_key="crash") + with host.sessions.begin() as session: + session.get(SkillProposalRow, proposal.proposal_id).expires_at = 0 + session.get(SkillOperationRow, operation.operation_id).rollback_expires_at = 0 + host.recovery.collect_garbage() + with host.sessions() as session: + assert session.get(SkillOperationRow, operation.operation_id).before_blob + assert session.get(SkillProposalRow, proposal.proposal_id).candidate_blob + host.recovery.recover_operation(operation.operation_id) + host.recovery.collect_garbage() + with host.sessions() as session: + row = session.get(SkillOperationRow, operation.operation_id) + assert row.before_blob is row.after_blob is None + assert (await host.service.find_operation(idempotency_key="crash")).publication == "ABORTED" + + +@pytest.mark.asyncio +async def test_concurrent_check_is_singleflight(host): + proposal = await stage(host) + first, second = await asyncio.gather(host.service.check(proposal_id=proposal.proposal_id), host.service.check(proposal_id=proposal.proposal_id)) + assert first == second + assert host.scanner.calls == 1 + + +@pytest.mark.asyncio +async def test_owner_quiesce_refuses_new_mutations_and_queries(host): + proposal = await stage(host) + host.recovery.quiesce_owner("owner") + with pytest.raises(HostCapabilityError, match="NOT_FOUND_OR_FORBIDDEN"): + await host.service.get_proposal(proposal_id=proposal.proposal_id) + with pytest.raises(HostCapabilityError, match="NOT_FOUND_OR_FORBIDDEN"): + await host.service.check(proposal_id=proposal.proposal_id) + + +@pytest.mark.asyncio +async def test_source_identity_change_invalidates_snapshot_even_when_new_agent_is_granted(host): + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + with host.sessions.begin() as session: + session.get(RunRow, "run").evidence_origin = "scheduled" + with pytest.raises(HostCapabilityError, match="SOURCE_STALE_OR_GONE"): + await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="changed-identity") + + +@pytest.mark.asyncio +async def test_revert_scan_cost_has_persisted_quota(host): + _, operation = await publish(host) + host.scanner.decision = "reject" + for index in range(19): + with pytest.raises(HostCapabilityError, match="CHECK_REQUIRED"): + await host.service.revert(operation_id=operation.operation_id, expected_current_revision=operation.after_revision, idempotency_key=f"reject-{index}") + with pytest.raises(HostCapabilityError, match="QUOTA_EXCEEDED"): + await host.service.revert(operation_id=operation.operation_id, expected_current_revision=operation.after_revision, idempotency_key="overflow") + + +@pytest.mark.asyncio +async def test_recovery_does_not_claim_applied_when_durability_barrier_fails(host, monkeypatch): + def interrupted(*_): + raise RuntimeError("simulated exit") + + monkeypatch.setattr("deerflow.skills.mutations.service.publish_prepared", interrupted) + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + response = await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="crash") + assert response.publication == "PREPARED" + operation = await host.service.find_operation(idempotency_key="crash") + host.storage.get_custom_skill_file("example").write_text(NEW, encoding="utf-8") + + def failed_fsync(_): + raise OSError("disk error") + + monkeypatch.setattr("deerflow.skills.mutations.publication.os.fsync", failed_fsync) + with pytest.raises(HostCapabilityError, match="NEEDS_REPAIR"): + host.recovery.recover_operation(operation.operation_id) + assert host.recovery.get_operation(operation.operation_id).publication == "PREPARED" + + +def test_replace_collision_does_not_remove_existing_file(host): + from deerflow.skills.mutations.publication import replace_main + + root = host.storage.get_custom_skill_dir("example") + temporary = root.parent / ".host-mutation-collision" + temporary.write_text("existing", encoding="utf-8") + with pytest.raises(FileExistsError): + replace_main(root, NEW.encode(), operation_id="collision") + assert temporary.read_text() == "existing" + + +@pytest.mark.asyncio +async def test_publication_preserves_permissions(host): + import stat + + main = host.storage.get_custom_skill_file("example") + main.chmod(0o640) + await publish(host) + assert stat.S_IMODE(main.stat().st_mode) == 0o640 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reader", ["load_skills", "load_custom_skill_diagnostics"]) +async def test_managed_reader_lazily_recovers_accepted_publication(host, monkeypatch, reader): + def interrupted(*_): + raise RuntimeError("simulated exit") + + monkeypatch.setattr("deerflow.skills.mutations.service.publish_prepared", interrupted) + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + operation = await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="crash") + host.storage.get_custom_skill_file("example").write_text(NEW, encoding="utf-8") + result = getattr(host.storage, reader)() + if reader == "load_skills": + assert result + else: + assert result == [] + assert host.recovery.get_operation(operation.operation_id).publication == "APPLIED" + + +@pytest.mark.asyncio +async def test_failed_legacy_write_after_publication_is_recoverable(host, monkeypatch): + _, operation = await publish(host) + + def no_space(*_args, **_kwargs): + raise OSError("no space left on device") + + monkeypatch.setattr("deerflow.skills.storage.user_scoped_skill_storage.tempfile.NamedTemporaryFile", no_space) + with pytest.raises(OSError, match="no space"): + host.storage.write_custom_skill("example", "SKILL.md", NEW + "manual") + + with host.sessions() as session: + asset = session.scalar(select(SkillAssetRow).where(SkillAssetRow.owner_id == "owner", SkillAssetRow.name == "example")) + assert asset.mutating is True + assert asset.operation_id == operation.operation_id + + assert host.storage.load_skills() + with host.sessions() as session: + asset = session.scalar(select(SkillAssetRow).where(SkillAssetRow.owner_id == "owner", SkillAssetRow.name == "example")) + assert asset.mutating is False + assert asset.operation_id is None + assert asset.mutation_seq == operation.after_revision.mutation_seq + 1 + + +@pytest.mark.asyncio +async def test_history_mirror_is_compact_and_deduplicated_by_operation(host): + _, operation = await publish(host) + host.recovery.recover_all() + host.recovery.recover_all() + mirrored = [entry for entry in host.storage.read_history("example") if entry.get("operation_id") == operation.operation_id] + assert len(mirrored) == 1 + assert mirrored[0]["scope"] == "main-file-only" + assert "prev_content" not in mirrored[0] and "new_content" not in mirrored[0] + + +@pytest.mark.asyncio +async def test_history_mirror_failure_does_not_change_publication(host, monkeypatch): + import deerflow.skills.mutations.history as history + + def fail(*_): + raise OSError("mirror unavailable") + + monkeypatch.setattr(history, "mirror_operations", fail) + _, operation = await publish(host) + assert (operation.publication, operation.views) == ("APPLIED", "READY") + + +@pytest.mark.asyncio +async def test_admin_operation_pagination_uses_creation_order(host): + _, committed = await publish(host) + reverted = await host.service.revert(operation_id=committed.operation_id, expected_current_revision=committed.after_revision, idempotency_key="revert-order") + newer_id, older_id = sorted((committed.operation_id, reverted.operation_id)) + with host.sessions.begin() as session: + older = session.get(SkillOperationRow, older_id) + newer = session.get(SkillOperationRow, newer_id) + older.created_at, newer.created_at = 1.0, 2.0 + older.views = newer.views = "ERROR" + + first = host.recovery.list_operations(limit=1) + assert [item.operation_id for item in first] == [older_id] + second = host.recovery.list_operations(limit=2, after_id=older_id) + assert [item.operation_id for item in second] == [newer_id] + + +@pytest.mark.asyncio +async def test_bound_identity_and_current_grants_are_required_for_queries(host): + proposal, operation = await publish(host) + original = host.service.binding + host.service.binding = BoundHostAccess("different-plugin", original.access) + for call in (host.service.get_proposal(proposal_id=proposal.proposal_id), host.service.get_operation(operation_id=operation.operation_id)): + with pytest.raises(HostCapabilityError, match="NOT_FOUND_OR_FORBIDDEN"): + await call + host.service.binding = BoundHostAccess(original.plugin_id, HostAccess()) + with pytest.raises(HostCapabilityError, match="NOT_FOUND_OR_FORBIDDEN"): + await host.service.get_operation(operation_id=operation.operation_id) + + +@pytest.mark.asyncio +async def test_close_drains_revert_scan_and_its_publication(host): + _, operation = await publish(host) + entered, release = asyncio.Event(), asyncio.Event() + scan = host.scanner.scan + + async def blocked(*args): + entered.set() + await release.wait() + return await scan(*args) + + host.scanner.scan = blocked + reverting = asyncio.create_task(host.service.revert(operation_id=operation.operation_id, expected_current_revision=operation.after_revision, idempotency_key="close-revert")) + await asyncio.wait_for(entered.wait(), 2) + closing = asyncio.create_task(host.service.close()) + await asyncio.sleep(0) + assert not closing.done() + with pytest.raises(HostCapabilityError, match="UNAVAILABLE"): + await host.service.revert(operation_id=operation.operation_id, expected_current_revision=operation.after_revision, idempotency_key="new") + release.set() + await closing + assert (await reverting).publication == "APPLIED" + + +@pytest.mark.asyncio +async def test_gc_does_not_load_retained_history_blobs(host): + from sqlalchemy import event + + await publish(host) + statements = [] + engine = host.sessions.kw["bind"] + + def observed(_conn, _cursor, statement, *_): + statements.append(statement.lower()) + + event.listen(engine, "before_cursor_execute", observed) + try: + host.recovery.collect_garbage() + finally: + event.remove(engine, "before_cursor_execute", observed) + assert not any(statement.startswith("select") and "_blob" in statement for statement in statements) + + +def test_capabilities_describe_persisted_scan_budgets(host): + capabilities = host.service.capabilities + assert capabilities.max_concurrent_scans == 4 + assert capabilities.max_owner_plugin_concurrent_scans == 2 + assert capabilities.max_owner_plugin_scans_per_hour == 20 + + +@pytest.mark.asyncio +async def test_nonbaseline_source_coverage_cannot_authorize_stage(host): + from dataclasses import replace + + source = replace(host.snapshot, coverage="unknown-journal-v2") + + async def resolve(_): + return source + + host.service.evidence.resolve_snapshot = resolve + with host.sessions.begin() as session: + session.get(CompletedRunSnapshotRow, "source").snapshot_json = asdict(source) + with pytest.raises(HostCapabilityError, match="SOURCE_STALE_OR_GONE"): + await stage(host) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field,blob", [("before_blob", None), ("before_blob", b"[]"), ("after_blob", None), ("after_blob", b"invalid")]) +async def test_recovery_quarantines_missing_or_corrupt_blobs(host, monkeypatch, field, blob): + def interrupted(*_): + raise RuntimeError("simulated exit") + + monkeypatch.setattr("deerflow.skills.mutations.service.publish_prepared", interrupted) + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + operation = await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="crash") + with host.sessions.begin() as session: + setattr(session.get(SkillOperationRow, operation.operation_id), field, blob) + host.storage.get_custom_skill_file("example").write_text(NEW, encoding="utf-8") + with pytest.raises(HostCapabilityError, match="NEEDS_REPAIR"): + host.recovery.recover_operation(operation.operation_id) + recovered = host.recovery.get_operation(operation.operation_id) + assert recovered.publication == "NEEDS_REPAIR" + assert recovered.error_code == "BLOB_INTEGRITY_ERROR" + assert host.storage.get_custom_skill_file("example").read_text() == NEW + + +@pytest.mark.asyncio +async def test_revert_rejects_blob_with_wrong_digest(host): + from deerflow.skills.mutations.codec import decode_package, encode_package + + _, operation = await publish(host) + with host.sessions.begin() as session: + row = session.get(SkillOperationRow, operation.operation_id) + row.before_blob = encode_package(decode_package(row.before_blob).with_main(CONTENT + "tampered")) + with pytest.raises(HostCapabilityError, match="ROLLBACK_UNAVAILABLE"): + await host.service.revert(operation_id=operation.operation_id, expected_current_revision=operation.after_revision, idempotency_key="bad-blob") + + +@pytest.mark.asyncio +async def test_aborted_later_attempt_preserves_previous_revert_head(host, monkeypatch): + _, first = await publish(host) + proposal = await stage(host, key="stage-later", content=NEW + "later") + await host.service.check(proposal_id=proposal.proposal_id) + import deerflow.skills.mutations.service as service_module + + publish_prepared = service_module.publish_prepared + + def interrupted(*_): + raise RuntimeError("simulated exit") + + monkeypatch.setattr(service_module, "publish_prepared", interrupted) + later = await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="commit-later") + assert host.recovery.recover_operation(later.operation_id).publication == "ABORTED" + monkeypatch.setattr(service_module, "publish_prepared", publish_prepared) + reverted = await host.service.revert(operation_id=first.operation_id, expected_current_revision=first.after_revision, idempotency_key="revert-earlier") + assert reverted.publication == "APPLIED" + assert host.storage.get_custom_skill_file("example").read_text() == CONTENT + + +@pytest.mark.asyncio +async def test_superseded_view_points_to_replacing_generation(host): + _, original = await publish(host) + restored = await host.service.revert(operation_id=original.operation_id, expected_current_revision=original.after_revision, idempotency_key="revert") + old = await host.service.get_operation(operation_id=original.operation_id) + assert old.views == "SUPERSEDED" + assert old.superseded_by_generation == restored.generation + + +@pytest.mark.asyncio +async def test_legacy_writer_immediately_supersedes_old_operation_view(host): + _, operation = await publish(host) + host.storage.write_custom_skill("example", "SKILL.md", NEW + "manual") + old = await host.service.get_operation(operation_id=operation.operation_id) + assert old.views == "SUPERSEDED" + assert old.superseded_by_generation == host.runtime.repository.generation("owner") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [asyncio.CancelledError, TimeoutError]) +async def test_interrupted_check_clears_lease_without_caching_unavailable(host, failure): + from deerflow.persistence.skill_mutations.model import SkillScanAttemptRow + + proposal = await stage(host) + scan = host.scanner.scan + + async def interrupted(*args): + raise failure() + + host.scanner.scan = interrupted + with pytest.raises(failure): + await host.service.check(proposal_id=proposal.proposal_id) + with host.sessions() as session: + assert session.get(SkillProposalRow, proposal.proposal_id).check_result is None + attempts = session.scalars(select(SkillScanAttemptRow)).all() + assert len(attempts) == 1 # Interrupted scans still consume cost quota. + assert attempts[0].lease_until == 0 + host.scanner.scan = scan + assert (await host.service.check(proposal_id=proposal.proposal_id)).decision == "allow" + assert (await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="after-interruption")).publication == "APPLIED" + + +@pytest.mark.asyncio +async def test_cancelled_check_waiter_preserves_shared_scan(host): + proposal = await stage(host) + entered, release = asyncio.Event(), asyncio.Event() + scan = host.scanner.scan + + async def blocked(*args): + entered.set() + await release.wait() + return await scan(*args) + + host.scanner.scan = blocked + first = asyncio.create_task(host.service.check(proposal_id=proposal.proposal_id)) + second = None + try: + await asyncio.wait_for(entered.wait(), 2) + second = asyncio.create_task(host.service.check(proposal_id=proposal.proposal_id)) + first.cancel() + with pytest.raises(asyncio.CancelledError): + await first + release.set() + assert (await second).decision == "allow" + assert host.scanner.calls == 1 + finally: + release.set() + await asyncio.gather(first, *([second] if second else []), return_exceptions=True) + + +@pytest.mark.asyncio +async def test_admin_recovery_reports_unresolved_barrier_and_allows_retry(host, monkeypatch): + import httpx + from fastapi import FastAPI, Request + + from app.gateway.routers.skill_mutations import router + + def interrupted(*_): + raise RuntimeError("simulated process exit after PREPARED") + + monkeypatch.setattr("deerflow.skills.mutations.service.publish_prepared", interrupted) + proposal = await stage(host) + await host.service.check(proposal_id=proposal.proposal_id) + operation = await host.service.commit(proposal_id=proposal.proposal_id, idempotency_key="recover-retry") + path = host.storage.get_custom_skill_file("example") + path.write_text(NEW + "external", encoding="utf-8") + app = FastAPI() + app.state.skill_mutation_host = SimpleNamespace(workers=host.service.workers, recovery=host.recovery) + + @app.middleware("http") + async def identity(request: Request, call_next): + request.state.user = SimpleNamespace(id="admin", system_role="admin") + return await call_next(request) + + app.include_router(router) + url = f"/api/skill-mutations/operations/{operation.operation_id}" + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client: + response = await client.post(url + "/recover") + assert response.status_code == 409 + assert response.json()["detail"] == "NEEDS_REPAIR" + assert (await client.get(url)).json()["publication"] == "NEEDS_REPAIR" + path.write_text(NEW, encoding="utf-8") + response = await client.post(url + "/recover") + assert response.status_code == 200 + assert response.json()["publication"] == "APPLIED" + assert response.json()["views"] == "READY" diff --git a/backend/tests/test_skill_sdk_enrollment.py b/backend/tests/test_skill_sdk_enrollment.py new file mode 100644 index 0000000000..65392bad31 --- /dev/null +++ b/backend/tests/test_skill_sdk_enrollment.py @@ -0,0 +1,151 @@ +"""Standalone SDK processes must not bypass another host's durable enrollment.""" + +import json +import os +import subprocess +import sys + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from deerflow.config.paths import Paths +from deerflow.persistence.base import Base +from deerflow.persistence.user.model import UserRow +from deerflow.skills.mutations.assets import capture_package +from deerflow.skills.mutations.repository import SkillMutationRepository + +CONTENT = "---\nname: example\ndescription: SDK enrollment regression\n---\nOriginal body.\n" + + +@pytest.fixture +def sdk_home(tmp_path): + home = tmp_path / "home" + root = Paths(base_dir=home).user_custom_skills_dir("owner") / "example" + root.mkdir(parents=True) + (root / "SKILL.md").write_text(CONTENT, encoding="utf-8") + public = tmp_path / "skills" / "public" / "example" + public.mkdir(parents=True) + (public / "SKILL.md").write_text(CONTENT, encoding="utf-8") + config = tmp_path / "config.yaml" + config.write_text( + json.dumps( + { + "models": [{"name": "test", "use": "langchain_openai:ChatOpenAI", "model": "test", "api_key": "test"}], + "sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}, + "skills": {"path": str(tmp_path / "skills")}, + "database": {"backend": "sqlite", "sqlite_dir": str(tmp_path)}, + "plugins": [], + } + ), + encoding="utf-8", + ) + extensions = tmp_path / "extensions.json" + extensions.write_text('{"skills": {}}', encoding="utf-8") + engine = create_engine(f"sqlite:///{tmp_path / 'deerflow.db'}") + Base.metadata.create_all(engine) + sessions = sessionmaker(engine, expire_on_commit=False) + with sessions.begin() as session: + session.add(UserRow(id="owner", email="owner@example.test")) + repository = SkillMutationRepository(sessions) + yield tmp_path, config, root, repository + engine.dispose() + + +def run_sdk(fixture, action, *, owner="owner"): + tmp_path, config, _, _ = fixture + source = """ +import json, sys +from types import SimpleNamespace +from deerflow.client import DeerFlowClient +from deerflow.runtime.user_context import set_current_user +from deerflow.skills.mutations import guard +from deerflow.skills.storage import get_or_new_user_skill_storage +from deerflow_extension_api import HostCapabilityError +assert guard._runtime is None +set_current_user(SimpleNamespace(id=sys.argv[2])) +client = DeerFlowClient(config_path=sys.argv[1]) +storage = get_or_new_user_skill_storage(sys.argv[2], app_config=client._app_config) +try: + exec(sys.argv[3]) +except HostCapabilityError as exc: + print(json.dumps({"error": exc.code})) +else: + print(json.dumps({"error": None})) +""" + result = subprocess.run( + [sys.executable, "-c", source, str(config), owner, action], + env={**os.environ, "DEER_FLOW_CONFIG_PATH": str(config), "DEER_FLOW_HOME": str(tmp_path / "home"), "DEER_FLOW_EXTENSIONS_CONFIG_PATH": str(tmp_path / "extensions.json")}, + text=True, + capture_output=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.splitlines()[-1]) + + +@pytest.mark.parametrize( + "action", + [ + "client.update_skill('example', enabled=False); client.update_skill('example', enabled=True)", + "storage.set_skill_enabled_state('example', False)", + "storage.write_custom_skill('example', 'SKILL.md', 'changed')", + "storage.delete_custom_skill('example')", + "client.get_skill('example')", + "storage.load_custom_skill_diagnostics()", + ], +) +def test_fresh_sdk_refuses_enrolled_owner_without_gateway_runtime(sdk_home, action): + _, _, root, repository = sdk_home + before = repository.observe("owner", "example", capture_package(root).digest, True, exists=True) + assert run_sdk(sdk_home, action) == {"error": "MUTATION_RUNTIME_REQUIRED"} + assert (root / "SKILL.md").read_text(encoding="utf-8") == CONTENT + assert not (root.parent.parent / "_skill_states.json").exists() + after = repository.observe("owner", "example", capture_package(root).digest, True, exists=True) + assert after.mutation_seq == before.mutation_seq + + +def test_fresh_sdk_global_toggle_cannot_bypass_another_enrolled_owner(sdk_home): + _, _, root, repository = sdk_home + repository.observe("owner", "example", capture_package(root).digest, True, exists=True) + assert run_sdk(sdk_home, "client.update_skill('example', enabled=False)", owner="other") == {"error": "MUTATION_RUNTIME_REQUIRED"} + assert json.loads((sdk_home[0] / "extensions.json").read_text(encoding="utf-8")) == {"skills": {}} + + +def test_unenrolled_sdk_retains_skill_management(sdk_home): + assert run_sdk(sdk_home, "client.update_skill('example', enabled=False); client.update_skill('example', enabled=True)") == {"error": None} + + +def test_existing_sdk_rechecks_enrollment_after_an_earlier_unmanaged_read(sdk_home): + action = """ +assert client.get_skill("example") is not None +# Another host can persist enrollment after this client has started. Only the +# durable row is shared; the SDK still has no injected mutation runtime. +import sqlite3 +with sqlite3.connect(client._app_config.database.sqlite_path) as connection: + connection.execute("INSERT INTO skill_mutation_owners (owner_id, generation, deleting) VALUES ('owner', 0, 0)") +storage.set_skill_enabled_state("example", False) +""" + assert run_sdk(sdk_home, action) == {"error": "MUTATION_RUNTIME_REQUIRED"} + assert not (sdk_home[2].parent.parent / "_skill_states.json").exists() + + +def test_sdk_cannot_treat_unavailable_enrollment_as_unmanaged(sdk_home): + action = """ +from pathlib import Path +Path(client._app_config.database.sqlite_path).write_bytes(b"unreadable database") +storage.set_skill_enabled_state("example", False) +""" + assert run_sdk(sdk_home, action) == {"error": "UNAVAILABLE"} + assert not (sdk_home[2].parent.parent / "_skill_states.json").exists() + + +def test_sdk_does_not_create_a_missing_application_database(sdk_home): + action = """ +from pathlib import Path +path = Path(client._app_config.database.sqlite_path) +path.unlink() +client.update_skill("example", enabled=False) +assert not path.exists() +""" + assert run_sdk(sdk_home, action) == {"error": None} diff --git a/backend/tests/test_user_scoped_skill_storage.py b/backend/tests/test_user_scoped_skill_storage.py index eb1ca804a2..37cff7eb4e 100644 --- a/backend/tests/test_user_scoped_skill_storage.py +++ b/backend/tests/test_user_scoped_skill_storage.py @@ -858,3 +858,40 @@ class TestInstallScanConfigParity: assert seen, "the LLM scan should have run for the installed SKILL.md" assert all(entry is config for entry in seen), "every LLM scan must receive the storage's own app_config" + + +@pytest.mark.parametrize("explicit_config", [False, True]) +def test_prompt_skill_catalog_preserves_user_scope_with_optional_config(base_dir, skills_root, config, monkeypatch, explicit_config): + from collections import OrderedDict + + from deerflow.agents.lead_agent import prompt + from deerflow.skills.mutations.guard import owner_is_managed + + paths = Paths(base_dir=base_dir) + monkeypatch.setattr("deerflow.config.paths._paths", paths) + monkeypatch.setattr("deerflow.config.get_app_config", lambda: config) + monkeypatch.setattr(prompt, "_enabled_skills_by_config_cache", OrderedDict()) + for root, name in [(skills_root / "public", "shared"), (paths.user_custom_skills_dir("alice"), "alice-only"), (paths.user_custom_skills_dir("bob"), "bob-only")]: + folder = root / name + folder.mkdir(parents=True) + (folder / "SKILL.md").write_text(_skill_content(name), encoding="utf-8") + public = LocalSkillStorage(host_path=str(skills_root)).load_skills(enabled_only=True) + monkeypatch.setattr(prompt, "_enabled_skills_cache", public) + + for owner in ("alice", "bob"): + assert not owner_is_managed(owner) + skills = prompt.get_enabled_skills_for_config(app_config=config if explicit_config else None, user_id=owner) + assert {skill.name for skill in skills} == {"shared", f"{owner}-only"} + + +def test_prompt_without_config_or_user_keeps_nonblocking_global_cache(monkeypatch): + from deerflow.agents.lead_agent import prompt + + cached = [SimpleNamespace(name="cached")] + monkeypatch.setattr(prompt, "_enabled_skills_cache", cached) + + def unexpected_storage(**_kwargs): + pytest.fail("Global cached access must not synchronously load storage") + + monkeypatch.setattr(prompt, "get_or_new_skill_storage", unexpected_storage) + assert prompt.get_enabled_skills_for_config() == cached diff --git a/config.example.yaml b/config.example.yaml index 1faa21a96b..c2cbb5cc87 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -3590,6 +3590,19 @@ authorization: # package: deerflow-extension-example # Python distribution managed by uv # use: deerflow_extension_example:install # enabled: true # false skips import and registration +# # Optional, restart-required host grants. Omit for legacy behavior. +# # Requires durable database + run_events.backend: db for automatic updates. +# # Only enable on one POSIX host with local disk and ALL writers upgraded. +# # SQLite additionally requires exactly one Gateway process. +# # host_access: +# # evidence: +# # owners: ["exact-user-id"] +# # skill_mutations: +# # owners: ["exact-user-id"] # subset of evidence.owners +# # operations: [stage, check, commit, revert] +# # trigger_agents: [lead_agent] # canonical agent name, not display name +# # target_skills: [example-skill] +# # topology: single_host_local # operator assertion; excludes NFS/CSI # required: false # true makes load failure abort Gateway startup # # (install --required opts in) # host_access: # optional host-owned grants (restart required) diff --git a/docs/skill-evolution-host.md b/docs/skill-evolution-host.md new file mode 100644 index 0000000000..bd08f5bc1f --- /dev/null +++ b/docs/skill-evolution-host.md @@ -0,0 +1,224 @@ +# Skill evolution host API (P0) + +DeerFlow supplies evidence, versioned candidate staging, checking, publication, +and recovery. The learning strategy, candidate generation, quality evaluation, +and product UI belong to the extension and are not included in this feature. + +## Enable deliberately + +Use `deerflow-extension-api==0.2.7`. Existing extensions need no changes: both +`ExtensionRuntimeDeps.completed_run_evidence` and `.skill_mutations` default to +`None`. The older cross-user `run_evidence_reader` is unchanged. + +Automatic publication requires: + +- A durable application database (`sqlite` or `postgres`) and `run_events.backend: db`. +- One POSIX host and shared **local** skill storage. NFS, CSI, and multi-node + deployments are unsupported, including when the database is PostgreSQL. +- All managed writers upgraded together; no concurrent direct filesystem edits. +- One Gateway process for SQLite, enforced by a process-lifetime file lock. +- Enabled static scanning and a working configured security-scanning model. + Unavailable or incomplete scans never allow automatic publication. + +Grant only the intended owners, trigger agents, operations, and existing skill +names in operator-controlled `config.yaml`, then restart the Gateway: + +```yaml +plugins: + - name: skill-evolution + use: your_package:install + host_access: + evidence: + owners: ["exact-user-id"] + skill_mutations: + owners: ["exact-user-id"] + operations: [stage, check, commit, revert] + trigger_agents: [lead_agent] + target_skills: [your-existing-custom-skill] + topology: single_host_local +``` + +This is an example grant, not a bundled plugin. Owner lists are exact and +mutation owners must also have evidence access. Trigger agents are canonical +`AgentConfig.name` values, not display names. Both plugin name and installation +entry point must be unique and stable; changing either changes its durable +identity. Grants bind to host-recorded contribution sources, never plugin-supplied +IDs. They take effect at restart, not live config reload. + +These controls prevent mistakes through the supported API. Trusted Python +extensions still have Gateway process privileges and legacy database access; +this is not an untrusted-code sandbox or proof of learning quality. +Ordinary host logs omit candidate contents and raw model errors. Security scans +still send candidate text to the configured model, and operator-enabled tracing +may retain its prompts; apply your provider and tracing data-retention policies. + +## Integrate a plugin + +Acquire the two optional dependencies in the extension service's `start(deps)`. +All methods are asynchronous; DTOs come from `deerflow_extension_api`. + +1. Discover changed runs and obtain a completed-run snapshot. Terminal status + alone does not mean complete evidence: inspect `seal_state`, coverage limits, + origin, and canonical agent identity. Pages stay within the snapshot's fixed + upper sequence; deletion/retention invalidates old references. +2. `read_skill(source_ref=..., name=...)` obtains the opaque target and revision. +3. `stage(source_refs=..., target_ref=..., expected_revision=..., content=..., + idempotency_key=...)` persists an immutable candidate. +4. Inspect `read_proposal_bundle(proposal_id=...)` to evaluate the exact full + baseline and candidate. These detached bytes are never activated as skills. +5. `check(proposal_id=...)` runs fail-closed host checks; require `allow`. +6. `commit(proposal_id=..., idempotency_key=..., assessment_ref=...)` returns a + durable operation. An optional assessment reference records the plugin's + evaluation; the host neither fetches its URL nor endorses its conclusion. +7. After response loss, use `find_operation(idempotency_key=..., method="commit")` + or retry with the **same** arguments and key. A different key is not a safe + way to retry a logically identical request. + +P0 updates only existing, enabled, user-custom `SKILL.md` body and description. +Every other parsed frontmatter value, including unknown fields, stays unchanged. +It cannot create, rename, delete, toggle, or modify support files. Symlinks and +special files anywhere in the automatic-update package are unsupported. + +Revisions include incarnation, mutation sequence, and whole-package digest. +Manual edits, support-file edits, delete/recreate, and enable toggles invalidate +stale proposals, even if content later returns to its original bytes. Unrelated +skills do not share a compare-and-swap revision. + +## Limits and retention + +The service advertises limits through `.capabilities`: + +| Resource | P0 limit | +| --- | --- | +| Main file | 128 KiB | +| Complete package | 256 files / 16 MiB | +| Evidence sources | 20, all for the same owner | +| Pending proposals | 20 per owner/plugin | +| Model scan | 32 text files / 1 MiB total / 128 KiB each / 120 seconds | +| Live scans | 4 globally; 2 per owner/plugin | +| Scan attempts | 20 per owner/plugin per hour, including failed checks and reverts | +| Proposal / check lifetime | 24 hours / 15 minutes | +| Rollback snapshot lifetime | 30 days | + +Oversized bundles fail as a whole. A scan never approves only the first part of +a package. Model-based checks can incur model cost, even though the host does +not generate evolved skills. Metadata and idempotency records survive blob +expiry; expired rollback bytes are no longer available. Recovery references for +unresolved operations are not garbage-collected. +The legacy JSONL history receives only an idempotent, compact metadata mirror, +not retained skill bodies. Repair replays at most the latest 100 applied +operations per owner; files above 8 MiB, lines above 64 KiB, or malformed history +are left untouched. The operation database remains authoritative even if the +display mirror is unavailable. + +A cancelled scan or expired scan deadline propagates to the caller and releases +its proposal/scan leases without caching an `unavailable` verdict for 15 minutes. +The attempt still counts against scan-cost quotas. Retrying `check()` starts a +fresh scan; cancelling one waiter alone leaves a shared in-flight scan running +for its other waiters. Completed unavailable/rejected verdicts retain the normal +cache policy. + +## Publication, recovery, and rollback + +Publication and derived-view readiness are separate. `APPLIED` means canonical +bytes are committed; `views=ERROR` does not mean they were rolled back. Keep the +operation ID and inspect its current state rather than blindly resubmitting. + +Before touching canonical bytes, the host durably records `PREPARED`, before/after +snapshots, and reserved versions. It then atomically replaces and fsyncs the main +file. Recovery compares the exact package with those snapshots: + +- Before snapshot: `ABORTED`; an unapplied candidate is never replayed. +- After snapshot: finalize `APPLIED` with the reserved version exactly once. +- Neither: `NEEDS_REPAIR`; do not overwrite possible human work. + +Affected owner reads remain gated while publication is unresolved. Other owners +remain usable. Recovery belongs to DeerFlow, so disabling or uninstalling the +originating plugin does not disable coordination or recovery. + +Administrators using session authentication can discover unresolved publication +or view operations through paginated `GET /api/skill-mutations/operations` +(`limit` up to 100, `after_id` from `next_cursor`). Operation pagination ends +with `has_more: false` and `next_cursor: null`, including an empty page after a +supplied cursor. They can query +`GET /api/skill-mutations/operations/{operation_id}`, reconcile through +`POST /api/skill-mutations/operations/{operation_id}/recover`, or reconcile an +owner's interrupted managed writes through +`POST /api/skill-mutations/owners/{owner_id}/recover`. Ordinary users and personal +access tokens cannot use these endpoints. If reconciliation cannot clear the +owner's readiness barrier, either recovery endpoint returns HTTP 409 with +`detail: "NEEDS_REPAIR"`; query the operation to inspect its durable state and +retry after repairing the underlying storage problem. A successful response +still requires inspecting `views`, since a projection rebuild can record +`views=ERROR` independently of publication. There is no force-overwrite endpoint. + +`revert(operation_id=..., expected_current_revision=..., idempotency_key=...)` +creates a new forward revision only if that operation is still the asset's +current head. It rescans the original package under current policy. A later +manual edit or toggle causes a conflict, not a destructive rollback. Revert and +operation queries can survive source-run deletion; a new commit cannot. + +Deleting a conversation does not erase information already distilled into a +skill. Skill removal and deletion of derived information are separate operations; +P0 does not implement automatic derived-content forgetting. + +Local measurements on macOS arm64 (five captures, warm local disk): canonical +capture plus digest took a median 4.43 ms for 4 files / 3,120 bytes and 94.03 ms +for 256 files / 16,711,473 bytes. These are package-processing measurements, not +model latency or a deployment throughput guarantee. Reproduce with +`uv run pytest tests/test_skill_mutation_package_cost.py -q -s` from `backend/`. + +Live PostgreSQL validation on 2026-09-23 used PostgreSQL 17.11 (aarch64, +`READ COMMITTED`), one pytest host process and two concurrent actors sharing a +disposable database schema: a publication-fence worker thread holding the source +run's `SELECT ... FOR SHARE` lock, and an async `RunRepository.delete()` worker. +The opt-in regression passed six times (five repetitions plus the related suite); +deletion waited for the share lock, then removed both the run and its completed +snapshot. The related mutation/evidence/host suite passed 209 tests. See the +[recorded validation](https://github.com/bytedance/deer-flow/pull/5645#issuecomment-5791306188) +for the command, environment and cleanup. + +This verifies the source-fence/deletion interleaving, not a multi-process +PostgreSQL soak. Idempotency, recovery, scan-budget and lease behavior were +covered by deterministic suites, not by concurrent PostgreSQL Gateway workers. +`tests/test_skill_mutation_postgres.py` remains opt-in via +`DEERFLOW_TEST_POSTGRES_URL` and creates/removes its own uniquely named schema. +Validate the intended same-host worker deployment before enabling it; SQLite +results alone do not establish PostgreSQL lock behavior. + +## Upgrade and operational rollback + +Back up the application database and user skill directories together. Stop all +writers, upgrade all Gateway/embedded writer processes and migrate through +`0046_skill_mutations`, then restart before enabling mutation grants. Do not mix +old and new writers on enrolled assets. + +Standalone SDK processes without a Gateway mutation runtime check persisted +owner/asset enrollment on every managed storage access. Enrolled owners are +rejected with `MUTATION_RUNTIME_REQUIRED`; use the Gateway for their skill reads +and writes. Global enable-state writes are also rejected while any owner is +enrolled, because they affect same-named custom skills. Removing the plugin +does not remove this barrier. Unavailable enrollment checks fail closed with +`UNAVAILABLE`; missing/legacy databases and unenrolled owners retain ordinary +SDK access. Owned custom-skill YAML diagnostics follow the same guard: Gateway +reads first recover pending publications, and standalone SDK reads refuse enrolled +owners without a mutation runtime. Processes sharing a skill home must use the same +application DB. + +To stop automation, disable the plugin or remove mutation grants and restart; +keep the upgraded host running so it can reconcile outstanding operations. +Before downgrading host code, quiesce writers, inspect unresolved operations, and +resolve readiness barriers. Do not downgrade while a `PREPARED` or +`NEEDS_REPAIR` record exists. Restore database and files as a coordinated backup +only under an explicit operator recovery procedure. + +This host slice follows up the +[Skill Self-Evolution RFC](https://github.com/bytedance/deer-flow/issues/1865) +and the bilingual +[Plugin Host APIs RFC](https://github.com/bytedance/deer-flow/issues/5539). +Neither RFC implies that this PR ships a learning plugin. + +Journal writes with unknown outcomes (including lost acknowledgements or cancelled +owned writes) prevent evidence sealing and are not replayed. Only a store-proven +not-committed batch can be retried; a cancelled waiter does not cancel the owned +write. Normal close stops evidence producers before draining accepted callbacks.