From c18df580f61aa104d823ecbd0e7b362576eb7395 Mon Sep 17 00:00:00 2001 From: hataa <79907651+hata33@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:46:20 +0800 Subject: [PATCH] feat(extensions): recover opt-in workflows unattended after restart (#8289) * feat(extensions): recover opt-in workflows unattended after restart Workflow launches from an authenticated browser session may opt in to unattended recovery. The launch captures the session's existing authority ceiling into a bounded, revocable grant on the workflow record; after a Gateway restart the single-Gateway SQLite store resumes recoverable records holding a usable grant through the existing journal-replay path, without a sign-in. Host: restore_delegated_authority (sibling of restore_batch_authority) plus capture_authority on bound handles; the per-operation chokepoint runs an extension-supplied grant validator so expiry and revocation fail closed on every operation, including artifact reads. Clustered stores and uncertain admissions stay on the manual path. * fix(extensions): address review findings on unattended workflow recovery - grant_validator validates the current persisted grant during locked admission, where the entry is not published yet, and switches to the live entry after publication; the previous entries-only lookup 403'd every unattended restore of a perfectly usable grant - delegated restore waits for the AgentRuns host installation signal instead of racing the rest of Gateway startup and failing the scan - cancellation defers grant revocation to the supervisor's post-cleanup save, keeping the delegated handle authorized to cancel children the recovered workflow already admitted - the per-operation delegated guard rechecks the current account disablement, which token_version does not reflect - restore stub revalidates per operation like the real host, with regression tests for all of the above; format the new fixture and condense the extension guidance within its budget * fix(tests): cancel the recovered workflow on its own service The fixture's action handlers are bound to the original service, which stop() already made unavailable; the recovered record lives on the replacement instance, so the cancel must go through replacement.cancel. * fix(extensions): let a cancelled replayed child keep its cancelled outcome The replay fence forced every errored re-dispatched step to needs_reconciliation, including the deterministic case where the error is the workflow stopping after its child was already cancelled and observed. Fence only genuinely uncertain outcomes; a clean cancel of a recovered workflow now ends cancelled like a fresh one. --- backend/app/gateway/deps.py | 35 +++ backend/app/gateway/extension_agent_runs.py | 60 +++- .../deerflow_extension_api/agent_runs.py | 12 + .../deerflow_extension_api/contracts.py | 6 + .../harness/deerflow/extensions/AGENTS.md | 1 + .../harness/deerflow/extensions/gateway.py | 2 + backend/tests/test_extension_agent_runs.py | 109 ++++++++ .../tests/test_gateway_lifespan_shutdown.py | 21 ++ backend/tests/test_workflow_plugin.py | 257 ++++++++++++++++++ .../deerflow-extension-workflows/README.md | 28 +- .../deerflow_extension_workflows/recovery.py | 70 ++++- .../deerflow_extension_workflows/service.py | 63 ++++- .../deerflow_extension_workflows/store.py | 4 + 13 files changed, 655 insertions(+), 13 deletions(-) diff --git a/backend/app/gateway/deps.py b/backend/app/gateway/deps.py index d69fd2f5d8..ee8c1ab53d 100644 --- a/backend/app/gateway/deps.py +++ b/backend/app/gateway/deps.py @@ -53,6 +53,30 @@ logger = logging.getLogger(__name__) # timeout. _RUN_DRAIN_TIMEOUT_SECONDS = 5.0 +# Upper bound (seconds) a delegated-restore call may wait for the AgentRuns +# host to be installed during Gateway startup. Startup normally signals within +# its first seconds; the bound only fails a stuck startup instead of leaving +# unattended recovery hanging, and per-record failures keep manual recovery. +_EXTENSION_RESTORE_READY_TIMEOUT_SECONDS = 60.0 + + +async def _agent_runs_host_after_install(app, installed: asyncio.Event): + """Resolve the AgentRuns host once startup has installed it. + + Extension services start before the run-admission stack (thread store, run + manager, AgentRuns host) exists, and unattended restoration can run during + that window, so wait for the explicit installation signal instead of + failing the whole recovery scan. + """ + try: + await asyncio.wait_for(installed.wait(), _EXTENSION_RESTORE_READY_TIMEOUT_SECONDS) + except TimeoutError: + pass + host = getattr(app.state, "agent_runs_host", None) + if host is None: + raise RuntimeError("Agent run host unavailable") + return host + def _browser_tools_enabled_in_config(config: AppConfig) -> bool: """Return whether process-local agentic browser sessions are configured.""" @@ -1020,6 +1044,13 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen # Register cleanup before starting: start() can partially acquire # resources and then fail or be cancelled. stack.push_async_callback(stop_extension_services) + + agent_runs_host_installed = asyncio.Event() + + async def restore_agent_runs_authority(*, user_id: str, grant: dict, validator=None, plugin_namespace: str | None = None): + host = await _agent_runs_host_after_install(app, agent_runs_host_installed) + return await host.restore_delegated_authority(user_id=user_id, grant=grant, validator=validator, plugin_namespace=plugin_namespace) + record_runtime_diagnostics( await start_services( extensions, @@ -1028,6 +1059,7 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen run_evidence_reader=app.state.run_evidence_reader, host_capabilities=host_capabilities.bindings, attempted_services=attempted_services, + agent_runs_restore=restore_agent_runs_authority, ) ) @@ -1125,6 +1157,9 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen app.state.agent_runs_host = GatewayAgentRunsHost(app, load_user=SQLiteUserRepository(sf).get_user_by_id if sf is not None else None) stack.callback(app.state.agent_runs_host.close) + # Extension unattended recovery waits on this before restoring grants; + # past it, the whole run-admission stack above is ready. + agent_runs_host_installed.set() # Start the lease heartbeat if enabled (multi-worker deployments). await app.state.run_manager.start_heartbeat() diff --git a/backend/app/gateway/extension_agent_runs.py b/backend/app/gateway/extension_agent_runs.py index 3f43188bfe..28fb29d11a 100644 --- a/backend/app/gateway/extension_agent_runs.py +++ b/backend/app/gateway/extension_agent_runs.py @@ -89,9 +89,43 @@ class GatewayAgentRunsHost: raise AgentRunError(403, "Batch follow-up authorization revoked") return _BoundAgentRuns(self, user_id, version, source, frozenset(permissions), origin=make_origin("subagent_batch")) + async def restore_delegated_authority( + self, + *, + user_id: str, + grant: dict, + validator: Callable[[], Awaitable[None]] | None = None, + plugin_namespace: str | None = None, + ): + """Restore a persisted delegated grant (e.g. workflow recovery) with its ceiling. + + The grant is the mapping ``capture_authority`` returned at mint time + (token_version, source, permissions); expiry and revocation are the + caller's contract, enforced through ``validator`` on every operation + this handle performs. Restoring re-loads the user row and fails closed + on a changed token_version, needs_setup or a disabled account, exactly + like ``restore_batch_authority``; it never creates fresh authority. + """ + source, version, permissions = grant.get("source"), grant.get("token_version"), grant.get("permissions") + if source not in (AUTH_SOURCE_SESSION, AUTH_SOURCE_AUTH_DISABLED) or type(version) is not int or not isinstance(permissions, list) or any(not isinstance(value, str) for value in permissions): + raise AgentRunError(403, "Invalid delegated authority") + user = get_auth_disabled_user() if source == AUTH_SOURCE_AUTH_DISABLED and is_auth_disabled() else await self.load_user(user_id) if source == AUTH_SOURCE_SESSION and self.load_user is not None else None + if user is None or str(user.id) != user_id or user.token_version != version or user.needs_setup or getattr(user, "disabled", False): + raise AgentRunError(403, "Delegated authorization revoked") + return _BoundAgentRuns( + self, + user_id, + version, + source, + frozenset(permissions), + plugin_namespace=plugin_namespace, + origin=make_origin("extension", namespace=plugin_namespace), + delegation_validator=validator, + ) + class _BoundAgentRuns: - def __init__(self, host, user_id: str, token_version: int, source: str, permissions: frozenset[str], plugin_namespace: str | None = None, origin: dict | None = None): + def __init__(self, host, user_id: str, token_version: int, source: str, permissions: frozenset[str], plugin_namespace: str | None = None, origin: dict | None = None, delegation_validator: Callable[[], Awaitable[None]] | None = None): self._host = host self._user_id = user_id self._token_version = token_version @@ -99,6 +133,19 @@ class _BoundAgentRuns: self._permissions = permissions self._plugin_namespace = plugin_namespace self._origin = origin + self._delegation_validator = delegation_validator + + def capture_authority(self) -> dict | None: + """Persistable restorable snapshot of this handle's existing authority. + + Returns the mint-time ceiling (token_version, source, permissions) for + a host-originated session handle, or None when this handle's authority + is not restorable. The host never validates the caller here: capture + only reflects authority the live handle already holds. + """ + if not self._host.active or self._source not in (AUTH_SOURCE_SESSION, AUTH_SOURCE_AUTH_DISABLED): + return None + return {"token_version": self._token_version, "source": self._source, "permissions": sorted(self._permissions)} def for_plugin(self, namespace: str): if not isinstance(namespace, str) or not namespace or len(namespace) > 96 or ":" in namespace: @@ -107,7 +154,7 @@ class _BoundAgentRuns: if namespace != self._plugin_namespace: raise AgentRunError(422, "Agent run handle is already bound to a plugin") return self - return _BoundAgentRuns(self._host, self._user_id, self._token_version, self._source, self._permissions, namespace) + return _BoundAgentRuns(self._host, self._user_id, self._token_version, self._source, self._permissions, namespace, self._origin, self._delegation_validator) @asynccontextmanager async def _request(self, permission: str, thread_id: str | None = None): @@ -117,8 +164,15 @@ class _BoundAgentRuns: user = get_auth_disabled_user() if is_auth_disabled() else None else: user = await self._host.load_user(self._user_id) if self._host.load_user is not None else None - if user is None or str(user.id) != self._user_id or user.token_version != self._token_version or user.needs_setup: + if user is None or str(user.id) != self._user_id or user.token_version != self._token_version or user.needs_setup or getattr(user, "disabled", False): raise AgentRunError(403, "Agent run authorization revoked") + if self._delegation_validator is not None: + try: + await self._delegation_validator() + except AgentRunError: + raise + except Exception as exc: + raise AgentRunError(403, "Delegated authorization revoked") from exc permissions = self._permissions.intersection(await resolve_route_permissions(user, is_internal=False)) if permission not in permissions: raise AgentRunError(403, f"Permission denied: {permission}") diff --git a/backend/packages/extension-api/deerflow_extension_api/agent_runs.py b/backend/packages/extension-api/deerflow_extension_api/agent_runs.py index 4f41f39c98..428239c264 100644 --- a/backend/packages/extension-api/deerflow_extension_api/agent_runs.py +++ b/backend/packages/extension-api/deerflow_extension_api/agent_runs.py @@ -76,6 +76,18 @@ class AgentRuns(Protocol): """ ... + def capture_authority(self) -> Mapping[str, Any] | None: + """Persistable snapshot of this handle's existing authority, or None. + + Returns the mint-time ceiling (token_version, source, permissions) of + the live session handle so a service can restore equivalent authority + later through the host's delegated-restore capability. Capture only + reflects authority the handle already holds; it is not a credential to + serialize across hosts, and None means this host or handle cannot be + restored. Hosts that predate this method lack it entirely. + """ + ... + async def send_input(self, *, thread_id: str, run_id: str, text: str, mode: str, request_id: str, context: Mapping[str, Any] | None = None) -> dict[str, Any]: """Queue a new turn or steer the active Run; never change its model.""" ... diff --git a/backend/packages/extension-api/deerflow_extension_api/contracts.py b/backend/packages/extension-api/deerflow_extension_api/contracts.py index 20428c8288..7eb3d64c1a 100644 --- a/backend/packages/extension-api/deerflow_extension_api/contracts.py +++ b/backend/packages/extension-api/deerflow_extension_api/contracts.py @@ -251,6 +251,12 @@ class ExtensionRuntimeDeps: model_invoker: ModelInvoker | None = None completed_run_evidence: CompletedRunEvidenceReader | None = None skill_mutations: SkillMutationService | None = None + # Async callable (user_id, grant, *, validator=None, plugin_namespace=None) + # -> AgentRuns restoring a captured session ceiling for unattended host-side + # recovery. The host revalidates the user row per operation and fails closed; + # expiry/revocation stay the extension's contract via the validator. None on + # hosts without the delegated-restore capability. + agent_runs_restore: Any | None = None class ExtensionService(Protocol): diff --git a/backend/packages/harness/deerflow/extensions/AGENTS.md b/backend/packages/harness/deerflow/extensions/AGENTS.md index 113c652ac2..91524739d6 100644 --- a/backend/packages/harness/deerflow/extensions/AGENTS.md +++ b/backend/packages/harness/deerflow/extensions/AGENTS.md @@ -101,6 +101,7 @@ hook directory, so they never run a developer's or CI's Git hooks. cannot collide. - **`configuration_revision` never exposes raw configuration.** It returns an HMAC. Sets hash the same way across process hash seeds; ordered lists keep their order. +- **Restorable authority is captured, never created.** Restores revalidate it. ## Extension-first exceptions diff --git a/backend/packages/harness/deerflow/extensions/gateway.py b/backend/packages/harness/deerflow/extensions/gateway.py index 5d3730e105..456dbb35d8 100644 --- a/backend/packages/harness/deerflow/extensions/gateway.py +++ b/backend/packages/harness/deerflow/extensions/gateway.py @@ -581,6 +581,7 @@ async def start_services( run_evidence_reader: Any | None = None, host_capabilities: dict[str, tuple[Any | None, Any | None]] | None = None, attempted_services: list[tuple[str, Any]] | None = None, + agent_runs_restore: Any | None = None, ) -> list[Diagnostic]: """Start extension services in registration order, failing open per item.""" diagnostics: list[Diagnostic] = [] @@ -592,6 +593,7 @@ async def start_services( policy=project_host_policy(app_config), session_factory=session_factory, run_evidence_reader=run_evidence_reader, + agent_runs_restore=agent_runs_restore, ) source_deps = {} for entry in extensions.services: diff --git a/backend/tests/test_extension_agent_runs.py b/backend/tests/test_extension_agent_runs.py index 1d55fe5e12..3975c5443c 100644 --- a/backend/tests/test_extension_agent_runs.py +++ b/backend/tests/test_extension_agent_runs.py @@ -71,6 +71,21 @@ async def test_handle_rechecks_user_and_shutdown_before_dispatch(host, monkeypat assert handler.await_count == 1 +@pytest.mark.asyncio +async def test_handle_rechecks_current_account_disablement(host, monkeypatch): + control, request, load_user = host + runs = control.bind(request) + handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a")) + monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler) + assert await runs.create_thread() == "thread-a" + # Disabling the account leaves token_version unchanged, so the per-operation + # guard must observe the current disabled state, not only the mint-time one. + load_user.return_value = request.state.user.model_copy(update={"disabled": True}) + with pytest.raises(AgentRunError, match="revoked"): + await runs.create_thread() + assert handler.await_count == 1 + + @pytest.mark.asyncio async def test_permissions_are_intersected_with_original_grant_and_current_policy(host, monkeypatch): control, request, _ = host @@ -667,3 +682,97 @@ async def test_configuration_revision_canonicalizes_sets_but_preserves_list_orde assert before == await runs.configuration_revision(assistant_ids=["lead_agent"], durable=True) data["tools"].reverse() assert before != await runs.configuration_revision(assistant_ids=["lead_agent"], durable=True) + + +@pytest.mark.asyncio +async def test_capture_authority_snapshots_the_session_ceiling(host, monkeypatch): + control, request, _ = host + runs = control.bind(request) + handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a")) + monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler) + assert await runs.create_thread() == "thread-a" + captured = runs.capture_authority() + assert captured == { + "token_version": request.state.user.token_version, + "source": "session", + "permissions": sorted(request.state.auth.permissions), + } + + +@pytest.mark.asyncio +async def test_restore_delegated_authority_enforces_validator_per_operation(host, monkeypatch): + control, request, load_user = host + handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a")) + monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler) + calls = [] + + async def validator(): + calls.append(1) + if len(calls) > 1: + raise PermissionError("grant revoked by the owner") + + runs = await control.restore_delegated_authority( + user_id=str(request.state.user.id), + grant={"token_version": request.state.user.token_version, "source": "session", "permissions": ["threads:write"]}, + validator=validator, + plugin_namespace="workflows", + ) + assert await runs.create_thread() == "thread-a" + with pytest.raises(AgentRunError, match="Delegated authorization revoked"): + await runs.create_thread() + assert handler.await_count == 1 + + +@pytest.mark.asyncio +async def test_restore_delegated_authority_maps_validator_errors_to_403(host, monkeypatch): + control, request, _ = host + handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a")) + monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler) + + async def validator(): + raise AgentRunError(403, "Workflow recovery grant expired or revoked") + + runs = await control.restore_delegated_authority( + user_id=str(request.state.user.id), + grant={"token_version": request.state.user.token_version, "source": "session", "permissions": ["threads:write"]}, + validator=validator, + ) + with pytest.raises(AgentRunError, match="grant expired or revoked"): + await runs.create_thread() + assert handler.await_count == 0 + + +@pytest.mark.asyncio +async def test_restore_delegated_authority_rejects_stale_grants(host): + control, request, load_user = host + user = request.state.user + load_user.return_value = user.model_copy(update={"token_version": user.token_version + 1}) + with pytest.raises(AgentRunError, match="Delegated authorization revoked"): + await control.restore_delegated_authority( + user_id=str(user.id), + grant={"token_version": user.token_version, "source": "session", "permissions": ["threads:write"]}, + ) + with pytest.raises(AgentRunError, match="Invalid delegated authority"): + await control.restore_delegated_authority( + user_id=str(user.id), + grant={"token_version": user.token_version, "source": "pat", "permissions": ["threads:write"]}, + ) + with pytest.raises(AgentRunError, match="Invalid delegated authority"): + await control.restore_delegated_authority( + user_id=str(user.id), + grant={"token_version": user.token_version, "source": "session", "permissions": [1]}, + ) + + +@pytest.mark.asyncio +async def test_restored_delegated_handle_is_plugin_scoped_with_extension_origin(host): + control, request, _ = host + runs = await control.restore_delegated_authority( + user_id=str(request.state.user.id), + grant={"token_version": request.state.user.token_version, "source": "session", "permissions": ["threads:write"]}, + plugin_namespace="workflows", + ) + assert runs.for_plugin("workflows") is runs + assert runs._origin == {"kind": "extension", "namespace": "workflows"} + with pytest.raises(AgentRunError, match="already bound"): + runs.for_plugin("other") diff --git a/backend/tests/test_gateway_lifespan_shutdown.py b/backend/tests/test_gateway_lifespan_shutdown.py index c765a4f299..365e0da69e 100644 --- a/backend/tests/test_gateway_lifespan_shutdown.py +++ b/backend/tests/test_gateway_lifespan_shutdown.py @@ -957,3 +957,24 @@ def test_lifespan_detaches_the_outbox_when_the_delivery_worker_cannot_start() -> # Startup survives, and nothing keeps enqueueing rows that no worker would send. scheduled_service.detach_notification_outbox.assert_called_once_with() + + +@pytest.mark.asyncio +async def test_delegated_restore_waits_for_agent_runs_host_installation(monkeypatch): + from app.gateway.deps import _agent_runs_host_after_install + + app = SimpleNamespace(state=SimpleNamespace()) + installed = asyncio.Event() + # Extension services start before the AgentRuns host exists, so a restore + # issued during that window waits for the installation signal instead of + # failing the unattended recovery scan outright. + waiter = asyncio.create_task(_agent_runs_host_after_install(app, installed)) + assert not waiter.done() + app.state.agent_runs_host = object() + installed.set() + assert await asyncio.wait_for(waiter, 1) is app.state.agent_runs_host + # A startup that never installs the host fails closed per record within + # the bounded budget; manual recovery stays available. + monkeypatch.setattr("app.gateway.deps._EXTENSION_RESTORE_READY_TIMEOUT_SECONDS", 0.01) + with pytest.raises(RuntimeError, match="Agent run host unavailable"): + await _agent_runs_host_after_install(SimpleNamespace(state=SimpleNamespace()), asyncio.Event()) diff --git a/backend/tests/test_workflow_plugin.py b/backend/tests/test_workflow_plugin.py index 0a25db5c23..0c59aa786f 100644 --- a/backend/tests/test_workflow_plugin.py +++ b/backend/tests/test_workflow_plugin.py @@ -41,6 +41,11 @@ class Runs: revision = "initial" + def capture_authority(self): + if self.denied: + return None + return {"token_version": 0, "source": "session", "permissions": ["runs:create", "threads:write"]} + async def configuration_evidence(self, *, thread_id, run_id): return {"run_id": run_id, "thread_id": thread_id, "source": "host_assembly", "version": "synthetic-assembly", "model": "test-model"} @@ -1162,3 +1167,255 @@ agent({stepId:args.second,agentRef:'lead_agent',task:'read',artifacts:[f.artifac assert len(result["steps"]) <= 2 assert len(runs.starts) <= 2 assert len({s["step_id"] for s in result["steps"]}) == len(result["steps"]) + + +class ValidatingRuns: + """Wraps every delegated operation in the host's per-operation validator gate.""" + + def __init__(self, runs, validator): + self._runs = runs + self._validator = validator + + def __getattr__(self, name): + attribute = getattr(self._runs, name) + if not callable(attribute): + return attribute + + async def operation(*args, **kwargs): + await self._validator() + return await attribute(*args, **kwargs) + + return operation + + +class Restorer: + """Host-side delegated restore stub; the real host revalidates the grant on + every operation of the returned handle, including before publication.""" + + def __init__(self, runs): + self.runs = runs + self.calls = [] + + async def __call__(self, *, user_id, grant, validator=None, plugin_namespace=None): + self.calls.append((user_id, grant, plugin_namespace)) + return self.runs if validator is None else ValidatingRuns(self.runs, validator) + + +async def crash_like_record(service, run_id, owner="alice"): + """What a SIGKILL leaves: recoverable, children back at their queued window.""" + record = await asyncio.to_thread(service.store.get, owner, run_id) + record["status"] = "recoverable" + record["error"] = None + for row in record["steps"]: + if row["status"] in ("queued", "creating_thread", "admitting", "running", "cancelled"): + row["status"] = "queued" + row["run_id"] = None + row.pop("error", None) + # A graceful stop records an error response for the in-flight frame; a real + # crash loses it, leaving the frame unanswered for replay to re-dispatch. + for item in record["journal"]: + item["responses"] = None + await asyncio.to_thread(service.store.put, owner, record) + return record + + +@pytest.mark.asyncio +async def test_unattended_recovery_requires_session_opt_in_and_capture(plugin): + from dataclasses import replace + + _, actions, service, runs = plugin + with pytest.raises(ValueError, match="Unattended recovery requires a session-authenticated launch"): + await actions["launch"]({"request_id": "a", "script": "return 1;", "args": {}, "unattended_recovery": True}, context(runs)) + with pytest.raises(ValueError, match="unattended_recovery must be boolean"): + await actions["launch"]({"request_id": "b", "script": "return 1;", "args": {}, "unattended_recovery": "yes"}, replace(context(runs), interaction="session")) + session = replace(context(runs), interaction="session") + plain = await actions["launch"]({"request_id": "c", "script": "return 1;"}, session) + assert (await asyncio.to_thread(service.store.get, "alice", plain["run_id"]))["grant"] is None + granted = await actions["launch"]({"request_id": "d", "script": "return 1;", "unattended_recovery": True}, session) + record = await asyncio.to_thread(service.store.get, "alice", granted["run_id"]) + assert record["grant"] == { + "authority": {"token_version": 0, "source": "session", "permissions": ["runs:create", "threads:write"]}, + "expires_at": pytest.approx(time.time() + service.config["pause_timeout_seconds"], abs=5), + "revoked": False, + "version": 1, + } + # The grant is authority metadata and never projected to API responses. + assert "grant" not in await actions["get"]({"run_id": granted["run_id"]}, context(runs)) + + +@pytest.mark.asyncio +async def test_unattended_recovery_resumes_crashed_workflow_to_child_results(plugin): + from dataclasses import replace + from types import SimpleNamespace + + from deerflow_extension_workflows.constants import NAMESPACE + from deerflow_extension_workflows.service import Workflows + + _, actions, service, runs = plugin + script = "const a = await agent({stepId:'a',agentRef:'lead_agent',task:'blocked'});const b = await agent({stepId:'b',agentRef:'lead_agent',task:'second'});return b.output;" + launched = await actions["launch"]({"request_id": "crash", "script": script, "unattended_recovery": True}, replace(context(runs), interaction="session")) + run_id = launched["run_id"] + # Freeze the crash window: the step row is saved queued, admission never starts. + gate = asyncio.Event() + original_create = runs.create_thread + + async def gated_create(**kwargs): + await gate.wait() + return await original_create(**kwargs) + + runs.create_thread = gated_create + for _ in range(500): + record = await asyncio.to_thread(service.store.get, "alice", run_id) + if record["steps"]: + break + await asyncio.sleep(0.01) + gate.set() + await service.stop() + runs.create_thread = original_create + record = await crash_like_record(service, run_id) + # Nothing was truly admitted before the crash; drop the interrupted stand-ins. + stale_thread = record["steps"][0]["thread_id"] + runs.runs = {key: value for key, value in runs.runs.items() if value.thread_id != stale_thread} + + restorer = Restorer(runs) + replacement = Workflows(service.config) + await replacement.start(SimpleNamespace(agent_runs_restore=restorer)) + try: + for _ in range(500): + if run_id in replacement.entries: + break + await asyncio.sleep(0.01) + assert run_id in replacement.entries, "unattended recovery never resumed the workflow" + runs.release.set() + await asyncio.wait_for(replacement.wait_idle(run_id), 10) + final = await replacement.get({"run_id": run_id}, context(runs)) + assert final["status"] == "completed", (final.get("error"), [(s["step_id"], s["status"], s.get("error")) for s in final["steps"]]) + # The unadmitted child was re-dispatched with its original identity and key. + assert final["steps"][0]["attempt"] == 1 + assert final["steps"][0]["idempotency_key"] == record["steps"][0]["idempotency_key"] + assert final["steps"][0]["thread_id"] == record["steps"][0]["thread_id"] + assert final["result"] == {"echo": "second"} + assert restorer.calls == [("alice", {"token_version": 0, "source": "session", "permissions": ["runs:create", "threads:write"]}, NAMESPACE)] + finally: + gate.set() + runs.release.set() + await replacement.stop() + + +@pytest.mark.asyncio +async def test_cancelling_a_recovered_workflow_keeps_child_cleanup_authorized(plugin): + from dataclasses import replace + from types import SimpleNamespace + + from deerflow_extension_workflows.service import Workflows + + _, actions, service, runs = plugin + launched = await actions["launch"]( + {"request_id": "recancel", "script": "return await agent({stepId:'a',agentRef:'lead_agent',task:'blocked'});", "unattended_recovery": True}, + replace(context(runs), interaction="session"), + ) + run_id = launched["run_id"] + # Freeze the crash window: the step row is saved queued, admission never starts. + gate = asyncio.Event() + original_create = runs.create_thread + + async def gated_create(**kwargs): + await gate.wait() + return await original_create(**kwargs) + + runs.create_thread = gated_create + for _ in range(500): + record = await asyncio.to_thread(service.store.get, "alice", run_id) + if record["steps"]: + break + await asyncio.sleep(0.01) + gate.set() + await service.stop() + runs.create_thread = original_create + record = await crash_like_record(service, run_id) + stale_thread = record["steps"][0]["thread_id"] + runs.runs = {key: value for key, value in runs.runs.items() if value.thread_id != stale_thread} + + replacement = Workflows(service.config) + await replacement.start(SimpleNamespace(agent_runs_restore=Restorer(runs))) + try: + for _ in range(500): + if runs.entered.is_set() and replacement.entries.get(run_id) is not None: + break + await asyncio.sleep(0.01) + assert runs.entered.is_set(), "recovered workflow never re-admitted its child" + # The delegated handle is the only authorized path to the running child; + # revoking before cleanup would strand it in needs_reconciliation. + # Cancel on the replacement service: the fixture's actions are bound to + # the original, which stop() already made unavailable. + assert (await replacement.cancel({"run_id": run_id}, context(runs)))["status"] == "cancelling" + await asyncio.wait_for(replacement.wait_idle(run_id), 10) + final = await replacement.get({"run_id": run_id}, context(runs)) + assert final["status"] == "cancelled", (final.get("error"), [(s["step_id"], s["status"], s.get("error")) for s in final["steps"]]) + assert final["steps"][0]["status"] == "cancelled", final["steps"][0] + persisted = await asyncio.to_thread(replacement.store.get, "alice", run_id) + assert persisted["grant"]["revoked"] is True + finally: + gate.set() + runs.release.set() + await replacement.stop() + + +@pytest.mark.asyncio +async def test_revoked_or_expired_grants_stay_on_the_manual_path(plugin): + from dataclasses import replace + from types import SimpleNamespace + + from deerflow_extension_workflows.service import Workflows + + _, actions, service, runs = plugin + service.config["pause_timeout_seconds"] = 1 + launched = await actions["launch"]({"request_id": "paused", "script": "await pause({stepId:'p'});", "unattended_recovery": True}, replace(context(runs), interaction="session")) + run_id = launched["run_id"] + await waiting(plugin, run_id, "paused") + await service.stop() + await crash_like_record(service, run_id) + + # Revoked: the owner's cancellation withdrew the standing authority. + record = await asyncio.to_thread(service.store.get, "alice", run_id) + record["grant"]["revoked"] = True + await asyncio.to_thread(service.store.put, "alice", record) + restorer = Restorer(runs) + replacement = Workflows(service.config) + await replacement.start(SimpleNamespace(agent_runs_restore=restorer)) + try: + await asyncio.sleep(0.05) + assert not restorer.calls and run_id not in replacement.entries + assert (await replacement.get({"run_id": run_id}, context(runs)))["status"] == "recoverable" + # Manual session recovery is unaffected by the unusable grant. + recovered = await replacement.recover({"run_id": run_id}, replace(context(runs), interaction="session")) + assert recovered["status"] == "queued" + finally: + await replacement.stop() + + +@pytest.mark.asyncio +async def test_expired_grant_never_restores_authority(plugin): + from dataclasses import replace + from types import SimpleNamespace + + from deerflow_extension_workflows.service import Workflows + + _, actions, service, runs = plugin + service.config["pause_timeout_seconds"] = 1 + launched = await actions["launch"]({"request_id": "stale", "script": "await pause({stepId:'p'});", "unattended_recovery": True}, replace(context(runs), interaction="session")) + run_id = launched["run_id"] + await waiting(plugin, run_id, "paused") + await service.stop() + await crash_like_record(service, run_id) + await asyncio.sleep(1.1) # the bounded mint-time TTL lapses + + restorer = Restorer(runs) + replacement = Workflows(service.config) + await replacement.start(SimpleNamespace(agent_runs_restore=restorer)) + try: + await asyncio.sleep(0.05) + assert not restorer.calls and run_id not in replacement.entries + assert (await replacement.get({"run_id": run_id}, context(runs)))["status"] == "recoverable" + finally: + await replacement.stop() diff --git a/examples/deerflow-extension-workflows/README.md b/examples/deerflow-extension-workflows/README.md index 44039060e6..d79153ffbc 100644 --- a/examples/deerflow-extension-workflows/README.md +++ b/examples/deerflow-extension-workflows/README.md @@ -399,8 +399,32 @@ and is distinct from this configuration preflight check. Phase 3 and Phase 4 share the design: stable step/attempt and idempotency identities, conditional record revisions, a durable intent/result journal, and fresh authority support the leases and fenced dispatch in Phase 4 below. SQLite retains the Phase 3 -single-owner contract and exclusive process lock. Both modes require explicit -session authorization for recovery; neither reconstructs credentials from records. +single-owner contract and exclusive process lock. Neither mode reconstructs +credentials from records; recovery authority comes from a live session or, on the +single-Gateway SQLite store, from the opt-in grant below. + + +### Opt-in unattended recovery after restart + +A workflow launched from an authenticated browser session may set +`unattended_recovery: true`. The launch then captures the session's existing +authority ceiling (token version, source, permission set) into a grant on the +workflow record, bounded by `pause_timeout_seconds` from launch time. Nothing is +captured for internal, PAT or model-tool launches; those never mint a grant. + +After a Gateway restart, the single-Gateway SQLite store scans `recoverable` +records and resumes the ones holding an unexpired, unrevoked grant without a +sign-in: journal replay, completed-step reuse and the uncertain-admission fence +behave exactly as in the explicit browser flow above. Clustered (PostgreSQL) +stores keep the manual recovery path. Cancellation revokes the grant; the next +delegated operation fails closed. + +The host re-validates the owner's user row on every delegated operation, keeps +the ceiling intersected with the owner's current route permissions, and re-checks +expiry and revocation per operation. Restoring a grant never creates authority +the owner did not have at launch: a demoted, disabled or reset account stops +recovered work at its next operation. Unfinished child effects and +`needs_reconciliation` outcomes remain manual in every mode. ## Phase 4: shared PostgreSQL deployment diff --git a/examples/deerflow-extension-workflows/deerflow_extension_workflows/recovery.py b/examples/deerflow-extension-workflows/deerflow_extension_workflows/recovery.py index e1f55cb10d..6caba62647 100644 --- a/examples/deerflow-extension-workflows/deerflow_extension_workflows/recovery.py +++ b/examples/deerflow-extension-workflows/deerflow_extension_workflows/recovery.py @@ -1,16 +1,24 @@ -"""Versioned bridge journal replay; authority is reacquired from a live session. +"""Versioned bridge journal replay; authority is reacquired from a live session +or, when the owner minted one at launch, from the persisted recovery grant. SQLite retains its exclusive store owner. PostgreSQL uses a shared execution lease and fenced admission; both replay the same committed bridge journal. +Unattended grant recovery runs on the single-Gateway SQLite store only. """ import asyncio import copy import json +import logging +import time +from deerflow_extension_api import AgentRunError from deerflow_extension_api.plugins import ToolContext +from .constants import NAMESPACE + RECOVERY_VERSION = 1 +logger = logging.getLogger(__name__) class Recovery: @@ -25,11 +33,15 @@ class Recovery: return await asyncio.shield(task) async def restore(self, payload, context): - from .service import identity, text + from .service import text + + run_id = text(payload["run_id"], 64) + return await self.restore_body(self.owner(context), run_id, context.agent_runs) + + async def restore_body(self, owner, run_id, runs): + from .service import identity async with self.launch_lock: - owner = self.owner(context) - run_id = text(payload["run_id"], 64) record = await asyncio.to_thread(self.store.get, owner, run_id) if run_id in self.entries: return {"run_id": run_id, "status": record["status"]} @@ -41,11 +53,11 @@ class Recovery: raise ValueError("Recovery script or arguments changed") if len(self.entries) >= self.config["max_workflows"] or sum(e["owner"] == owner for e in self.entries.values()) >= self.config["max_workflows_per_owner"]: raise ValueError("Active workflow limit") - entry = self.new_entry(owner, record, context.agent_runs) + entry = self.new_entry(owner, record, runs) await self.check_configuration(entry) if self.clustered: record = await asyncio.to_thread(self.store.claim, owner, run_id) - entry = self.new_entry(owner, record, context.agent_runs) + entry = self.new_entry(owner, record, runs) # A committed admission intent with no receipt is never dispatched # again. An operator must reconcile its external effects first. uncertain = [s for s in record["steps"] if s["status"] not in ("queued", "completed", "running") or (s["status"] == "running" and not s.get("run_id"))] @@ -64,6 +76,52 @@ class Recovery: self.publish(entry) return {"run_id": run_id, "status": "queued"} + def grant_usable(self, record): + grant = record.get("grant") + return ( + isinstance(grant, dict) and grant.get("version") == 1 and grant.get("revoked") is not True and isinstance(grant.get("expires_at"), (int, float)) and grant["expires_at"] > time.time() and isinstance(grant.get("authority"), dict) + ) + + def grant_validator(self, owner, run_id): + async def validate(): + entry = self.entries.get(run_id) + if entry is not None: + grant = entry["record"].get("grant") + else: + # Locked admission publishes the entry only after restore, while + # the delegated handle already operates; validate the current + # persisted grant until publication switches to the live entry. + record = await asyncio.to_thread(self.store.get, owner, run_id) + grant = record.get("grant") if isinstance(record, dict) else None + if not isinstance(grant, dict) or grant.get("revoked") is True or grant.get("expires_at", 0) <= time.time(): + raise AgentRunError(403, "Workflow recovery grant expired or revoked") + + return validate + + async def recover_granted_workflows(self): + """Resume opt-in recoverable workflows after a restart, without a session. + + Records without a usable grant stay on the manual sign-in path; per-record + failures never block the others or Gateway startup. + """ + if self.agent_runs_restore is None: + return + try: + records = await asyncio.to_thread(self.store.recoverable) + except Exception: + logger.exception("Workflow grant scan failed; unattended recovery unavailable") + return + for owner, run_id, record in records: + if not self.available: + return + if run_id in self.entries or not self.grant_usable(record): + continue + try: + runs = await self.agent_runs_restore(user_id=owner, grant=record["grant"]["authority"], validator=self.grant_validator(owner, run_id), plugin_namespace=NAMESPACE) + await self.restore_body(owner, run_id, runs) + except Exception: + logger.exception("Workflow %s unattended recovery failed; manual recovery remains available", run_id) + async def begin_frame(self, entry, frame): """Persist intent before effects; replay verifies every emitted request.""" record = entry["record"] diff --git a/examples/deerflow-extension-workflows/deerflow_extension_workflows/service.py b/examples/deerflow-extension-workflows/deerflow_extension_workflows/service.py index 820018dcb1..2629ebda45 100644 --- a/examples/deerflow-extension-workflows/deerflow_extension_workflows/service.py +++ b/examples/deerflow-extension-workflows/deerflow_extension_workflows/service.py @@ -116,6 +116,10 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts): self.write_lock = asyncio.Lock() self.global_slots = asyncio.Semaphore(config["global_concurrency"]) self.lockfile = None + # Host capability restoring captured session authority; None on hosts + # without delegated restore (unattended recovery then stays manual). + self.agent_runs_restore = None + self.recovery_scan = None async def start(self, deps): def initialize(): @@ -157,6 +161,7 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts): raise self.lockfile, self.store, self.runtime_version = await asyncio.to_thread(initialize) + self.agent_runs_restore = getattr(deps, "agent_runs_restore", None) if self.clustered and deps is not None: from sqlalchemy import text as sql_text @@ -169,10 +174,17 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts): if host_identity != await asyncio.to_thread(self.store.database_identity): raise ValueError("Workflow and host must share the same PostgreSQL database and schema") self.available = True + if not self.clustered: + # Single-Gateway stores may hold recoverable records whose owner + # minted an unattended-recovery grant before the restart. Clustered + # stores coordinate through leases and stay on the manual path. + self.recovery_scan = asyncio.create_task(self.recover_granted_workflows()) async def stop(self): async with self.launch_lock: self.available = False + scan = self.recovery_scan + self.recovery_scan = None for entry in self.entries.values(): entry["suspending"] = entry["record"]["status"] != "cancelling" entry["cancel"].set() @@ -183,6 +195,14 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts): await asyncio.gather(*list(self.admissions), return_exceptions=True) if workers: await asyncio.gather(*workers, return_exceptions=True) + if scan is not None: + scan.cancel() + try: + await scan + except asyncio.CancelledError: + pass + except Exception: + logger.exception("Workflow recovery scan failed during shutdown") if self.heartbeats: heartbeats = list(self.heartbeats.values()) for heartbeat in heartbeats: @@ -203,6 +223,16 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts): raise ValueError("Workflow service unavailable or disabled") return context.principal.user_id + @staticmethod + def _revoke_grant(record): + # Withdraw the owner's standing recovery authority with the record, so + # the next delegated operation fails closed. A live supervisor defers + # this until its cleanup of already-admitted children finishes, since + # that cleanup is itself an authorized use of the handle. + grant = record.get("grant") + if isinstance(grant, dict): + grant["revoked"] = True + async def save(self, owner, record): # SQLite commits survive cancellation of to_thread. Keep the lock and # observe the new revision before propagating cancellation to cleanup. @@ -234,8 +264,23 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts): owner = self.owner(context) if context.agent_runs is None: raise ValueError("Session-bound AgentRuns required; PAT/internal callers unsupported") - if set(payload) - {"request_id", "script", "args", "definition"}: + if set(payload) - {"request_id", "script", "args", "definition", "unattended_recovery"}: raise ValueError("Unexpected launch fields") + unattended = payload.get("unattended_recovery", False) + if type(unattended) is not bool: + raise ValueError("unattended_recovery must be boolean") + grant = None + if unattended: + capture = getattr(context.agent_runs, "capture_authority", None) + captured = capture() if callable(capture) else None + if context.interaction != "session" or captured is None: + raise ValueError("Unattended recovery requires a session-authenticated launch") + grant = { + "authority": captured, + "expires_at": time.time() + self.config["pause_timeout_seconds"], + "revoked": False, + "version": 1, + } definition = payload.get("definition") if definition is not None: if "script" in payload or not isinstance(definition, dict): @@ -299,6 +344,7 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts): "steps": [], "logs": [], "result": None, + "grant": grant, } # SQLite threads survive caller cancellation. Retain a service-owned task # through commit AND supervisor publication, shielding only the caller's wait. @@ -401,12 +447,18 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts): entry["suspending"] = False if entry["record"]["status"] not in TERMINAL: entry["record"]["status"] = "cancelling" + # Defer revocation to the supervisor's cleanup: the delegated + # handle is the authorized path that cancels children this + # workflow already admitted, and revoking now would 403 that + # cleanup into needs_reconciliation. + entry["revoke_grant_on_finish"] = True await self.save(owner, entry["record"]) return {"run_id": run_id, "status": "cancelling"} if record["status"] == "recoverable": if context.agent_runs is None: raise ValueError("Session-bound AgentRuns required") record["status"] = "cancelling" + self._revoke_grant(record) await self.save(owner, record) for row in record["steps"]: if row.get("run_id") and row["status"] != "completed": @@ -636,7 +688,10 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts): row["status"] = "failed" response["error"] = str(exc)[:240] if isinstance(exc, ValueError) else "Agent step failed" finally: - if replay and "error" in response: + if replay and "error" in response and not (entry["cancel"].is_set() and row is not None and row["status"] == "cancelled"): + # A replayed step error is fenced as uncertain — unless the + # error is the workflow stopping while its re-dispatched child + # was already cancelled and observed, which is deterministic. entry["fault"] = "Recovered child outcome requires reconciliation" if row: row["status"] = "needs_reconciliation" @@ -829,6 +884,10 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts): cleanup = [self.cancel_child(runs, row, **({"record": record} if self.clustered else {})) for row in record["steps"] if row.get("run_id") and row["status"] not in ("completed", "waiting_input", "cancelled")] if cleanup: await asyncio.gather(*cleanup) + if entry.pop("revoke_grant_on_finish", False): + # Cleanup above was the last authorized user of the delegated + # handle; withdraw the persisted grant before the final save. + self._revoke_grant(record) if self.clustered: for row in record["steps"]: if self.slot_releasable(row): diff --git a/examples/deerflow-extension-workflows/deerflow_extension_workflows/store.py b/examples/deerflow-extension-workflows/deerflow_extension_workflows/store.py index 66111d370b..d37e26a57e 100644 --- a/examples/deerflow-extension-workflows/deerflow_extension_workflows/store.py +++ b/examples/deerflow-extension-workflows/deerflow_extension_workflows/store.py @@ -220,6 +220,10 @@ class Store: ) ] + def recoverable(self): + with closing(self.connect()) as db: + return [(row[0], row[1], json.loads(row[2])) for row in db.execute("SELECT owner,id,data FROM workflows WHERE status='recoverable'").fetchall()] + def delete(self, owner, run_id): with closing(self.connect()) as db, db: row = db.execute("SELECT status FROM workflows WHERE owner=? AND id=?", (owner, run_id)).fetchone()