feat(extensions): recover opt-in workflows unattended after restart (#8289)

* feat(extensions): recover opt-in workflows unattended after restart

Workflow launches from an authenticated browser session may opt in to
unattended recovery. The launch captures the session's existing authority
ceiling into a bounded, revocable grant on the workflow record; after a
Gateway restart the single-Gateway SQLite store resumes recoverable
records holding a usable grant through the existing journal-replay path,
without a sign-in.

Host: restore_delegated_authority (sibling of restore_batch_authority)
plus capture_authority on bound handles; the per-operation chokepoint
runs an extension-supplied grant validator so expiry and revocation fail
closed on every operation, including artifact reads. Clustered stores
and uncertain admissions stay on the manual path.

* fix(extensions): address review findings on unattended workflow recovery

- grant_validator validates the current persisted grant during locked
  admission, where the entry is not published yet, and switches to the
  live entry after publication; the previous entries-only lookup 403'd
  every unattended restore of a perfectly usable grant
- delegated restore waits for the AgentRuns host installation signal
  instead of racing the rest of Gateway startup and failing the scan
- cancellation defers grant revocation to the supervisor's post-cleanup
  save, keeping the delegated handle authorized to cancel children the
  recovered workflow already admitted
- the per-operation delegated guard rechecks the current account
  disablement, which token_version does not reflect
- restore stub revalidates per operation like the real host, with
  regression tests for all of the above; format the new fixture and
  condense the extension guidance within its budget

* fix(tests): cancel the recovered workflow on its own service

The fixture's action handlers are bound to the original service, which
stop() already made unavailable; the recovered record lives on the
replacement instance, so the cancel must go through replacement.cancel.

* fix(extensions): let a cancelled replayed child keep its cancelled outcome

The replay fence forced every errored re-dispatched step to
needs_reconciliation, including the deterministic case where the error
is the workflow stopping after its child was already cancelled and
observed. Fence only genuinely uncertain outcomes; a clean cancel of a
recovered workflow now ends cancelled like a fresh one.
This commit is contained in:
hataa 2026-11-18 10:46:20 +08:00 committed by GitHub
parent b6515b1415
commit c18df580f6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 655 additions and 13 deletions

View File

@ -53,6 +53,30 @@ logger = logging.getLogger(__name__)
# timeout.
_RUN_DRAIN_TIMEOUT_SECONDS = 5.0
# Upper bound (seconds) a delegated-restore call may wait for the AgentRuns
# host to be installed during Gateway startup. Startup normally signals within
# its first seconds; the bound only fails a stuck startup instead of leaving
# unattended recovery hanging, and per-record failures keep manual recovery.
_EXTENSION_RESTORE_READY_TIMEOUT_SECONDS = 60.0
async def _agent_runs_host_after_install(app, installed: asyncio.Event):
"""Resolve the AgentRuns host once startup has installed it.
Extension services start before the run-admission stack (thread store, run
manager, AgentRuns host) exists, and unattended restoration can run during
that window, so wait for the explicit installation signal instead of
failing the whole recovery scan.
"""
try:
await asyncio.wait_for(installed.wait(), _EXTENSION_RESTORE_READY_TIMEOUT_SECONDS)
except TimeoutError:
pass
host = getattr(app.state, "agent_runs_host", None)
if host is None:
raise RuntimeError("Agent run host unavailable")
return host
def _browser_tools_enabled_in_config(config: AppConfig) -> bool:
"""Return whether process-local agentic browser sessions are configured."""
@ -1020,6 +1044,13 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
# Register cleanup before starting: start() can partially acquire
# resources and then fail or be cancelled.
stack.push_async_callback(stop_extension_services)
agent_runs_host_installed = asyncio.Event()
async def restore_agent_runs_authority(*, user_id: str, grant: dict, validator=None, plugin_namespace: str | None = None):
host = await _agent_runs_host_after_install(app, agent_runs_host_installed)
return await host.restore_delegated_authority(user_id=user_id, grant=grant, validator=validator, plugin_namespace=plugin_namespace)
record_runtime_diagnostics(
await start_services(
extensions,
@ -1028,6 +1059,7 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
run_evidence_reader=app.state.run_evidence_reader,
host_capabilities=host_capabilities.bindings,
attempted_services=attempted_services,
agent_runs_restore=restore_agent_runs_authority,
)
)
@ -1125,6 +1157,9 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
app.state.agent_runs_host = GatewayAgentRunsHost(app, load_user=SQLiteUserRepository(sf).get_user_by_id if sf is not None else None)
stack.callback(app.state.agent_runs_host.close)
# Extension unattended recovery waits on this before restoring grants;
# past it, the whole run-admission stack above is ready.
agent_runs_host_installed.set()
# Start the lease heartbeat if enabled (multi-worker deployments).
await app.state.run_manager.start_heartbeat()

View File

@ -89,9 +89,43 @@ class GatewayAgentRunsHost:
raise AgentRunError(403, "Batch follow-up authorization revoked")
return _BoundAgentRuns(self, user_id, version, source, frozenset(permissions), origin=make_origin("subagent_batch"))
async def restore_delegated_authority(
self,
*,
user_id: str,
grant: dict,
validator: Callable[[], Awaitable[None]] | None = None,
plugin_namespace: str | None = None,
):
"""Restore a persisted delegated grant (e.g. workflow recovery) with its ceiling.
The grant is the mapping ``capture_authority`` returned at mint time
(token_version, source, permissions); expiry and revocation are the
caller's contract, enforced through ``validator`` on every operation
this handle performs. Restoring re-loads the user row and fails closed
on a changed token_version, needs_setup or a disabled account, exactly
like ``restore_batch_authority``; it never creates fresh authority.
"""
source, version, permissions = grant.get("source"), grant.get("token_version"), grant.get("permissions")
if source not in (AUTH_SOURCE_SESSION, AUTH_SOURCE_AUTH_DISABLED) or type(version) is not int or not isinstance(permissions, list) or any(not isinstance(value, str) for value in permissions):
raise AgentRunError(403, "Invalid delegated authority")
user = get_auth_disabled_user() if source == AUTH_SOURCE_AUTH_DISABLED and is_auth_disabled() else await self.load_user(user_id) if source == AUTH_SOURCE_SESSION and self.load_user is not None else None
if user is None or str(user.id) != user_id or user.token_version != version or user.needs_setup or getattr(user, "disabled", False):
raise AgentRunError(403, "Delegated authorization revoked")
return _BoundAgentRuns(
self,
user_id,
version,
source,
frozenset(permissions),
plugin_namespace=plugin_namespace,
origin=make_origin("extension", namespace=plugin_namespace),
delegation_validator=validator,
)
class _BoundAgentRuns:
def __init__(self, host, user_id: str, token_version: int, source: str, permissions: frozenset[str], plugin_namespace: str | None = None, origin: dict | None = None):
def __init__(self, host, user_id: str, token_version: int, source: str, permissions: frozenset[str], plugin_namespace: str | None = None, origin: dict | None = None, delegation_validator: Callable[[], Awaitable[None]] | None = None):
self._host = host
self._user_id = user_id
self._token_version = token_version
@ -99,6 +133,19 @@ class _BoundAgentRuns:
self._permissions = permissions
self._plugin_namespace = plugin_namespace
self._origin = origin
self._delegation_validator = delegation_validator
def capture_authority(self) -> dict | None:
"""Persistable restorable snapshot of this handle's existing authority.
Returns the mint-time ceiling (token_version, source, permissions) for
a host-originated session handle, or None when this handle's authority
is not restorable. The host never validates the caller here: capture
only reflects authority the live handle already holds.
"""
if not self._host.active or self._source not in (AUTH_SOURCE_SESSION, AUTH_SOURCE_AUTH_DISABLED):
return None
return {"token_version": self._token_version, "source": self._source, "permissions": sorted(self._permissions)}
def for_plugin(self, namespace: str):
if not isinstance(namespace, str) or not namespace or len(namespace) > 96 or ":" in namespace:
@ -107,7 +154,7 @@ class _BoundAgentRuns:
if namespace != self._plugin_namespace:
raise AgentRunError(422, "Agent run handle is already bound to a plugin")
return self
return _BoundAgentRuns(self._host, self._user_id, self._token_version, self._source, self._permissions, namespace)
return _BoundAgentRuns(self._host, self._user_id, self._token_version, self._source, self._permissions, namespace, self._origin, self._delegation_validator)
@asynccontextmanager
async def _request(self, permission: str, thread_id: str | None = None):
@ -117,8 +164,15 @@ class _BoundAgentRuns:
user = get_auth_disabled_user() if is_auth_disabled() else None
else:
user = await self._host.load_user(self._user_id) if self._host.load_user is not None else None
if user is None or str(user.id) != self._user_id or user.token_version != self._token_version or user.needs_setup:
if user is None or str(user.id) != self._user_id or user.token_version != self._token_version or user.needs_setup or getattr(user, "disabled", False):
raise AgentRunError(403, "Agent run authorization revoked")
if self._delegation_validator is not None:
try:
await self._delegation_validator()
except AgentRunError:
raise
except Exception as exc:
raise AgentRunError(403, "Delegated authorization revoked") from exc
permissions = self._permissions.intersection(await resolve_route_permissions(user, is_internal=False))
if permission not in permissions:
raise AgentRunError(403, f"Permission denied: {permission}")

View File

@ -76,6 +76,18 @@ class AgentRuns(Protocol):
"""
...
def capture_authority(self) -> Mapping[str, Any] | None:
"""Persistable snapshot of this handle's existing authority, or None.
Returns the mint-time ceiling (token_version, source, permissions) of
the live session handle so a service can restore equivalent authority
later through the host's delegated-restore capability. Capture only
reflects authority the handle already holds; it is not a credential to
serialize across hosts, and None means this host or handle cannot be
restored. Hosts that predate this method lack it entirely.
"""
...
async def send_input(self, *, thread_id: str, run_id: str, text: str, mode: str, request_id: str, context: Mapping[str, Any] | None = None) -> dict[str, Any]:
"""Queue a new turn or steer the active Run; never change its model."""
...

View File

@ -251,6 +251,12 @@ class ExtensionRuntimeDeps:
model_invoker: ModelInvoker | None = None
completed_run_evidence: CompletedRunEvidenceReader | None = None
skill_mutations: SkillMutationService | None = None
# Async callable (user_id, grant, *, validator=None, plugin_namespace=None)
# -> AgentRuns restoring a captured session ceiling for unattended host-side
# recovery. The host revalidates the user row per operation and fails closed;
# expiry/revocation stay the extension's contract via the validator. None on
# hosts without the delegated-restore capability.
agent_runs_restore: Any | None = None
class ExtensionService(Protocol):

View File

@ -101,6 +101,7 @@ hook directory, so they never run a developer's or CI's Git hooks.
cannot collide.
- **`configuration_revision` never exposes raw configuration.** It returns an HMAC.
Sets hash the same way across process hash seeds; ordered lists keep their order.
- **Restorable authority is captured, never created.** Restores revalidate it.
## Extension-first exceptions

View File

@ -581,6 +581,7 @@ async def start_services(
run_evidence_reader: Any | None = None,
host_capabilities: dict[str, tuple[Any | None, Any | None]] | None = None,
attempted_services: list[tuple[str, Any]] | None = None,
agent_runs_restore: Any | None = None,
) -> list[Diagnostic]:
"""Start extension services in registration order, failing open per item."""
diagnostics: list[Diagnostic] = []
@ -592,6 +593,7 @@ async def start_services(
policy=project_host_policy(app_config),
session_factory=session_factory,
run_evidence_reader=run_evidence_reader,
agent_runs_restore=agent_runs_restore,
)
source_deps = {}
for entry in extensions.services:

View File

@ -71,6 +71,21 @@ async def test_handle_rechecks_user_and_shutdown_before_dispatch(host, monkeypat
assert handler.await_count == 1
@pytest.mark.asyncio
async def test_handle_rechecks_current_account_disablement(host, monkeypatch):
control, request, load_user = host
runs = control.bind(request)
handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a"))
monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler)
assert await runs.create_thread() == "thread-a"
# Disabling the account leaves token_version unchanged, so the per-operation
# guard must observe the current disabled state, not only the mint-time one.
load_user.return_value = request.state.user.model_copy(update={"disabled": True})
with pytest.raises(AgentRunError, match="revoked"):
await runs.create_thread()
assert handler.await_count == 1
@pytest.mark.asyncio
async def test_permissions_are_intersected_with_original_grant_and_current_policy(host, monkeypatch):
control, request, _ = host
@ -667,3 +682,97 @@ async def test_configuration_revision_canonicalizes_sets_but_preserves_list_orde
assert before == await runs.configuration_revision(assistant_ids=["lead_agent"], durable=True)
data["tools"].reverse()
assert before != await runs.configuration_revision(assistant_ids=["lead_agent"], durable=True)
@pytest.mark.asyncio
async def test_capture_authority_snapshots_the_session_ceiling(host, monkeypatch):
control, request, _ = host
runs = control.bind(request)
handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a"))
monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler)
assert await runs.create_thread() == "thread-a"
captured = runs.capture_authority()
assert captured == {
"token_version": request.state.user.token_version,
"source": "session",
"permissions": sorted(request.state.auth.permissions),
}
@pytest.mark.asyncio
async def test_restore_delegated_authority_enforces_validator_per_operation(host, monkeypatch):
control, request, load_user = host
handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a"))
monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler)
calls = []
async def validator():
calls.append(1)
if len(calls) > 1:
raise PermissionError("grant revoked by the owner")
runs = await control.restore_delegated_authority(
user_id=str(request.state.user.id),
grant={"token_version": request.state.user.token_version, "source": "session", "permissions": ["threads:write"]},
validator=validator,
plugin_namespace="workflows",
)
assert await runs.create_thread() == "thread-a"
with pytest.raises(AgentRunError, match="Delegated authorization revoked"):
await runs.create_thread()
assert handler.await_count == 1
@pytest.mark.asyncio
async def test_restore_delegated_authority_maps_validator_errors_to_403(host, monkeypatch):
control, request, _ = host
handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a"))
monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler)
async def validator():
raise AgentRunError(403, "Workflow recovery grant expired or revoked")
runs = await control.restore_delegated_authority(
user_id=str(request.state.user.id),
grant={"token_version": request.state.user.token_version, "source": "session", "permissions": ["threads:write"]},
validator=validator,
)
with pytest.raises(AgentRunError, match="grant expired or revoked"):
await runs.create_thread()
assert handler.await_count == 0
@pytest.mark.asyncio
async def test_restore_delegated_authority_rejects_stale_grants(host):
control, request, load_user = host
user = request.state.user
load_user.return_value = user.model_copy(update={"token_version": user.token_version + 1})
with pytest.raises(AgentRunError, match="Delegated authorization revoked"):
await control.restore_delegated_authority(
user_id=str(user.id),
grant={"token_version": user.token_version, "source": "session", "permissions": ["threads:write"]},
)
with pytest.raises(AgentRunError, match="Invalid delegated authority"):
await control.restore_delegated_authority(
user_id=str(user.id),
grant={"token_version": user.token_version, "source": "pat", "permissions": ["threads:write"]},
)
with pytest.raises(AgentRunError, match="Invalid delegated authority"):
await control.restore_delegated_authority(
user_id=str(user.id),
grant={"token_version": user.token_version, "source": "session", "permissions": [1]},
)
@pytest.mark.asyncio
async def test_restored_delegated_handle_is_plugin_scoped_with_extension_origin(host):
control, request, _ = host
runs = await control.restore_delegated_authority(
user_id=str(request.state.user.id),
grant={"token_version": request.state.user.token_version, "source": "session", "permissions": ["threads:write"]},
plugin_namespace="workflows",
)
assert runs.for_plugin("workflows") is runs
assert runs._origin == {"kind": "extension", "namespace": "workflows"}
with pytest.raises(AgentRunError, match="already bound"):
runs.for_plugin("other")

View File

@ -957,3 +957,24 @@ def test_lifespan_detaches_the_outbox_when_the_delivery_worker_cannot_start() ->
# Startup survives, and nothing keeps enqueueing rows that no worker would send.
scheduled_service.detach_notification_outbox.assert_called_once_with()
@pytest.mark.asyncio
async def test_delegated_restore_waits_for_agent_runs_host_installation(monkeypatch):
from app.gateway.deps import _agent_runs_host_after_install
app = SimpleNamespace(state=SimpleNamespace())
installed = asyncio.Event()
# Extension services start before the AgentRuns host exists, so a restore
# issued during that window waits for the installation signal instead of
# failing the unattended recovery scan outright.
waiter = asyncio.create_task(_agent_runs_host_after_install(app, installed))
assert not waiter.done()
app.state.agent_runs_host = object()
installed.set()
assert await asyncio.wait_for(waiter, 1) is app.state.agent_runs_host
# A startup that never installs the host fails closed per record within
# the bounded budget; manual recovery stays available.
monkeypatch.setattr("app.gateway.deps._EXTENSION_RESTORE_READY_TIMEOUT_SECONDS", 0.01)
with pytest.raises(RuntimeError, match="Agent run host unavailable"):
await _agent_runs_host_after_install(SimpleNamespace(state=SimpleNamespace()), asyncio.Event())

View File

@ -41,6 +41,11 @@ class Runs:
revision = "initial"
def capture_authority(self):
if self.denied:
return None
return {"token_version": 0, "source": "session", "permissions": ["runs:create", "threads:write"]}
async def configuration_evidence(self, *, thread_id, run_id):
return {"run_id": run_id, "thread_id": thread_id, "source": "host_assembly", "version": "synthetic-assembly", "model": "test-model"}
@ -1162,3 +1167,255 @@ agent({stepId:args.second,agentRef:'lead_agent',task:'read',artifacts:[f.artifac
assert len(result["steps"]) <= 2
assert len(runs.starts) <= 2
assert len({s["step_id"] for s in result["steps"]}) == len(result["steps"])
class ValidatingRuns:
"""Wraps every delegated operation in the host's per-operation validator gate."""
def __init__(self, runs, validator):
self._runs = runs
self._validator = validator
def __getattr__(self, name):
attribute = getattr(self._runs, name)
if not callable(attribute):
return attribute
async def operation(*args, **kwargs):
await self._validator()
return await attribute(*args, **kwargs)
return operation
class Restorer:
"""Host-side delegated restore stub; the real host revalidates the grant on
every operation of the returned handle, including before publication."""
def __init__(self, runs):
self.runs = runs
self.calls = []
async def __call__(self, *, user_id, grant, validator=None, plugin_namespace=None):
self.calls.append((user_id, grant, plugin_namespace))
return self.runs if validator is None else ValidatingRuns(self.runs, validator)
async def crash_like_record(service, run_id, owner="alice"):
"""What a SIGKILL leaves: recoverable, children back at their queued window."""
record = await asyncio.to_thread(service.store.get, owner, run_id)
record["status"] = "recoverable"
record["error"] = None
for row in record["steps"]:
if row["status"] in ("queued", "creating_thread", "admitting", "running", "cancelled"):
row["status"] = "queued"
row["run_id"] = None
row.pop("error", None)
# A graceful stop records an error response for the in-flight frame; a real
# crash loses it, leaving the frame unanswered for replay to re-dispatch.
for item in record["journal"]:
item["responses"] = None
await asyncio.to_thread(service.store.put, owner, record)
return record
@pytest.mark.asyncio
async def test_unattended_recovery_requires_session_opt_in_and_capture(plugin):
from dataclasses import replace
_, actions, service, runs = plugin
with pytest.raises(ValueError, match="Unattended recovery requires a session-authenticated launch"):
await actions["launch"]({"request_id": "a", "script": "return 1;", "args": {}, "unattended_recovery": True}, context(runs))
with pytest.raises(ValueError, match="unattended_recovery must be boolean"):
await actions["launch"]({"request_id": "b", "script": "return 1;", "args": {}, "unattended_recovery": "yes"}, replace(context(runs), interaction="session"))
session = replace(context(runs), interaction="session")
plain = await actions["launch"]({"request_id": "c", "script": "return 1;"}, session)
assert (await asyncio.to_thread(service.store.get, "alice", plain["run_id"]))["grant"] is None
granted = await actions["launch"]({"request_id": "d", "script": "return 1;", "unattended_recovery": True}, session)
record = await asyncio.to_thread(service.store.get, "alice", granted["run_id"])
assert record["grant"] == {
"authority": {"token_version": 0, "source": "session", "permissions": ["runs:create", "threads:write"]},
"expires_at": pytest.approx(time.time() + service.config["pause_timeout_seconds"], abs=5),
"revoked": False,
"version": 1,
}
# The grant is authority metadata and never projected to API responses.
assert "grant" not in await actions["get"]({"run_id": granted["run_id"]}, context(runs))
@pytest.mark.asyncio
async def test_unattended_recovery_resumes_crashed_workflow_to_child_results(plugin):
from dataclasses import replace
from types import SimpleNamespace
from deerflow_extension_workflows.constants import NAMESPACE
from deerflow_extension_workflows.service import Workflows
_, actions, service, runs = plugin
script = "const a = await agent({stepId:'a',agentRef:'lead_agent',task:'blocked'});const b = await agent({stepId:'b',agentRef:'lead_agent',task:'second'});return b.output;"
launched = await actions["launch"]({"request_id": "crash", "script": script, "unattended_recovery": True}, replace(context(runs), interaction="session"))
run_id = launched["run_id"]
# Freeze the crash window: the step row is saved queued, admission never starts.
gate = asyncio.Event()
original_create = runs.create_thread
async def gated_create(**kwargs):
await gate.wait()
return await original_create(**kwargs)
runs.create_thread = gated_create
for _ in range(500):
record = await asyncio.to_thread(service.store.get, "alice", run_id)
if record["steps"]:
break
await asyncio.sleep(0.01)
gate.set()
await service.stop()
runs.create_thread = original_create
record = await crash_like_record(service, run_id)
# Nothing was truly admitted before the crash; drop the interrupted stand-ins.
stale_thread = record["steps"][0]["thread_id"]
runs.runs = {key: value for key, value in runs.runs.items() if value.thread_id != stale_thread}
restorer = Restorer(runs)
replacement = Workflows(service.config)
await replacement.start(SimpleNamespace(agent_runs_restore=restorer))
try:
for _ in range(500):
if run_id in replacement.entries:
break
await asyncio.sleep(0.01)
assert run_id in replacement.entries, "unattended recovery never resumed the workflow"
runs.release.set()
await asyncio.wait_for(replacement.wait_idle(run_id), 10)
final = await replacement.get({"run_id": run_id}, context(runs))
assert final["status"] == "completed", (final.get("error"), [(s["step_id"], s["status"], s.get("error")) for s in final["steps"]])
# The unadmitted child was re-dispatched with its original identity and key.
assert final["steps"][0]["attempt"] == 1
assert final["steps"][0]["idempotency_key"] == record["steps"][0]["idempotency_key"]
assert final["steps"][0]["thread_id"] == record["steps"][0]["thread_id"]
assert final["result"] == {"echo": "second"}
assert restorer.calls == [("alice", {"token_version": 0, "source": "session", "permissions": ["runs:create", "threads:write"]}, NAMESPACE)]
finally:
gate.set()
runs.release.set()
await replacement.stop()
@pytest.mark.asyncio
async def test_cancelling_a_recovered_workflow_keeps_child_cleanup_authorized(plugin):
from dataclasses import replace
from types import SimpleNamespace
from deerflow_extension_workflows.service import Workflows
_, actions, service, runs = plugin
launched = await actions["launch"](
{"request_id": "recancel", "script": "return await agent({stepId:'a',agentRef:'lead_agent',task:'blocked'});", "unattended_recovery": True},
replace(context(runs), interaction="session"),
)
run_id = launched["run_id"]
# Freeze the crash window: the step row is saved queued, admission never starts.
gate = asyncio.Event()
original_create = runs.create_thread
async def gated_create(**kwargs):
await gate.wait()
return await original_create(**kwargs)
runs.create_thread = gated_create
for _ in range(500):
record = await asyncio.to_thread(service.store.get, "alice", run_id)
if record["steps"]:
break
await asyncio.sleep(0.01)
gate.set()
await service.stop()
runs.create_thread = original_create
record = await crash_like_record(service, run_id)
stale_thread = record["steps"][0]["thread_id"]
runs.runs = {key: value for key, value in runs.runs.items() if value.thread_id != stale_thread}
replacement = Workflows(service.config)
await replacement.start(SimpleNamespace(agent_runs_restore=Restorer(runs)))
try:
for _ in range(500):
if runs.entered.is_set() and replacement.entries.get(run_id) is not None:
break
await asyncio.sleep(0.01)
assert runs.entered.is_set(), "recovered workflow never re-admitted its child"
# The delegated handle is the only authorized path to the running child;
# revoking before cleanup would strand it in needs_reconciliation.
# Cancel on the replacement service: the fixture's actions are bound to
# the original, which stop() already made unavailable.
assert (await replacement.cancel({"run_id": run_id}, context(runs)))["status"] == "cancelling"
await asyncio.wait_for(replacement.wait_idle(run_id), 10)
final = await replacement.get({"run_id": run_id}, context(runs))
assert final["status"] == "cancelled", (final.get("error"), [(s["step_id"], s["status"], s.get("error")) for s in final["steps"]])
assert final["steps"][0]["status"] == "cancelled", final["steps"][0]
persisted = await asyncio.to_thread(replacement.store.get, "alice", run_id)
assert persisted["grant"]["revoked"] is True
finally:
gate.set()
runs.release.set()
await replacement.stop()
@pytest.mark.asyncio
async def test_revoked_or_expired_grants_stay_on_the_manual_path(plugin):
from dataclasses import replace
from types import SimpleNamespace
from deerflow_extension_workflows.service import Workflows
_, actions, service, runs = plugin
service.config["pause_timeout_seconds"] = 1
launched = await actions["launch"]({"request_id": "paused", "script": "await pause({stepId:'p'});", "unattended_recovery": True}, replace(context(runs), interaction="session"))
run_id = launched["run_id"]
await waiting(plugin, run_id, "paused")
await service.stop()
await crash_like_record(service, run_id)
# Revoked: the owner's cancellation withdrew the standing authority.
record = await asyncio.to_thread(service.store.get, "alice", run_id)
record["grant"]["revoked"] = True
await asyncio.to_thread(service.store.put, "alice", record)
restorer = Restorer(runs)
replacement = Workflows(service.config)
await replacement.start(SimpleNamespace(agent_runs_restore=restorer))
try:
await asyncio.sleep(0.05)
assert not restorer.calls and run_id not in replacement.entries
assert (await replacement.get({"run_id": run_id}, context(runs)))["status"] == "recoverable"
# Manual session recovery is unaffected by the unusable grant.
recovered = await replacement.recover({"run_id": run_id}, replace(context(runs), interaction="session"))
assert recovered["status"] == "queued"
finally:
await replacement.stop()
@pytest.mark.asyncio
async def test_expired_grant_never_restores_authority(plugin):
from dataclasses import replace
from types import SimpleNamespace
from deerflow_extension_workflows.service import Workflows
_, actions, service, runs = plugin
service.config["pause_timeout_seconds"] = 1
launched = await actions["launch"]({"request_id": "stale", "script": "await pause({stepId:'p'});", "unattended_recovery": True}, replace(context(runs), interaction="session"))
run_id = launched["run_id"]
await waiting(plugin, run_id, "paused")
await service.stop()
await crash_like_record(service, run_id)
await asyncio.sleep(1.1) # the bounded mint-time TTL lapses
restorer = Restorer(runs)
replacement = Workflows(service.config)
await replacement.start(SimpleNamespace(agent_runs_restore=restorer))
try:
await asyncio.sleep(0.05)
assert not restorer.calls and run_id not in replacement.entries
assert (await replacement.get({"run_id": run_id}, context(runs)))["status"] == "recoverable"
finally:
await replacement.stop()

View File

@ -399,8 +399,32 @@ and is distinct from this configuration preflight check.
Phase 3 and Phase 4 share the design: stable step/attempt and idempotency identities,
conditional record revisions, a durable intent/result journal, and fresh authority
support the leases and fenced dispatch in Phase 4 below. SQLite retains the Phase 3
single-owner contract and exclusive process lock. Both modes require explicit
session authorization for recovery; neither reconstructs credentials from records.
single-owner contract and exclusive process lock. Neither mode reconstructs
credentials from records; recovery authority comes from a live session or, on the
single-Gateway SQLite store, from the opt-in grant below.
### Opt-in unattended recovery after restart
A workflow launched from an authenticated browser session may set
`unattended_recovery: true`. The launch then captures the session's existing
authority ceiling (token version, source, permission set) into a grant on the
workflow record, bounded by `pause_timeout_seconds` from launch time. Nothing is
captured for internal, PAT or model-tool launches; those never mint a grant.
After a Gateway restart, the single-Gateway SQLite store scans `recoverable`
records and resumes the ones holding an unexpired, unrevoked grant without a
sign-in: journal replay, completed-step reuse and the uncertain-admission fence
behave exactly as in the explicit browser flow above. Clustered (PostgreSQL)
stores keep the manual recovery path. Cancellation revokes the grant; the next
delegated operation fails closed.
The host re-validates the owner's user row on every delegated operation, keeps
the ceiling intersected with the owner's current route permissions, and re-checks
expiry and revocation per operation. Restoring a grant never creates authority
the owner did not have at launch: a demoted, disabled or reset account stops
recovered work at its next operation. Unfinished child effects and
`needs_reconciliation` outcomes remain manual in every mode.
## Phase 4: shared PostgreSQL deployment

View File

@ -1,16 +1,24 @@
"""Versioned bridge journal replay; authority is reacquired from a live session.
"""Versioned bridge journal replay; authority is reacquired from a live session
or, when the owner minted one at launch, from the persisted recovery grant.
SQLite retains its exclusive store owner. PostgreSQL uses a shared execution
lease and fenced admission; both replay the same committed bridge journal.
Unattended grant recovery runs on the single-Gateway SQLite store only.
"""
import asyncio
import copy
import json
import logging
import time
from deerflow_extension_api import AgentRunError
from deerflow_extension_api.plugins import ToolContext
from .constants import NAMESPACE
RECOVERY_VERSION = 1
logger = logging.getLogger(__name__)
class Recovery:
@ -25,11 +33,15 @@ class Recovery:
return await asyncio.shield(task)
async def restore(self, payload, context):
from .service import identity, text
from .service import text
run_id = text(payload["run_id"], 64)
return await self.restore_body(self.owner(context), run_id, context.agent_runs)
async def restore_body(self, owner, run_id, runs):
from .service import identity
async with self.launch_lock:
owner = self.owner(context)
run_id = text(payload["run_id"], 64)
record = await asyncio.to_thread(self.store.get, owner, run_id)
if run_id in self.entries:
return {"run_id": run_id, "status": record["status"]}
@ -41,11 +53,11 @@ class Recovery:
raise ValueError("Recovery script or arguments changed")
if len(self.entries) >= self.config["max_workflows"] or sum(e["owner"] == owner for e in self.entries.values()) >= self.config["max_workflows_per_owner"]:
raise ValueError("Active workflow limit")
entry = self.new_entry(owner, record, context.agent_runs)
entry = self.new_entry(owner, record, runs)
await self.check_configuration(entry)
if self.clustered:
record = await asyncio.to_thread(self.store.claim, owner, run_id)
entry = self.new_entry(owner, record, context.agent_runs)
entry = self.new_entry(owner, record, runs)
# A committed admission intent with no receipt is never dispatched
# again. An operator must reconcile its external effects first.
uncertain = [s for s in record["steps"] if s["status"] not in ("queued", "completed", "running") or (s["status"] == "running" and not s.get("run_id"))]
@ -64,6 +76,52 @@ class Recovery:
self.publish(entry)
return {"run_id": run_id, "status": "queued"}
def grant_usable(self, record):
grant = record.get("grant")
return (
isinstance(grant, dict) and grant.get("version") == 1 and grant.get("revoked") is not True and isinstance(grant.get("expires_at"), (int, float)) and grant["expires_at"] > time.time() and isinstance(grant.get("authority"), dict)
)
def grant_validator(self, owner, run_id):
async def validate():
entry = self.entries.get(run_id)
if entry is not None:
grant = entry["record"].get("grant")
else:
# Locked admission publishes the entry only after restore, while
# the delegated handle already operates; validate the current
# persisted grant until publication switches to the live entry.
record = await asyncio.to_thread(self.store.get, owner, run_id)
grant = record.get("grant") if isinstance(record, dict) else None
if not isinstance(grant, dict) or grant.get("revoked") is True or grant.get("expires_at", 0) <= time.time():
raise AgentRunError(403, "Workflow recovery grant expired or revoked")
return validate
async def recover_granted_workflows(self):
"""Resume opt-in recoverable workflows after a restart, without a session.
Records without a usable grant stay on the manual sign-in path; per-record
failures never block the others or Gateway startup.
"""
if self.agent_runs_restore is None:
return
try:
records = await asyncio.to_thread(self.store.recoverable)
except Exception:
logger.exception("Workflow grant scan failed; unattended recovery unavailable")
return
for owner, run_id, record in records:
if not self.available:
return
if run_id in self.entries or not self.grant_usable(record):
continue
try:
runs = await self.agent_runs_restore(user_id=owner, grant=record["grant"]["authority"], validator=self.grant_validator(owner, run_id), plugin_namespace=NAMESPACE)
await self.restore_body(owner, run_id, runs)
except Exception:
logger.exception("Workflow %s unattended recovery failed; manual recovery remains available", run_id)
async def begin_frame(self, entry, frame):
"""Persist intent before effects; replay verifies every emitted request."""
record = entry["record"]

View File

@ -116,6 +116,10 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
self.write_lock = asyncio.Lock()
self.global_slots = asyncio.Semaphore(config["global_concurrency"])
self.lockfile = None
# Host capability restoring captured session authority; None on hosts
# without delegated restore (unattended recovery then stays manual).
self.agent_runs_restore = None
self.recovery_scan = None
async def start(self, deps):
def initialize():
@ -157,6 +161,7 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
raise
self.lockfile, self.store, self.runtime_version = await asyncio.to_thread(initialize)
self.agent_runs_restore = getattr(deps, "agent_runs_restore", None)
if self.clustered and deps is not None:
from sqlalchemy import text as sql_text
@ -169,10 +174,17 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
if host_identity != await asyncio.to_thread(self.store.database_identity):
raise ValueError("Workflow and host must share the same PostgreSQL database and schema")
self.available = True
if not self.clustered:
# Single-Gateway stores may hold recoverable records whose owner
# minted an unattended-recovery grant before the restart. Clustered
# stores coordinate through leases and stay on the manual path.
self.recovery_scan = asyncio.create_task(self.recover_granted_workflows())
async def stop(self):
async with self.launch_lock:
self.available = False
scan = self.recovery_scan
self.recovery_scan = None
for entry in self.entries.values():
entry["suspending"] = entry["record"]["status"] != "cancelling"
entry["cancel"].set()
@ -183,6 +195,14 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
await asyncio.gather(*list(self.admissions), return_exceptions=True)
if workers:
await asyncio.gather(*workers, return_exceptions=True)
if scan is not None:
scan.cancel()
try:
await scan
except asyncio.CancelledError:
pass
except Exception:
logger.exception("Workflow recovery scan failed during shutdown")
if self.heartbeats:
heartbeats = list(self.heartbeats.values())
for heartbeat in heartbeats:
@ -203,6 +223,16 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
raise ValueError("Workflow service unavailable or disabled")
return context.principal.user_id
@staticmethod
def _revoke_grant(record):
# Withdraw the owner's standing recovery authority with the record, so
# the next delegated operation fails closed. A live supervisor defers
# this until its cleanup of already-admitted children finishes, since
# that cleanup is itself an authorized use of the handle.
grant = record.get("grant")
if isinstance(grant, dict):
grant["revoked"] = True
async def save(self, owner, record):
# SQLite commits survive cancellation of to_thread. Keep the lock and
# observe the new revision before propagating cancellation to cleanup.
@ -234,8 +264,23 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
owner = self.owner(context)
if context.agent_runs is None:
raise ValueError("Session-bound AgentRuns required; PAT/internal callers unsupported")
if set(payload) - {"request_id", "script", "args", "definition"}:
if set(payload) - {"request_id", "script", "args", "definition", "unattended_recovery"}:
raise ValueError("Unexpected launch fields")
unattended = payload.get("unattended_recovery", False)
if type(unattended) is not bool:
raise ValueError("unattended_recovery must be boolean")
grant = None
if unattended:
capture = getattr(context.agent_runs, "capture_authority", None)
captured = capture() if callable(capture) else None
if context.interaction != "session" or captured is None:
raise ValueError("Unattended recovery requires a session-authenticated launch")
grant = {
"authority": captured,
"expires_at": time.time() + self.config["pause_timeout_seconds"],
"revoked": False,
"version": 1,
}
definition = payload.get("definition")
if definition is not None:
if "script" in payload or not isinstance(definition, dict):
@ -299,6 +344,7 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
"steps": [],
"logs": [],
"result": None,
"grant": grant,
}
# SQLite threads survive caller cancellation. Retain a service-owned task
# through commit AND supervisor publication, shielding only the caller's wait.
@ -401,12 +447,18 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
entry["suspending"] = False
if entry["record"]["status"] not in TERMINAL:
entry["record"]["status"] = "cancelling"
# Defer revocation to the supervisor's cleanup: the delegated
# handle is the authorized path that cancels children this
# workflow already admitted, and revoking now would 403 that
# cleanup into needs_reconciliation.
entry["revoke_grant_on_finish"] = True
await self.save(owner, entry["record"])
return {"run_id": run_id, "status": "cancelling"}
if record["status"] == "recoverable":
if context.agent_runs is None:
raise ValueError("Session-bound AgentRuns required")
record["status"] = "cancelling"
self._revoke_grant(record)
await self.save(owner, record)
for row in record["steps"]:
if row.get("run_id") and row["status"] != "completed":
@ -636,7 +688,10 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
row["status"] = "failed"
response["error"] = str(exc)[:240] if isinstance(exc, ValueError) else "Agent step failed"
finally:
if replay and "error" in response:
if replay and "error" in response and not (entry["cancel"].is_set() and row is not None and row["status"] == "cancelled"):
# A replayed step error is fenced as uncertain — unless the
# error is the workflow stopping while its re-dispatched child
# was already cancelled and observed, which is deterministic.
entry["fault"] = "Recovered child outcome requires reconciliation"
if row:
row["status"] = "needs_reconciliation"
@ -829,6 +884,10 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
cleanup = [self.cancel_child(runs, row, **({"record": record} if self.clustered else {})) for row in record["steps"] if row.get("run_id") and row["status"] not in ("completed", "waiting_input", "cancelled")]
if cleanup:
await asyncio.gather(*cleanup)
if entry.pop("revoke_grant_on_finish", False):
# Cleanup above was the last authorized user of the delegated
# handle; withdraw the persisted grant before the final save.
self._revoke_grant(record)
if self.clustered:
for row in record["steps"]:
if self.slot_releasable(row):

View File

@ -220,6 +220,10 @@ class Store:
)
]
def recoverable(self):
with closing(self.connect()) as db:
return [(row[0], row[1], json.loads(row[2])) for row in db.execute("SELECT owner,id,data FROM workflows WHERE status='recoverable'").fetchall()]
def delete(self, owner, run_id):
with closing(self.connect()) as db, db:
row = db.execute("SELECT status FROM workflows WHERE owner=? AND id=?", (owner, run_id)).fetchone()