mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
feat(extensions): recover opt-in workflows unattended after restart (#8289)
* feat(extensions): recover opt-in workflows unattended after restart Workflow launches from an authenticated browser session may opt in to unattended recovery. The launch captures the session's existing authority ceiling into a bounded, revocable grant on the workflow record; after a Gateway restart the single-Gateway SQLite store resumes recoverable records holding a usable grant through the existing journal-replay path, without a sign-in. Host: restore_delegated_authority (sibling of restore_batch_authority) plus capture_authority on bound handles; the per-operation chokepoint runs an extension-supplied grant validator so expiry and revocation fail closed on every operation, including artifact reads. Clustered stores and uncertain admissions stay on the manual path. * fix(extensions): address review findings on unattended workflow recovery - grant_validator validates the current persisted grant during locked admission, where the entry is not published yet, and switches to the live entry after publication; the previous entries-only lookup 403'd every unattended restore of a perfectly usable grant - delegated restore waits for the AgentRuns host installation signal instead of racing the rest of Gateway startup and failing the scan - cancellation defers grant revocation to the supervisor's post-cleanup save, keeping the delegated handle authorized to cancel children the recovered workflow already admitted - the per-operation delegated guard rechecks the current account disablement, which token_version does not reflect - restore stub revalidates per operation like the real host, with regression tests for all of the above; format the new fixture and condense the extension guidance within its budget * fix(tests): cancel the recovered workflow on its own service The fixture's action handlers are bound to the original service, which stop() already made unavailable; the recovered record lives on the replacement instance, so the cancel must go through replacement.cancel. * fix(extensions): let a cancelled replayed child keep its cancelled outcome The replay fence forced every errored re-dispatched step to needs_reconciliation, including the deterministic case where the error is the workflow stopping after its child was already cancelled and observed. Fence only genuinely uncertain outcomes; a clean cancel of a recovered workflow now ends cancelled like a fresh one.
This commit is contained in:
parent
b6515b1415
commit
c18df580f6
@ -53,6 +53,30 @@ logger = logging.getLogger(__name__)
|
||||
# timeout.
|
||||
_RUN_DRAIN_TIMEOUT_SECONDS = 5.0
|
||||
|
||||
# Upper bound (seconds) a delegated-restore call may wait for the AgentRuns
|
||||
# host to be installed during Gateway startup. Startup normally signals within
|
||||
# its first seconds; the bound only fails a stuck startup instead of leaving
|
||||
# unattended recovery hanging, and per-record failures keep manual recovery.
|
||||
_EXTENSION_RESTORE_READY_TIMEOUT_SECONDS = 60.0
|
||||
|
||||
|
||||
async def _agent_runs_host_after_install(app, installed: asyncio.Event):
|
||||
"""Resolve the AgentRuns host once startup has installed it.
|
||||
|
||||
Extension services start before the run-admission stack (thread store, run
|
||||
manager, AgentRuns host) exists, and unattended restoration can run during
|
||||
that window, so wait for the explicit installation signal instead of
|
||||
failing the whole recovery scan.
|
||||
"""
|
||||
try:
|
||||
await asyncio.wait_for(installed.wait(), _EXTENSION_RESTORE_READY_TIMEOUT_SECONDS)
|
||||
except TimeoutError:
|
||||
pass
|
||||
host = getattr(app.state, "agent_runs_host", None)
|
||||
if host is None:
|
||||
raise RuntimeError("Agent run host unavailable")
|
||||
return host
|
||||
|
||||
|
||||
def _browser_tools_enabled_in_config(config: AppConfig) -> bool:
|
||||
"""Return whether process-local agentic browser sessions are configured."""
|
||||
@ -1020,6 +1044,13 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
|
||||
# Register cleanup before starting: start() can partially acquire
|
||||
# resources and then fail or be cancelled.
|
||||
stack.push_async_callback(stop_extension_services)
|
||||
|
||||
agent_runs_host_installed = asyncio.Event()
|
||||
|
||||
async def restore_agent_runs_authority(*, user_id: str, grant: dict, validator=None, plugin_namespace: str | None = None):
|
||||
host = await _agent_runs_host_after_install(app, agent_runs_host_installed)
|
||||
return await host.restore_delegated_authority(user_id=user_id, grant=grant, validator=validator, plugin_namespace=plugin_namespace)
|
||||
|
||||
record_runtime_diagnostics(
|
||||
await start_services(
|
||||
extensions,
|
||||
@ -1028,6 +1059,7 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
|
||||
run_evidence_reader=app.state.run_evidence_reader,
|
||||
host_capabilities=host_capabilities.bindings,
|
||||
attempted_services=attempted_services,
|
||||
agent_runs_restore=restore_agent_runs_authority,
|
||||
)
|
||||
)
|
||||
|
||||
@ -1125,6 +1157,9 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
|
||||
|
||||
app.state.agent_runs_host = GatewayAgentRunsHost(app, load_user=SQLiteUserRepository(sf).get_user_by_id if sf is not None else None)
|
||||
stack.callback(app.state.agent_runs_host.close)
|
||||
# Extension unattended recovery waits on this before restoring grants;
|
||||
# past it, the whole run-admission stack above is ready.
|
||||
agent_runs_host_installed.set()
|
||||
|
||||
# Start the lease heartbeat if enabled (multi-worker deployments).
|
||||
await app.state.run_manager.start_heartbeat()
|
||||
|
||||
@ -89,9 +89,43 @@ class GatewayAgentRunsHost:
|
||||
raise AgentRunError(403, "Batch follow-up authorization revoked")
|
||||
return _BoundAgentRuns(self, user_id, version, source, frozenset(permissions), origin=make_origin("subagent_batch"))
|
||||
|
||||
async def restore_delegated_authority(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
grant: dict,
|
||||
validator: Callable[[], Awaitable[None]] | None = None,
|
||||
plugin_namespace: str | None = None,
|
||||
):
|
||||
"""Restore a persisted delegated grant (e.g. workflow recovery) with its ceiling.
|
||||
|
||||
The grant is the mapping ``capture_authority`` returned at mint time
|
||||
(token_version, source, permissions); expiry and revocation are the
|
||||
caller's contract, enforced through ``validator`` on every operation
|
||||
this handle performs. Restoring re-loads the user row and fails closed
|
||||
on a changed token_version, needs_setup or a disabled account, exactly
|
||||
like ``restore_batch_authority``; it never creates fresh authority.
|
||||
"""
|
||||
source, version, permissions = grant.get("source"), grant.get("token_version"), grant.get("permissions")
|
||||
if source not in (AUTH_SOURCE_SESSION, AUTH_SOURCE_AUTH_DISABLED) or type(version) is not int or not isinstance(permissions, list) or any(not isinstance(value, str) for value in permissions):
|
||||
raise AgentRunError(403, "Invalid delegated authority")
|
||||
user = get_auth_disabled_user() if source == AUTH_SOURCE_AUTH_DISABLED and is_auth_disabled() else await self.load_user(user_id) if source == AUTH_SOURCE_SESSION and self.load_user is not None else None
|
||||
if user is None or str(user.id) != user_id or user.token_version != version or user.needs_setup or getattr(user, "disabled", False):
|
||||
raise AgentRunError(403, "Delegated authorization revoked")
|
||||
return _BoundAgentRuns(
|
||||
self,
|
||||
user_id,
|
||||
version,
|
||||
source,
|
||||
frozenset(permissions),
|
||||
plugin_namespace=plugin_namespace,
|
||||
origin=make_origin("extension", namespace=plugin_namespace),
|
||||
delegation_validator=validator,
|
||||
)
|
||||
|
||||
|
||||
class _BoundAgentRuns:
|
||||
def __init__(self, host, user_id: str, token_version: int, source: str, permissions: frozenset[str], plugin_namespace: str | None = None, origin: dict | None = None):
|
||||
def __init__(self, host, user_id: str, token_version: int, source: str, permissions: frozenset[str], plugin_namespace: str | None = None, origin: dict | None = None, delegation_validator: Callable[[], Awaitable[None]] | None = None):
|
||||
self._host = host
|
||||
self._user_id = user_id
|
||||
self._token_version = token_version
|
||||
@ -99,6 +133,19 @@ class _BoundAgentRuns:
|
||||
self._permissions = permissions
|
||||
self._plugin_namespace = plugin_namespace
|
||||
self._origin = origin
|
||||
self._delegation_validator = delegation_validator
|
||||
|
||||
def capture_authority(self) -> dict | None:
|
||||
"""Persistable restorable snapshot of this handle's existing authority.
|
||||
|
||||
Returns the mint-time ceiling (token_version, source, permissions) for
|
||||
a host-originated session handle, or None when this handle's authority
|
||||
is not restorable. The host never validates the caller here: capture
|
||||
only reflects authority the live handle already holds.
|
||||
"""
|
||||
if not self._host.active or self._source not in (AUTH_SOURCE_SESSION, AUTH_SOURCE_AUTH_DISABLED):
|
||||
return None
|
||||
return {"token_version": self._token_version, "source": self._source, "permissions": sorted(self._permissions)}
|
||||
|
||||
def for_plugin(self, namespace: str):
|
||||
if not isinstance(namespace, str) or not namespace or len(namespace) > 96 or ":" in namespace:
|
||||
@ -107,7 +154,7 @@ class _BoundAgentRuns:
|
||||
if namespace != self._plugin_namespace:
|
||||
raise AgentRunError(422, "Agent run handle is already bound to a plugin")
|
||||
return self
|
||||
return _BoundAgentRuns(self._host, self._user_id, self._token_version, self._source, self._permissions, namespace)
|
||||
return _BoundAgentRuns(self._host, self._user_id, self._token_version, self._source, self._permissions, namespace, self._origin, self._delegation_validator)
|
||||
|
||||
@asynccontextmanager
|
||||
async def _request(self, permission: str, thread_id: str | None = None):
|
||||
@ -117,8 +164,15 @@ class _BoundAgentRuns:
|
||||
user = get_auth_disabled_user() if is_auth_disabled() else None
|
||||
else:
|
||||
user = await self._host.load_user(self._user_id) if self._host.load_user is not None else None
|
||||
if user is None or str(user.id) != self._user_id or user.token_version != self._token_version or user.needs_setup:
|
||||
if user is None or str(user.id) != self._user_id or user.token_version != self._token_version or user.needs_setup or getattr(user, "disabled", False):
|
||||
raise AgentRunError(403, "Agent run authorization revoked")
|
||||
if self._delegation_validator is not None:
|
||||
try:
|
||||
await self._delegation_validator()
|
||||
except AgentRunError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise AgentRunError(403, "Delegated authorization revoked") from exc
|
||||
permissions = self._permissions.intersection(await resolve_route_permissions(user, is_internal=False))
|
||||
if permission not in permissions:
|
||||
raise AgentRunError(403, f"Permission denied: {permission}")
|
||||
|
||||
@ -76,6 +76,18 @@ class AgentRuns(Protocol):
|
||||
"""
|
||||
...
|
||||
|
||||
def capture_authority(self) -> Mapping[str, Any] | None:
|
||||
"""Persistable snapshot of this handle's existing authority, or None.
|
||||
|
||||
Returns the mint-time ceiling (token_version, source, permissions) of
|
||||
the live session handle so a service can restore equivalent authority
|
||||
later through the host's delegated-restore capability. Capture only
|
||||
reflects authority the handle already holds; it is not a credential to
|
||||
serialize across hosts, and None means this host or handle cannot be
|
||||
restored. Hosts that predate this method lack it entirely.
|
||||
"""
|
||||
...
|
||||
|
||||
async def send_input(self, *, thread_id: str, run_id: str, text: str, mode: str, request_id: str, context: Mapping[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""Queue a new turn or steer the active Run; never change its model."""
|
||||
...
|
||||
|
||||
@ -251,6 +251,12 @@ class ExtensionRuntimeDeps:
|
||||
model_invoker: ModelInvoker | None = None
|
||||
completed_run_evidence: CompletedRunEvidenceReader | None = None
|
||||
skill_mutations: SkillMutationService | None = None
|
||||
# Async callable (user_id, grant, *, validator=None, plugin_namespace=None)
|
||||
# -> AgentRuns restoring a captured session ceiling for unattended host-side
|
||||
# recovery. The host revalidates the user row per operation and fails closed;
|
||||
# expiry/revocation stay the extension's contract via the validator. None on
|
||||
# hosts without the delegated-restore capability.
|
||||
agent_runs_restore: Any | None = None
|
||||
|
||||
|
||||
class ExtensionService(Protocol):
|
||||
|
||||
@ -101,6 +101,7 @@ hook directory, so they never run a developer's or CI's Git hooks.
|
||||
cannot collide.
|
||||
- **`configuration_revision` never exposes raw configuration.** It returns an HMAC.
|
||||
Sets hash the same way across process hash seeds; ordered lists keep their order.
|
||||
- **Restorable authority is captured, never created.** Restores revalidate it.
|
||||
|
||||
## Extension-first exceptions
|
||||
|
||||
|
||||
@ -581,6 +581,7 @@ async def start_services(
|
||||
run_evidence_reader: Any | None = None,
|
||||
host_capabilities: dict[str, tuple[Any | None, Any | None]] | None = None,
|
||||
attempted_services: list[tuple[str, Any]] | None = None,
|
||||
agent_runs_restore: Any | None = None,
|
||||
) -> list[Diagnostic]:
|
||||
"""Start extension services in registration order, failing open per item."""
|
||||
diagnostics: list[Diagnostic] = []
|
||||
@ -592,6 +593,7 @@ async def start_services(
|
||||
policy=project_host_policy(app_config),
|
||||
session_factory=session_factory,
|
||||
run_evidence_reader=run_evidence_reader,
|
||||
agent_runs_restore=agent_runs_restore,
|
||||
)
|
||||
source_deps = {}
|
||||
for entry in extensions.services:
|
||||
|
||||
@ -71,6 +71,21 @@ async def test_handle_rechecks_user_and_shutdown_before_dispatch(host, monkeypat
|
||||
assert handler.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_rechecks_current_account_disablement(host, monkeypatch):
|
||||
control, request, load_user = host
|
||||
runs = control.bind(request)
|
||||
handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a"))
|
||||
monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler)
|
||||
assert await runs.create_thread() == "thread-a"
|
||||
# Disabling the account leaves token_version unchanged, so the per-operation
|
||||
# guard must observe the current disabled state, not only the mint-time one.
|
||||
load_user.return_value = request.state.user.model_copy(update={"disabled": True})
|
||||
with pytest.raises(AgentRunError, match="revoked"):
|
||||
await runs.create_thread()
|
||||
assert handler.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permissions_are_intersected_with_original_grant_and_current_policy(host, monkeypatch):
|
||||
control, request, _ = host
|
||||
@ -667,3 +682,97 @@ async def test_configuration_revision_canonicalizes_sets_but_preserves_list_orde
|
||||
assert before == await runs.configuration_revision(assistant_ids=["lead_agent"], durable=True)
|
||||
data["tools"].reverse()
|
||||
assert before != await runs.configuration_revision(assistant_ids=["lead_agent"], durable=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_authority_snapshots_the_session_ceiling(host, monkeypatch):
|
||||
control, request, _ = host
|
||||
runs = control.bind(request)
|
||||
handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a"))
|
||||
monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler)
|
||||
assert await runs.create_thread() == "thread-a"
|
||||
captured = runs.capture_authority()
|
||||
assert captured == {
|
||||
"token_version": request.state.user.token_version,
|
||||
"source": "session",
|
||||
"permissions": sorted(request.state.auth.permissions),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restore_delegated_authority_enforces_validator_per_operation(host, monkeypatch):
|
||||
control, request, load_user = host
|
||||
handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a"))
|
||||
monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler)
|
||||
calls = []
|
||||
|
||||
async def validator():
|
||||
calls.append(1)
|
||||
if len(calls) > 1:
|
||||
raise PermissionError("grant revoked by the owner")
|
||||
|
||||
runs = await control.restore_delegated_authority(
|
||||
user_id=str(request.state.user.id),
|
||||
grant={"token_version": request.state.user.token_version, "source": "session", "permissions": ["threads:write"]},
|
||||
validator=validator,
|
||||
plugin_namespace="workflows",
|
||||
)
|
||||
assert await runs.create_thread() == "thread-a"
|
||||
with pytest.raises(AgentRunError, match="Delegated authorization revoked"):
|
||||
await runs.create_thread()
|
||||
assert handler.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restore_delegated_authority_maps_validator_errors_to_403(host, monkeypatch):
|
||||
control, request, _ = host
|
||||
handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a"))
|
||||
monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler)
|
||||
|
||||
async def validator():
|
||||
raise AgentRunError(403, "Workflow recovery grant expired or revoked")
|
||||
|
||||
runs = await control.restore_delegated_authority(
|
||||
user_id=str(request.state.user.id),
|
||||
grant={"token_version": request.state.user.token_version, "source": "session", "permissions": ["threads:write"]},
|
||||
validator=validator,
|
||||
)
|
||||
with pytest.raises(AgentRunError, match="grant expired or revoked"):
|
||||
await runs.create_thread()
|
||||
assert handler.await_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restore_delegated_authority_rejects_stale_grants(host):
|
||||
control, request, load_user = host
|
||||
user = request.state.user
|
||||
load_user.return_value = user.model_copy(update={"token_version": user.token_version + 1})
|
||||
with pytest.raises(AgentRunError, match="Delegated authorization revoked"):
|
||||
await control.restore_delegated_authority(
|
||||
user_id=str(user.id),
|
||||
grant={"token_version": user.token_version, "source": "session", "permissions": ["threads:write"]},
|
||||
)
|
||||
with pytest.raises(AgentRunError, match="Invalid delegated authority"):
|
||||
await control.restore_delegated_authority(
|
||||
user_id=str(user.id),
|
||||
grant={"token_version": user.token_version, "source": "pat", "permissions": ["threads:write"]},
|
||||
)
|
||||
with pytest.raises(AgentRunError, match="Invalid delegated authority"):
|
||||
await control.restore_delegated_authority(
|
||||
user_id=str(user.id),
|
||||
grant={"token_version": user.token_version, "source": "session", "permissions": [1]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restored_delegated_handle_is_plugin_scoped_with_extension_origin(host):
|
||||
control, request, _ = host
|
||||
runs = await control.restore_delegated_authority(
|
||||
user_id=str(request.state.user.id),
|
||||
grant={"token_version": request.state.user.token_version, "source": "session", "permissions": ["threads:write"]},
|
||||
plugin_namespace="workflows",
|
||||
)
|
||||
assert runs.for_plugin("workflows") is runs
|
||||
assert runs._origin == {"kind": "extension", "namespace": "workflows"}
|
||||
with pytest.raises(AgentRunError, match="already bound"):
|
||||
runs.for_plugin("other")
|
||||
|
||||
@ -957,3 +957,24 @@ def test_lifespan_detaches_the_outbox_when_the_delivery_worker_cannot_start() ->
|
||||
|
||||
# Startup survives, and nothing keeps enqueueing rows that no worker would send.
|
||||
scheduled_service.detach_notification_outbox.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delegated_restore_waits_for_agent_runs_host_installation(monkeypatch):
|
||||
from app.gateway.deps import _agent_runs_host_after_install
|
||||
|
||||
app = SimpleNamespace(state=SimpleNamespace())
|
||||
installed = asyncio.Event()
|
||||
# Extension services start before the AgentRuns host exists, so a restore
|
||||
# issued during that window waits for the installation signal instead of
|
||||
# failing the unattended recovery scan outright.
|
||||
waiter = asyncio.create_task(_agent_runs_host_after_install(app, installed))
|
||||
assert not waiter.done()
|
||||
app.state.agent_runs_host = object()
|
||||
installed.set()
|
||||
assert await asyncio.wait_for(waiter, 1) is app.state.agent_runs_host
|
||||
# A startup that never installs the host fails closed per record within
|
||||
# the bounded budget; manual recovery stays available.
|
||||
monkeypatch.setattr("app.gateway.deps._EXTENSION_RESTORE_READY_TIMEOUT_SECONDS", 0.01)
|
||||
with pytest.raises(RuntimeError, match="Agent run host unavailable"):
|
||||
await _agent_runs_host_after_install(SimpleNamespace(state=SimpleNamespace()), asyncio.Event())
|
||||
|
||||
@ -41,6 +41,11 @@ class Runs:
|
||||
|
||||
revision = "initial"
|
||||
|
||||
def capture_authority(self):
|
||||
if self.denied:
|
||||
return None
|
||||
return {"token_version": 0, "source": "session", "permissions": ["runs:create", "threads:write"]}
|
||||
|
||||
async def configuration_evidence(self, *, thread_id, run_id):
|
||||
return {"run_id": run_id, "thread_id": thread_id, "source": "host_assembly", "version": "synthetic-assembly", "model": "test-model"}
|
||||
|
||||
@ -1162,3 +1167,255 @@ agent({stepId:args.second,agentRef:'lead_agent',task:'read',artifacts:[f.artifac
|
||||
assert len(result["steps"]) <= 2
|
||||
assert len(runs.starts) <= 2
|
||||
assert len({s["step_id"] for s in result["steps"]}) == len(result["steps"])
|
||||
|
||||
|
||||
class ValidatingRuns:
|
||||
"""Wraps every delegated operation in the host's per-operation validator gate."""
|
||||
|
||||
def __init__(self, runs, validator):
|
||||
self._runs = runs
|
||||
self._validator = validator
|
||||
|
||||
def __getattr__(self, name):
|
||||
attribute = getattr(self._runs, name)
|
||||
if not callable(attribute):
|
||||
return attribute
|
||||
|
||||
async def operation(*args, **kwargs):
|
||||
await self._validator()
|
||||
return await attribute(*args, **kwargs)
|
||||
|
||||
return operation
|
||||
|
||||
|
||||
class Restorer:
|
||||
"""Host-side delegated restore stub; the real host revalidates the grant on
|
||||
every operation of the returned handle, including before publication."""
|
||||
|
||||
def __init__(self, runs):
|
||||
self.runs = runs
|
||||
self.calls = []
|
||||
|
||||
async def __call__(self, *, user_id, grant, validator=None, plugin_namespace=None):
|
||||
self.calls.append((user_id, grant, plugin_namespace))
|
||||
return self.runs if validator is None else ValidatingRuns(self.runs, validator)
|
||||
|
||||
|
||||
async def crash_like_record(service, run_id, owner="alice"):
|
||||
"""What a SIGKILL leaves: recoverable, children back at their queued window."""
|
||||
record = await asyncio.to_thread(service.store.get, owner, run_id)
|
||||
record["status"] = "recoverable"
|
||||
record["error"] = None
|
||||
for row in record["steps"]:
|
||||
if row["status"] in ("queued", "creating_thread", "admitting", "running", "cancelled"):
|
||||
row["status"] = "queued"
|
||||
row["run_id"] = None
|
||||
row.pop("error", None)
|
||||
# A graceful stop records an error response for the in-flight frame; a real
|
||||
# crash loses it, leaving the frame unanswered for replay to re-dispatch.
|
||||
for item in record["journal"]:
|
||||
item["responses"] = None
|
||||
await asyncio.to_thread(service.store.put, owner, record)
|
||||
return record
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unattended_recovery_requires_session_opt_in_and_capture(plugin):
|
||||
from dataclasses import replace
|
||||
|
||||
_, actions, service, runs = plugin
|
||||
with pytest.raises(ValueError, match="Unattended recovery requires a session-authenticated launch"):
|
||||
await actions["launch"]({"request_id": "a", "script": "return 1;", "args": {}, "unattended_recovery": True}, context(runs))
|
||||
with pytest.raises(ValueError, match="unattended_recovery must be boolean"):
|
||||
await actions["launch"]({"request_id": "b", "script": "return 1;", "args": {}, "unattended_recovery": "yes"}, replace(context(runs), interaction="session"))
|
||||
session = replace(context(runs), interaction="session")
|
||||
plain = await actions["launch"]({"request_id": "c", "script": "return 1;"}, session)
|
||||
assert (await asyncio.to_thread(service.store.get, "alice", plain["run_id"]))["grant"] is None
|
||||
granted = await actions["launch"]({"request_id": "d", "script": "return 1;", "unattended_recovery": True}, session)
|
||||
record = await asyncio.to_thread(service.store.get, "alice", granted["run_id"])
|
||||
assert record["grant"] == {
|
||||
"authority": {"token_version": 0, "source": "session", "permissions": ["runs:create", "threads:write"]},
|
||||
"expires_at": pytest.approx(time.time() + service.config["pause_timeout_seconds"], abs=5),
|
||||
"revoked": False,
|
||||
"version": 1,
|
||||
}
|
||||
# The grant is authority metadata and never projected to API responses.
|
||||
assert "grant" not in await actions["get"]({"run_id": granted["run_id"]}, context(runs))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unattended_recovery_resumes_crashed_workflow_to_child_results(plugin):
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
|
||||
from deerflow_extension_workflows.constants import NAMESPACE
|
||||
from deerflow_extension_workflows.service import Workflows
|
||||
|
||||
_, actions, service, runs = plugin
|
||||
script = "const a = await agent({stepId:'a',agentRef:'lead_agent',task:'blocked'});const b = await agent({stepId:'b',agentRef:'lead_agent',task:'second'});return b.output;"
|
||||
launched = await actions["launch"]({"request_id": "crash", "script": script, "unattended_recovery": True}, replace(context(runs), interaction="session"))
|
||||
run_id = launched["run_id"]
|
||||
# Freeze the crash window: the step row is saved queued, admission never starts.
|
||||
gate = asyncio.Event()
|
||||
original_create = runs.create_thread
|
||||
|
||||
async def gated_create(**kwargs):
|
||||
await gate.wait()
|
||||
return await original_create(**kwargs)
|
||||
|
||||
runs.create_thread = gated_create
|
||||
for _ in range(500):
|
||||
record = await asyncio.to_thread(service.store.get, "alice", run_id)
|
||||
if record["steps"]:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
gate.set()
|
||||
await service.stop()
|
||||
runs.create_thread = original_create
|
||||
record = await crash_like_record(service, run_id)
|
||||
# Nothing was truly admitted before the crash; drop the interrupted stand-ins.
|
||||
stale_thread = record["steps"][0]["thread_id"]
|
||||
runs.runs = {key: value for key, value in runs.runs.items() if value.thread_id != stale_thread}
|
||||
|
||||
restorer = Restorer(runs)
|
||||
replacement = Workflows(service.config)
|
||||
await replacement.start(SimpleNamespace(agent_runs_restore=restorer))
|
||||
try:
|
||||
for _ in range(500):
|
||||
if run_id in replacement.entries:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
assert run_id in replacement.entries, "unattended recovery never resumed the workflow"
|
||||
runs.release.set()
|
||||
await asyncio.wait_for(replacement.wait_idle(run_id), 10)
|
||||
final = await replacement.get({"run_id": run_id}, context(runs))
|
||||
assert final["status"] == "completed", (final.get("error"), [(s["step_id"], s["status"], s.get("error")) for s in final["steps"]])
|
||||
# The unadmitted child was re-dispatched with its original identity and key.
|
||||
assert final["steps"][0]["attempt"] == 1
|
||||
assert final["steps"][0]["idempotency_key"] == record["steps"][0]["idempotency_key"]
|
||||
assert final["steps"][0]["thread_id"] == record["steps"][0]["thread_id"]
|
||||
assert final["result"] == {"echo": "second"}
|
||||
assert restorer.calls == [("alice", {"token_version": 0, "source": "session", "permissions": ["runs:create", "threads:write"]}, NAMESPACE)]
|
||||
finally:
|
||||
gate.set()
|
||||
runs.release.set()
|
||||
await replacement.stop()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelling_a_recovered_workflow_keeps_child_cleanup_authorized(plugin):
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
|
||||
from deerflow_extension_workflows.service import Workflows
|
||||
|
||||
_, actions, service, runs = plugin
|
||||
launched = await actions["launch"](
|
||||
{"request_id": "recancel", "script": "return await agent({stepId:'a',agentRef:'lead_agent',task:'blocked'});", "unattended_recovery": True},
|
||||
replace(context(runs), interaction="session"),
|
||||
)
|
||||
run_id = launched["run_id"]
|
||||
# Freeze the crash window: the step row is saved queued, admission never starts.
|
||||
gate = asyncio.Event()
|
||||
original_create = runs.create_thread
|
||||
|
||||
async def gated_create(**kwargs):
|
||||
await gate.wait()
|
||||
return await original_create(**kwargs)
|
||||
|
||||
runs.create_thread = gated_create
|
||||
for _ in range(500):
|
||||
record = await asyncio.to_thread(service.store.get, "alice", run_id)
|
||||
if record["steps"]:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
gate.set()
|
||||
await service.stop()
|
||||
runs.create_thread = original_create
|
||||
record = await crash_like_record(service, run_id)
|
||||
stale_thread = record["steps"][0]["thread_id"]
|
||||
runs.runs = {key: value for key, value in runs.runs.items() if value.thread_id != stale_thread}
|
||||
|
||||
replacement = Workflows(service.config)
|
||||
await replacement.start(SimpleNamespace(agent_runs_restore=Restorer(runs)))
|
||||
try:
|
||||
for _ in range(500):
|
||||
if runs.entered.is_set() and replacement.entries.get(run_id) is not None:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
assert runs.entered.is_set(), "recovered workflow never re-admitted its child"
|
||||
# The delegated handle is the only authorized path to the running child;
|
||||
# revoking before cleanup would strand it in needs_reconciliation.
|
||||
# Cancel on the replacement service: the fixture's actions are bound to
|
||||
# the original, which stop() already made unavailable.
|
||||
assert (await replacement.cancel({"run_id": run_id}, context(runs)))["status"] == "cancelling"
|
||||
await asyncio.wait_for(replacement.wait_idle(run_id), 10)
|
||||
final = await replacement.get({"run_id": run_id}, context(runs))
|
||||
assert final["status"] == "cancelled", (final.get("error"), [(s["step_id"], s["status"], s.get("error")) for s in final["steps"]])
|
||||
assert final["steps"][0]["status"] == "cancelled", final["steps"][0]
|
||||
persisted = await asyncio.to_thread(replacement.store.get, "alice", run_id)
|
||||
assert persisted["grant"]["revoked"] is True
|
||||
finally:
|
||||
gate.set()
|
||||
runs.release.set()
|
||||
await replacement.stop()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_revoked_or_expired_grants_stay_on_the_manual_path(plugin):
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
|
||||
from deerflow_extension_workflows.service import Workflows
|
||||
|
||||
_, actions, service, runs = plugin
|
||||
service.config["pause_timeout_seconds"] = 1
|
||||
launched = await actions["launch"]({"request_id": "paused", "script": "await pause({stepId:'p'});", "unattended_recovery": True}, replace(context(runs), interaction="session"))
|
||||
run_id = launched["run_id"]
|
||||
await waiting(plugin, run_id, "paused")
|
||||
await service.stop()
|
||||
await crash_like_record(service, run_id)
|
||||
|
||||
# Revoked: the owner's cancellation withdrew the standing authority.
|
||||
record = await asyncio.to_thread(service.store.get, "alice", run_id)
|
||||
record["grant"]["revoked"] = True
|
||||
await asyncio.to_thread(service.store.put, "alice", record)
|
||||
restorer = Restorer(runs)
|
||||
replacement = Workflows(service.config)
|
||||
await replacement.start(SimpleNamespace(agent_runs_restore=restorer))
|
||||
try:
|
||||
await asyncio.sleep(0.05)
|
||||
assert not restorer.calls and run_id not in replacement.entries
|
||||
assert (await replacement.get({"run_id": run_id}, context(runs)))["status"] == "recoverable"
|
||||
# Manual session recovery is unaffected by the unusable grant.
|
||||
recovered = await replacement.recover({"run_id": run_id}, replace(context(runs), interaction="session"))
|
||||
assert recovered["status"] == "queued"
|
||||
finally:
|
||||
await replacement.stop()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expired_grant_never_restores_authority(plugin):
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
|
||||
from deerflow_extension_workflows.service import Workflows
|
||||
|
||||
_, actions, service, runs = plugin
|
||||
service.config["pause_timeout_seconds"] = 1
|
||||
launched = await actions["launch"]({"request_id": "stale", "script": "await pause({stepId:'p'});", "unattended_recovery": True}, replace(context(runs), interaction="session"))
|
||||
run_id = launched["run_id"]
|
||||
await waiting(plugin, run_id, "paused")
|
||||
await service.stop()
|
||||
await crash_like_record(service, run_id)
|
||||
await asyncio.sleep(1.1) # the bounded mint-time TTL lapses
|
||||
|
||||
restorer = Restorer(runs)
|
||||
replacement = Workflows(service.config)
|
||||
await replacement.start(SimpleNamespace(agent_runs_restore=restorer))
|
||||
try:
|
||||
await asyncio.sleep(0.05)
|
||||
assert not restorer.calls and run_id not in replacement.entries
|
||||
assert (await replacement.get({"run_id": run_id}, context(runs)))["status"] == "recoverable"
|
||||
finally:
|
||||
await replacement.stop()
|
||||
|
||||
@ -399,8 +399,32 @@ and is distinct from this configuration preflight check.
|
||||
Phase 3 and Phase 4 share the design: stable step/attempt and idempotency identities,
|
||||
conditional record revisions, a durable intent/result journal, and fresh authority
|
||||
support the leases and fenced dispatch in Phase 4 below. SQLite retains the Phase 3
|
||||
single-owner contract and exclusive process lock. Both modes require explicit
|
||||
session authorization for recovery; neither reconstructs credentials from records.
|
||||
single-owner contract and exclusive process lock. Neither mode reconstructs
|
||||
credentials from records; recovery authority comes from a live session or, on the
|
||||
single-Gateway SQLite store, from the opt-in grant below.
|
||||
|
||||
|
||||
### Opt-in unattended recovery after restart
|
||||
|
||||
A workflow launched from an authenticated browser session may set
|
||||
`unattended_recovery: true`. The launch then captures the session's existing
|
||||
authority ceiling (token version, source, permission set) into a grant on the
|
||||
workflow record, bounded by `pause_timeout_seconds` from launch time. Nothing is
|
||||
captured for internal, PAT or model-tool launches; those never mint a grant.
|
||||
|
||||
After a Gateway restart, the single-Gateway SQLite store scans `recoverable`
|
||||
records and resumes the ones holding an unexpired, unrevoked grant without a
|
||||
sign-in: journal replay, completed-step reuse and the uncertain-admission fence
|
||||
behave exactly as in the explicit browser flow above. Clustered (PostgreSQL)
|
||||
stores keep the manual recovery path. Cancellation revokes the grant; the next
|
||||
delegated operation fails closed.
|
||||
|
||||
The host re-validates the owner's user row on every delegated operation, keeps
|
||||
the ceiling intersected with the owner's current route permissions, and re-checks
|
||||
expiry and revocation per operation. Restoring a grant never creates authority
|
||||
the owner did not have at launch: a demoted, disabled or reset account stops
|
||||
recovered work at its next operation. Unfinished child effects and
|
||||
`needs_reconciliation` outcomes remain manual in every mode.
|
||||
|
||||
|
||||
## Phase 4: shared PostgreSQL deployment
|
||||
|
||||
@ -1,16 +1,24 @@
|
||||
"""Versioned bridge journal replay; authority is reacquired from a live session.
|
||||
"""Versioned bridge journal replay; authority is reacquired from a live session
|
||||
or, when the owner minted one at launch, from the persisted recovery grant.
|
||||
|
||||
SQLite retains its exclusive store owner. PostgreSQL uses a shared execution
|
||||
lease and fenced admission; both replay the same committed bridge journal.
|
||||
Unattended grant recovery runs on the single-Gateway SQLite store only.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
|
||||
from deerflow_extension_api import AgentRunError
|
||||
from deerflow_extension_api.plugins import ToolContext
|
||||
|
||||
from .constants import NAMESPACE
|
||||
|
||||
RECOVERY_VERSION = 1
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Recovery:
|
||||
@ -25,11 +33,15 @@ class Recovery:
|
||||
return await asyncio.shield(task)
|
||||
|
||||
async def restore(self, payload, context):
|
||||
from .service import identity, text
|
||||
from .service import text
|
||||
|
||||
run_id = text(payload["run_id"], 64)
|
||||
return await self.restore_body(self.owner(context), run_id, context.agent_runs)
|
||||
|
||||
async def restore_body(self, owner, run_id, runs):
|
||||
from .service import identity
|
||||
|
||||
async with self.launch_lock:
|
||||
owner = self.owner(context)
|
||||
run_id = text(payload["run_id"], 64)
|
||||
record = await asyncio.to_thread(self.store.get, owner, run_id)
|
||||
if run_id in self.entries:
|
||||
return {"run_id": run_id, "status": record["status"]}
|
||||
@ -41,11 +53,11 @@ class Recovery:
|
||||
raise ValueError("Recovery script or arguments changed")
|
||||
if len(self.entries) >= self.config["max_workflows"] or sum(e["owner"] == owner for e in self.entries.values()) >= self.config["max_workflows_per_owner"]:
|
||||
raise ValueError("Active workflow limit")
|
||||
entry = self.new_entry(owner, record, context.agent_runs)
|
||||
entry = self.new_entry(owner, record, runs)
|
||||
await self.check_configuration(entry)
|
||||
if self.clustered:
|
||||
record = await asyncio.to_thread(self.store.claim, owner, run_id)
|
||||
entry = self.new_entry(owner, record, context.agent_runs)
|
||||
entry = self.new_entry(owner, record, runs)
|
||||
# A committed admission intent with no receipt is never dispatched
|
||||
# again. An operator must reconcile its external effects first.
|
||||
uncertain = [s for s in record["steps"] if s["status"] not in ("queued", "completed", "running") or (s["status"] == "running" and not s.get("run_id"))]
|
||||
@ -64,6 +76,52 @@ class Recovery:
|
||||
self.publish(entry)
|
||||
return {"run_id": run_id, "status": "queued"}
|
||||
|
||||
def grant_usable(self, record):
|
||||
grant = record.get("grant")
|
||||
return (
|
||||
isinstance(grant, dict) and grant.get("version") == 1 and grant.get("revoked") is not True and isinstance(grant.get("expires_at"), (int, float)) and grant["expires_at"] > time.time() and isinstance(grant.get("authority"), dict)
|
||||
)
|
||||
|
||||
def grant_validator(self, owner, run_id):
|
||||
async def validate():
|
||||
entry = self.entries.get(run_id)
|
||||
if entry is not None:
|
||||
grant = entry["record"].get("grant")
|
||||
else:
|
||||
# Locked admission publishes the entry only after restore, while
|
||||
# the delegated handle already operates; validate the current
|
||||
# persisted grant until publication switches to the live entry.
|
||||
record = await asyncio.to_thread(self.store.get, owner, run_id)
|
||||
grant = record.get("grant") if isinstance(record, dict) else None
|
||||
if not isinstance(grant, dict) or grant.get("revoked") is True or grant.get("expires_at", 0) <= time.time():
|
||||
raise AgentRunError(403, "Workflow recovery grant expired or revoked")
|
||||
|
||||
return validate
|
||||
|
||||
async def recover_granted_workflows(self):
|
||||
"""Resume opt-in recoverable workflows after a restart, without a session.
|
||||
|
||||
Records without a usable grant stay on the manual sign-in path; per-record
|
||||
failures never block the others or Gateway startup.
|
||||
"""
|
||||
if self.agent_runs_restore is None:
|
||||
return
|
||||
try:
|
||||
records = await asyncio.to_thread(self.store.recoverable)
|
||||
except Exception:
|
||||
logger.exception("Workflow grant scan failed; unattended recovery unavailable")
|
||||
return
|
||||
for owner, run_id, record in records:
|
||||
if not self.available:
|
||||
return
|
||||
if run_id in self.entries or not self.grant_usable(record):
|
||||
continue
|
||||
try:
|
||||
runs = await self.agent_runs_restore(user_id=owner, grant=record["grant"]["authority"], validator=self.grant_validator(owner, run_id), plugin_namespace=NAMESPACE)
|
||||
await self.restore_body(owner, run_id, runs)
|
||||
except Exception:
|
||||
logger.exception("Workflow %s unattended recovery failed; manual recovery remains available", run_id)
|
||||
|
||||
async def begin_frame(self, entry, frame):
|
||||
"""Persist intent before effects; replay verifies every emitted request."""
|
||||
record = entry["record"]
|
||||
|
||||
@ -116,6 +116,10 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
|
||||
self.write_lock = asyncio.Lock()
|
||||
self.global_slots = asyncio.Semaphore(config["global_concurrency"])
|
||||
self.lockfile = None
|
||||
# Host capability restoring captured session authority; None on hosts
|
||||
# without delegated restore (unattended recovery then stays manual).
|
||||
self.agent_runs_restore = None
|
||||
self.recovery_scan = None
|
||||
|
||||
async def start(self, deps):
|
||||
def initialize():
|
||||
@ -157,6 +161,7 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
|
||||
raise
|
||||
|
||||
self.lockfile, self.store, self.runtime_version = await asyncio.to_thread(initialize)
|
||||
self.agent_runs_restore = getattr(deps, "agent_runs_restore", None)
|
||||
if self.clustered and deps is not None:
|
||||
from sqlalchemy import text as sql_text
|
||||
|
||||
@ -169,10 +174,17 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
|
||||
if host_identity != await asyncio.to_thread(self.store.database_identity):
|
||||
raise ValueError("Workflow and host must share the same PostgreSQL database and schema")
|
||||
self.available = True
|
||||
if not self.clustered:
|
||||
# Single-Gateway stores may hold recoverable records whose owner
|
||||
# minted an unattended-recovery grant before the restart. Clustered
|
||||
# stores coordinate through leases and stay on the manual path.
|
||||
self.recovery_scan = asyncio.create_task(self.recover_granted_workflows())
|
||||
|
||||
async def stop(self):
|
||||
async with self.launch_lock:
|
||||
self.available = False
|
||||
scan = self.recovery_scan
|
||||
self.recovery_scan = None
|
||||
for entry in self.entries.values():
|
||||
entry["suspending"] = entry["record"]["status"] != "cancelling"
|
||||
entry["cancel"].set()
|
||||
@ -183,6 +195,14 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
|
||||
await asyncio.gather(*list(self.admissions), return_exceptions=True)
|
||||
if workers:
|
||||
await asyncio.gather(*workers, return_exceptions=True)
|
||||
if scan is not None:
|
||||
scan.cancel()
|
||||
try:
|
||||
await scan
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Workflow recovery scan failed during shutdown")
|
||||
if self.heartbeats:
|
||||
heartbeats = list(self.heartbeats.values())
|
||||
for heartbeat in heartbeats:
|
||||
@ -203,6 +223,16 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
|
||||
raise ValueError("Workflow service unavailable or disabled")
|
||||
return context.principal.user_id
|
||||
|
||||
@staticmethod
|
||||
def _revoke_grant(record):
|
||||
# Withdraw the owner's standing recovery authority with the record, so
|
||||
# the next delegated operation fails closed. A live supervisor defers
|
||||
# this until its cleanup of already-admitted children finishes, since
|
||||
# that cleanup is itself an authorized use of the handle.
|
||||
grant = record.get("grant")
|
||||
if isinstance(grant, dict):
|
||||
grant["revoked"] = True
|
||||
|
||||
async def save(self, owner, record):
|
||||
# SQLite commits survive cancellation of to_thread. Keep the lock and
|
||||
# observe the new revision before propagating cancellation to cleanup.
|
||||
@ -234,8 +264,23 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
|
||||
owner = self.owner(context)
|
||||
if context.agent_runs is None:
|
||||
raise ValueError("Session-bound AgentRuns required; PAT/internal callers unsupported")
|
||||
if set(payload) - {"request_id", "script", "args", "definition"}:
|
||||
if set(payload) - {"request_id", "script", "args", "definition", "unattended_recovery"}:
|
||||
raise ValueError("Unexpected launch fields")
|
||||
unattended = payload.get("unattended_recovery", False)
|
||||
if type(unattended) is not bool:
|
||||
raise ValueError("unattended_recovery must be boolean")
|
||||
grant = None
|
||||
if unattended:
|
||||
capture = getattr(context.agent_runs, "capture_authority", None)
|
||||
captured = capture() if callable(capture) else None
|
||||
if context.interaction != "session" or captured is None:
|
||||
raise ValueError("Unattended recovery requires a session-authenticated launch")
|
||||
grant = {
|
||||
"authority": captured,
|
||||
"expires_at": time.time() + self.config["pause_timeout_seconds"],
|
||||
"revoked": False,
|
||||
"version": 1,
|
||||
}
|
||||
definition = payload.get("definition")
|
||||
if definition is not None:
|
||||
if "script" in payload or not isinstance(definition, dict):
|
||||
@ -299,6 +344,7 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
|
||||
"steps": [],
|
||||
"logs": [],
|
||||
"result": None,
|
||||
"grant": grant,
|
||||
}
|
||||
# SQLite threads survive caller cancellation. Retain a service-owned task
|
||||
# through commit AND supervisor publication, shielding only the caller's wait.
|
||||
@ -401,12 +447,18 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
|
||||
entry["suspending"] = False
|
||||
if entry["record"]["status"] not in TERMINAL:
|
||||
entry["record"]["status"] = "cancelling"
|
||||
# Defer revocation to the supervisor's cleanup: the delegated
|
||||
# handle is the authorized path that cancels children this
|
||||
# workflow already admitted, and revoking now would 403 that
|
||||
# cleanup into needs_reconciliation.
|
||||
entry["revoke_grant_on_finish"] = True
|
||||
await self.save(owner, entry["record"])
|
||||
return {"run_id": run_id, "status": "cancelling"}
|
||||
if record["status"] == "recoverable":
|
||||
if context.agent_runs is None:
|
||||
raise ValueError("Session-bound AgentRuns required")
|
||||
record["status"] = "cancelling"
|
||||
self._revoke_grant(record)
|
||||
await self.save(owner, record)
|
||||
for row in record["steps"]:
|
||||
if row.get("run_id") and row["status"] != "completed":
|
||||
@ -636,7 +688,10 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
|
||||
row["status"] = "failed"
|
||||
response["error"] = str(exc)[:240] if isinstance(exc, ValueError) else "Agent step failed"
|
||||
finally:
|
||||
if replay and "error" in response:
|
||||
if replay and "error" in response and not (entry["cancel"].is_set() and row is not None and row["status"] == "cancelled"):
|
||||
# A replayed step error is fenced as uncertain — unless the
|
||||
# error is the workflow stopping while its re-dispatched child
|
||||
# was already cancelled and observed, which is deterministic.
|
||||
entry["fault"] = "Recovered child outcome requires reconciliation"
|
||||
if row:
|
||||
row["status"] = "needs_reconciliation"
|
||||
@ -829,6 +884,10 @@ class Workflows(Cluster, Recovery, Lifecycle, Artifacts):
|
||||
cleanup = [self.cancel_child(runs, row, **({"record": record} if self.clustered else {})) for row in record["steps"] if row.get("run_id") and row["status"] not in ("completed", "waiting_input", "cancelled")]
|
||||
if cleanup:
|
||||
await asyncio.gather(*cleanup)
|
||||
if entry.pop("revoke_grant_on_finish", False):
|
||||
# Cleanup above was the last authorized user of the delegated
|
||||
# handle; withdraw the persisted grant before the final save.
|
||||
self._revoke_grant(record)
|
||||
if self.clustered:
|
||||
for row in record["steps"]:
|
||||
if self.slot_releasable(row):
|
||||
|
||||
@ -220,6 +220,10 @@ class Store:
|
||||
)
|
||||
]
|
||||
|
||||
def recoverable(self):
|
||||
with closing(self.connect()) as db:
|
||||
return [(row[0], row[1], json.loads(row[2])) for row in db.execute("SELECT owner,id,data FROM workflows WHERE status='recoverable'").fetchall()]
|
||||
|
||||
def delete(self, owner, run_id):
|
||||
with closing(self.connect()) as db, db:
|
||||
row = db.execute("SELECT status FROM workflows WHERE owner=? AND id=?", (owner, run_id)).fetchone()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user