mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 11:20:35 +00:00
* feat(extensions): recover opt-in workflows unattended after restart Workflow launches from an authenticated browser session may opt in to unattended recovery. The launch captures the session's existing authority ceiling into a bounded, revocable grant on the workflow record; after a Gateway restart the single-Gateway SQLite store resumes recoverable records holding a usable grant through the existing journal-replay path, without a sign-in. Host: restore_delegated_authority (sibling of restore_batch_authority) plus capture_authority on bound handles; the per-operation chokepoint runs an extension-supplied grant validator so expiry and revocation fail closed on every operation, including artifact reads. Clustered stores and uncertain admissions stay on the manual path. * fix(extensions): address review findings on unattended workflow recovery - grant_validator validates the current persisted grant during locked admission, where the entry is not published yet, and switches to the live entry after publication; the previous entries-only lookup 403'd every unattended restore of a perfectly usable grant - delegated restore waits for the AgentRuns host installation signal instead of racing the rest of Gateway startup and failing the scan - cancellation defers grant revocation to the supervisor's post-cleanup save, keeping the delegated handle authorized to cancel children the recovered workflow already admitted - the per-operation delegated guard rechecks the current account disablement, which token_version does not reflect - restore stub revalidates per operation like the real host, with regression tests for all of the above; format the new fixture and condense the extension guidance within its budget * fix(tests): cancel the recovered workflow on its own service The fixture's action handlers are bound to the original service, which stop() already made unavailable; the recovered record lives on the replacement instance, so the cancel must go through replacement.cancel. * fix(extensions): let a cancelled replayed child keep its cancelled outcome The replay fence forced every errored re-dispatched step to needs_reconciliation, including the deterministic case where the error is the workflow stopping after its child was already cancelled and observed. Fence only genuinely uncertain outcomes; a clean cancel of a recovered workflow now ends cancelled like a fresh one.
171 lines
8.6 KiB
Python
171 lines
8.6 KiB
Python
"""Versioned bridge journal replay; authority is reacquired from a live session
|
|
or, when the owner minted one at launch, from the persisted recovery grant.
|
|
|
|
SQLite retains its exclusive store owner. PostgreSQL uses a shared execution
|
|
lease and fenced admission; both replay the same committed bridge journal.
|
|
Unattended grant recovery runs on the single-Gateway SQLite store only.
|
|
"""
|
|
|
|
import asyncio
|
|
import copy
|
|
import json
|
|
import logging
|
|
import time
|
|
|
|
from deerflow_extension_api import AgentRunError
|
|
from deerflow_extension_api.plugins import ToolContext
|
|
|
|
from .constants import NAMESPACE
|
|
|
|
RECOVERY_VERSION = 1
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class Recovery:
|
|
async def recover(self, payload, context):
|
|
if set(payload) != {"run_id"}:
|
|
raise ValueError("Recovery requires run_id")
|
|
if isinstance(context, ToolContext) or context.interaction != "session" or context.principal.is_internal or context.agent_runs is None:
|
|
raise ValueError("Recovery requires an authenticated browser session")
|
|
task = asyncio.create_task(self.restore(payload, context))
|
|
self.admissions.add(task)
|
|
task.add_done_callback(self.admission_finished)
|
|
return await asyncio.shield(task)
|
|
|
|
async def restore(self, payload, context):
|
|
from .service import text
|
|
|
|
run_id = text(payload["run_id"], 64)
|
|
return await self.restore_body(self.owner(context), run_id, context.agent_runs)
|
|
|
|
async def restore_body(self, owner, run_id, runs):
|
|
from .service import identity
|
|
|
|
async with self.launch_lock:
|
|
record = await asyncio.to_thread(self.store.get, owner, run_id)
|
|
if run_id in self.entries:
|
|
return {"run_id": run_id, "status": record["status"]}
|
|
if record["status"] != "recoverable" or record.get("recovery_version") != RECOVERY_VERSION:
|
|
raise ValueError("Workflow is not recoverable; inspect retained child outcomes")
|
|
if record.get("runtime_version") != self.runtime_version:
|
|
raise ValueError("Workflow runtime version changed; recovery requires the original compatible runtime")
|
|
if record["script_hash"] != self.script_version(record["script"])[1] or record["args_hash"] != identity(record["args"]):
|
|
raise ValueError("Recovery script or arguments changed")
|
|
if len(self.entries) >= self.config["max_workflows"] or sum(e["owner"] == owner for e in self.entries.values()) >= self.config["max_workflows_per_owner"]:
|
|
raise ValueError("Active workflow limit")
|
|
entry = self.new_entry(owner, record, runs)
|
|
await self.check_configuration(entry)
|
|
if self.clustered:
|
|
record = await asyncio.to_thread(self.store.claim, owner, run_id)
|
|
entry = self.new_entry(owner, record, runs)
|
|
# A committed admission intent with no receipt is never dispatched
|
|
# again. An operator must reconcile its external effects first.
|
|
uncertain = [s for s in record["steps"] if s["status"] not in ("queued", "completed", "running") or (s["status"] == "running" and not s.get("run_id"))]
|
|
if uncertain:
|
|
record.update(
|
|
status="needs_reconciliation",
|
|
error="Uncertain child admission or outcome; inspect child identities before retrying",
|
|
)
|
|
await self.save(owner, record)
|
|
return {"run_id": run_id, "status": record["status"]}
|
|
for artifact in record.get("artifacts", []):
|
|
await asyncio.to_thread(self.store.artifact, owner, run_id, artifact["version"])
|
|
entry["replay"] = copy.deepcopy(record["journal"])
|
|
record.update(status="queued", error=None)
|
|
await self.save(owner, record)
|
|
self.publish(entry)
|
|
return {"run_id": run_id, "status": "queued"}
|
|
|
|
def grant_usable(self, record):
|
|
grant = record.get("grant")
|
|
return (
|
|
isinstance(grant, dict) and grant.get("version") == 1 and grant.get("revoked") is not True and isinstance(grant.get("expires_at"), (int, float)) and grant["expires_at"] > time.time() and isinstance(grant.get("authority"), dict)
|
|
)
|
|
|
|
def grant_validator(self, owner, run_id):
|
|
async def validate():
|
|
entry = self.entries.get(run_id)
|
|
if entry is not None:
|
|
grant = entry["record"].get("grant")
|
|
else:
|
|
# Locked admission publishes the entry only after restore, while
|
|
# the delegated handle already operates; validate the current
|
|
# persisted grant until publication switches to the live entry.
|
|
record = await asyncio.to_thread(self.store.get, owner, run_id)
|
|
grant = record.get("grant") if isinstance(record, dict) else None
|
|
if not isinstance(grant, dict) or grant.get("revoked") is True or grant.get("expires_at", 0) <= time.time():
|
|
raise AgentRunError(403, "Workflow recovery grant expired or revoked")
|
|
|
|
return validate
|
|
|
|
async def recover_granted_workflows(self):
|
|
"""Resume opt-in recoverable workflows after a restart, without a session.
|
|
|
|
Records without a usable grant stay on the manual sign-in path; per-record
|
|
failures never block the others or Gateway startup.
|
|
"""
|
|
if self.agent_runs_restore is None:
|
|
return
|
|
try:
|
|
records = await asyncio.to_thread(self.store.recoverable)
|
|
except Exception:
|
|
logger.exception("Workflow grant scan failed; unattended recovery unavailable")
|
|
return
|
|
for owner, run_id, record in records:
|
|
if not self.available:
|
|
return
|
|
if run_id in self.entries or not self.grant_usable(record):
|
|
continue
|
|
try:
|
|
runs = await self.agent_runs_restore(user_id=owner, grant=record["grant"]["authority"], validator=self.grant_validator(owner, run_id), plugin_namespace=NAMESPACE)
|
|
await self.restore_body(owner, run_id, runs)
|
|
except Exception:
|
|
logger.exception("Workflow %s unattended recovery failed; manual recovery remains available", run_id)
|
|
|
|
async def begin_frame(self, entry, frame):
|
|
"""Persist intent before effects; replay verifies every emitted request."""
|
|
record = entry["record"]
|
|
replay = entry.get("replay", [])
|
|
index = entry.setdefault("frame_index", 0)
|
|
entry["frame_index"] = index + 1
|
|
entry["recover_frame"] = False
|
|
if index < len(replay):
|
|
expected = replay[index]
|
|
if expected["frame"] != frame:
|
|
raise ValueError("Recovery diverged from committed bridge history")
|
|
pending = entry.setdefault("replay_pending", {})
|
|
for message in frame["messages"]:
|
|
if message.get("type") == "agent":
|
|
pending[message["id"]] = message
|
|
if expected["responses"] is not None:
|
|
for response in expected["responses"]:
|
|
pending.pop(response["id"], None)
|
|
return expected["responses"]
|
|
entry["recover_frame"] = True
|
|
else:
|
|
item = {"frame": copy.deepcopy(frame), "responses": None}
|
|
size = len(json.dumps(item).encode())
|
|
if record.get("journal_bytes", 0) + size > 4 * 1024 * 1024:
|
|
raise ValueError("Recovery journal limit (4 MiB)")
|
|
record["journal_bytes"] = record.get("journal_bytes", 0) + size
|
|
record["journal"].append(item)
|
|
await self.save(entry["owner"], record)
|
|
# Restore requests outstanding across earlier response batches too.
|
|
pending = entry.pop("replay_pending", {})
|
|
current = {m.get("id") for m in frame["messages"]}
|
|
for message in pending.values():
|
|
if message["id"] not in current:
|
|
entry["tasks"].add(asyncio.create_task(self.step(entry, message, replay=True)))
|
|
return None
|
|
|
|
async def deliver_frame(self, entry, process, responses):
|
|
record = entry["record"]
|
|
record["remaining_seconds"] = max(0, entry["deadline"].when() - asyncio.get_running_loop().time())
|
|
size = len(json.dumps(responses).encode())
|
|
if record.get("journal_bytes", 0) + size > 4 * 1024 * 1024:
|
|
raise ValueError("Recovery journal limit (4 MiB)")
|
|
record["journal_bytes"] = record.get("journal_bytes", 0) + size
|
|
record["journal"][entry["frame_index"] - 1]["responses"] = copy.deepcopy(responses)
|
|
await self.save(entry["owner"], record)
|
|
await self.send(process, {"responses": responses})
|