feat(coordination): run tasks on managed worker threads (#8387)

* feat(coordination): run tasks on managed worker threads

* fix(coordination): expose task artifact history

* test(boxd): make partial timeout output check deterministic

* fix(coordination): expose task inbox and bound cancellation waits
This commit is contained in:
Wenchao An 2026-11-20 06:26:29 +08:00 committed by GitHub
parent b4b489d7f2
commit 50d04ddf71
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 1254 additions and 37 deletions

View File

@ -31,6 +31,7 @@ from app.gateway.routers import (
channel_connections,
channels,
console,
coordination,
features,
feedback,
github_webhooks,
@ -1358,6 +1359,7 @@ This gateway provides runtime endpoints for agent runs plus custom endpoints for
# Durable MCP tasks are scoped to their owning thread.
app.include_router(mcp_tasks.router)
app.include_router(coordination.router)
app.include_router(subagent_batches.router)
# Memory API is mounted at /api/memory

View File

@ -0,0 +1,238 @@
"""Gateway execution of durable coordination tasks using caller-bound AgentRuns."""
import asyncio
import logging
from hashlib import sha256
from uuid import uuid4
from deerflow_extension_api.agent_runs import AgentRunError
from fastapi import HTTPException
from app.gateway.deps import get_agent_runs
from deerflow.persistence.coordination.model import TaskStatus
from deerflow.persistence.coordination.sql import TaskConflict, TaskNotFound
from deerflow.persistence.coordination.workers import THREAD_PREFIX
logger = logging.getLogger(__name__)
CANCEL_RESPONSE_TIMEOUT = 5
class CoordinationWorkers:
def __init__(self, repository):
self.repo = repository
self._drivers = {}
self._wakeups = set()
self._cancellations = {}
async def close(self):
jobs = [*self._drivers.values(), *self._cancellations.values()]
for job in jobs:
job.cancel()
await asyncio.gather(*jobs, return_exceptions=True)
async def dispatch(self, runs, owner, task_id, assistant_id):
turn = await self.repo.prepare(owner, task_id, assistant_id)
if turn is not None:
await self._admit(runs, owner, task_id, turn)
self.watch(runs, owner, task_id)
return await self.repo.view(owner, task_id)
async def _admit(self, runs, owner, task_id, turn):
worker, execution, project = turn
thread = worker["thread_id"]
if execution["run_id"] is None:
try:
await runs.create_thread(thread_id=thread, assistant_id=worker["assistant_id"], project_id=project)
admitted = await runs.start_coordination_worker(task_id=task_id, execution_id=execution["id"])
return admitted.run_id
except AgentRunError as exc:
if exc.status_code in (400, 403, 404, 422):
await self.repo.settle(owner, task_id, execution["id"], None, "admission_failed", [])
raise
return execution["run_id"]
def watch(self, runs, owner, task_id):
key = (owner, task_id)
if (existing := self._drivers.get(key)) is not None and not existing.done():
self._wakeups.add(key)
return existing
async def drive():
delay = 1
while True:
self._wakeups.discard(key)
try:
await self._drive(runs, owner, task_id)
except AgentRunError as exc:
if exc.status_code < 500:
raise
await asyncio.sleep(delay)
delay = min(30, delay * 2)
continue
if key not in self._wakeups:
return
task = asyncio.create_task(drive())
self._drivers[key] = task
def finished(job):
if self._drivers.get(key) is job:
self._drivers.pop(key)
self._wakeups.discard(key)
if not job.cancelled() and (error := job.exception()) is not None:
logger.warning("Coordination worker observation stopped for task %s (%s)", task_id, type(error).__name__)
task.add_done_callback(finished)
return task
async def _drive(self, runs, owner, task_id):
while (turn := await self.repo.prepare(owner, task_id)) is not None:
worker, execution, _project = turn
if execution["status"] == "waiting_for_input":
return
if execution["run_id"] is None:
await self._admit(runs, owner, task_id, turn)
continue
thread = worker["thread_id"]
try:
run = await runs.wait_coordination_worker(thread_id=thread, run_id=execution["run_id"], timeout=60)
except TimeoutError:
continue
artifacts = []
status = run.status
if status == "success" and not run.stop_reason:
try:
artifacts = await runs.coordination_artifacts(thread_id=thread, run_id=run.run_id)
except AgentRunError as exc:
if exc.status_code >= 500:
raise
status = "delivery_failed"
elif status == "success":
status = "stopped"
await self.repo.settle(owner, task_id, execution["id"], run.run_id, status, artifacts)
async def cancel(self, runs, owner, task_id, revision):
task = await self.repo.request_cancel(owner, task_id, expected_revision=revision)
if task["status"] == TaskStatus.CANCELLED:
return task
key = (owner, task_id)
job = self._cancellations.get(key)
if job is None or job.done():
job = asyncio.create_task(self._finish_cancellation(runs, owner, task_id, task["execution_epoch"]))
self._cancellations[key] = job
def finished(completed):
if self._cancellations.get(key) is completed:
self._cancellations.pop(key)
if not completed.cancelled() and (error := completed.exception()) is not None:
logger.warning("Coordination cancellation stopped for task %s (%s)", task_id, type(error).__name__)
job.add_done_callback(finished)
# asyncio.wait leaves the tracked job running after the response deadline.
done, _ = await asyncio.wait({job}, timeout=CANCEL_RESPONSE_TIMEOUT)
return job.result() if done else task
async def _finish_cancellation(self, runs, owner, task_id, epoch):
delay = 1
while True:
try:
await self._stop_subtree(runs, owner, task_id)
return await self.repo.confirm_cancel(owner, task_id, execution_epoch=epoch)
except AgentRunError as exc:
if exc.status_code < 500:
raise
await asyncio.sleep(delay)
delay = min(30, delay * 2)
async def _stop_subtree(self, runs, owner, task_id):
active = await self.repo.cancellations(owner, task_id)
for thread, run_id in active:
run = await runs.get(thread_id=thread, run_id=run_id)
if run.status in ("pending", "running"):
try:
await runs.cancel(thread_id=thread, run_id=run_id)
except AgentRunError as exc:
if exc.status_code != 409 or (await runs.get(thread_id=thread, run_id=run_id)).status in ("pending", "running"):
raise
for thread, run_id in active:
while True:
try:
await runs.wait_coordination_worker(thread_id=thread, run_id=run_id, timeout=60)
break
except TimeoutError:
continue
for batch in await self.repo.active_batches(owner, task_id):
await runs.cancel_worker_batch(thread_id=batch.thread_id, batch_id=batch.id)
def worker_context(request):
service = getattr(request.app.state, "coordination_workers", None)
if service is None:
raise HTTPException(503, "Coordination workers require SQL persistence")
runs = get_agent_runs(request)
if runs is None:
raise HTTPException(403, "Coordination workers require a user session")
return service, str(request.state.user.id), runs.for_plugin("native-coordination")
def worker_error(exc):
if isinstance(exc, TaskNotFound):
return HTTPException(404, "Task not found")
if isinstance(exc, AgentRunError):
return HTTPException(exc.status_code, str(exc) if exc.status_code < 500 else "Worker host unavailable")
return HTTPException(409, str(exc))
async def managed_task(request, thread_id):
if not thread_id.startswith(THREAD_PREFIX):
return None
service = getattr(request.app.state, "coordination_workers", None)
return await service.repo.task_for_thread(thread_id) if service is not None else None
async def direct_worker_run(request, task_id, body, key):
from app.gateway.deps import get_run_manager
service, owner, runs = worker_context(request)
if getattr(request.state, "run_origin", None) or body.checkpoint or body.checkpoint_id or body.conversation_references or body.multitask_strategy != "reject":
raise HTTPException(409, "Use a task input to continue this managed worker")
try:
if body.command is not None:
if set(body.command) != {"resume"} or body.command["resume"] is None or body.input:
raise HTTPException(422, "An explicit interrupt response is required")
view = await service.repo.view(owner, task_id)
current = view["execution"]
if current and current["status"] == "running" and current["run_id"]:
native = await runs.get(thread_id=view["assignment"]["thread_id"], run_id=current["run_id"])
if native.status == "waiting_for_input":
await service.repo.settle(owner, task_id, current["id"], native.run_id, native.status, [])
turn = await service.repo.resume(owner, task_id, key=key or uuid4().hex, value=body.command["resume"])
run_id = await service._admit(runs, owner, task_id, turn)
service.watch(runs, owner, task_id)
else:
messages = (body.input or {}).get("messages")
if not isinstance(messages, list) or len(messages) != 1 or not isinstance(messages[0], dict):
raise HTTPException(422, "Managed workers accept one text message")
message = messages[0]
content = message.get("content")
if isinstance(content, list) and all(isinstance(part, dict) and part.get("type") == "text" and isinstance(part.get("text"), str) for part in content):
content = "".join(part["text"] for part in content)
if message.get("role", message.get("type")) not in ("user", "human") or not isinstance(content, str) or not 0 < len(content.strip()) <= 16384:
raise HTTPException(422, "Managed workers accept one text message")
identifier = key or message.get("id") or uuid4().hex
if not isinstance(identifier, str):
raise HTTPException(422, "Message ID must be a string")
row = await service.repo.append_input(owner, task_id, request_id="chat:" + sha256(identifier.encode()).hexdigest(), content=content)
driver = service.watch(runs, owner, task_id)
# Streaming and wait endpoints must observe the Run that consumed this input.
while not (run_id := await service.repo.run_for_input(owner, task_id, row["sequence"])):
if driver.done():
driver.result()
raise HTTPException(409, "Input queued; the worker is waiting for an explicit interrupt response")
await asyncio.sleep(0.25)
record = await get_run_manager(request).get(run_id, user_id=owner)
if record is None:
raise HTTPException(503, "Worker run is unavailable")
return record
except (TaskNotFound, TaskConflict, AgentRunError) as exc:
raise worker_error(exc) from exc

View File

@ -963,10 +963,14 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
stack.callback(reset_login_throttle_store)
if sf is not None:
from app.gateway.coordination_workers import CoordinationWorkers
from deerflow.persistence.coordination.workers import CoordinationWorkerRepository
from deerflow.persistence.feedback import FeedbackRepository
from deerflow.persistence.personal_access_tokens import PersonalAccessTokenRepository
from deerflow.persistence.run import RunRepository
app.state.coordination_workers = CoordinationWorkers(CoordinationWorkerRepository(sf))
stack.push_async_callback(app.state.coordination_workers.close)
app.state.run_store = RunRepository(sf)
app.state.feedback_repo = FeedbackRepository(sf)
from app.gateway.auth.pat import PAT_LAST_USED_WRITE_INTERVAL_SECONDS
@ -975,6 +979,7 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
else:
from deerflow.runtime.runs.store.memory import MemoryRunStore
app.state.coordination_workers = None
app.state.run_store = MemoryRunStore()
app.state.feedback_repo = None
# Memory backend has no durable PAT store, so Bearer credentials

View File

@ -305,11 +305,11 @@ class _BoundAgentRuns:
return await asyncio.to_thread(snapshot)
async def create_thread(self, *, assistant_id="lead_agent", thread_id=None, metadata=None) -> str:
async def create_thread(self, *, assistant_id="lead_agent", thread_id=None, metadata=None, project_id=None) -> str:
from app.gateway.routers.threads import ThreadCreateRequest, create_thread
async with self._request("threads:write") as request:
response = await create_thread(ThreadCreateRequest(assistant_id=assistant_id, thread_id=thread_id, metadata=_json_object(metadata) or {}), request)
response = await create_thread(ThreadCreateRequest(assistant_id=assistant_id, thread_id=thread_id, project_id=project_id, metadata=_json_object(metadata) or {}), request)
return response.thread_id
async def _start(self, *, thread_id, input=None, resume=None, context=None, idempotency_key=None, execution_fence=None, artifact_imports=None):
@ -350,6 +350,93 @@ class _BoundAgentRuns:
async def start(self, *, thread_id, input, context=None, idempotency_key=None, execution_fence=None, artifact_imports=None) -> AgentRun:
return await self._start(thread_id=thread_id, input=input, context=context, idempotency_key=idempotency_key, execution_fence=execution_fence, artifact_imports=artifact_imports)
async def start_coordination_worker(self, *, task_id, execution_id):
from app.gateway.run_models import RunCreateRequest
from app.gateway.services import start_run
from deerflow.persistence.coordination.sql import TaskConflict, TaskNotFound
from deerflow.persistence.coordination.workers import RUN_KEY_PREFIX
from deerflow.persistence.thread_meta.base import THREAD_PROJECT_METADATA_KEY
async with self._request("runs:create") as request:
try:
worker, frozen, project_id = await request.app.state.coordination_workers.repo.admission(self._user_id, task_id, execution_id)
except TaskNotFound as exc:
raise AgentRunError(404, "Worker task unavailable") from exc
except TaskConflict as exc:
raise AgentRunError(409, str(exc)) from exc
thread_id = worker["thread_id"]
async with self._request("runs:create", thread_id) as request:
record = await request.app.state.thread_store.get(thread_id, user_id=self._user_id)
if record is None or record.get("assistant_id") != worker["assistant_id"] or (record.get("metadata") or {}).get(THREAD_PROJECT_METADATA_KEY) != project_id:
raise AgentRunError(409, "Worker thread binding changed")
body = RunCreateRequest(
assistant_id=worker["assistant_id"],
input=None if "command" in frozen else frozen,
command=frozen.get("command"),
context={"subagent_enabled": True},
on_disconnect="continue",
)
request.state.coordination_execution_id = execution_id
record = await start_run(body, thread_id, request, require_existing_thread=True, idempotency_key=RUN_KEY_PREFIX + execution_id)
return _view(record)
async def wait_coordination_worker(self, *, thread_id, run_id, timeout=60):
"""Observe the committed terminal status, written after the delivery receipt."""
from app.gateway.deps import get_run_store
async with asyncio.timeout(timeout):
delay = 0.25
while True:
async with self._request("runs:read", thread_id) as request:
row = await get_run_store(request).get(run_id, user_id=self._user_id)
if row is None or row["thread_id"] != thread_id:
raise AgentRunError(404, "Worker run unavailable")
if row["status"] not in ("pending", "running"):
return AgentRun(thread_id, run_id, row["status"], row.get("assistant_id"), row.get("stop_reason"))
await asyncio.sleep(delay)
delay = min(delay * 2, 4)
async def cancel_worker_batch(self, *, thread_id, batch_id):
from app.gateway.deps import get_subagent_batch_service
async with self._request("runs:cancel", thread_id) as request:
await get_subagent_batch_service(request).cancel_batch(batch_id=batch_id, user_id=self._user_id, continuation_authority=self)
async def coordination_artifacts(self, *, thread_id, run_id):
from app.gateway.artifact_archive import MAX_FILE_BYTES, MAX_FILES, MAX_TOTAL_BYTES
from app.gateway.deps import get_run_event_store
from app.gateway.extension_artifacts import read_snapshot
from app.gateway.routers.artifacts import reserve_artifact_write
from deerflow.constants import DELIVERY_PRESENTATION_TOOL_NAMES
from deerflow.runtime.runs.manager import ConflictError
from deerflow.utils.file_io import await_drained
async with self._request("runs:read", thread_id) as request:
events = await get_run_event_store(request).list_events(thread_id, run_id, event_types=["run.delivery"], limit=2, user_id=self._user_id)
if len(events) != 1:
raise AgentRunError(409, "Run delivery receipt is unavailable")
by_tool = (events[0].get("content") or {}).get("by_tool")
if not isinstance(by_tool, dict) or any(not isinstance(paths, list) or any(not isinstance(path, str) for path in paths) for paths in by_tool.values()):
raise AgentRunError(409, "Invalid run delivery receipt")
paths = dict.fromkeys(path for tool in DELIVERY_PRESENTATION_TOOL_NAMES for path in by_tool.get(tool, []))
if not paths:
return []
if len(paths) > MAX_FILES:
raise AgentRunError(422, "Too many task artifacts")
artifacts = []
remaining = MAX_TOTAL_BYTES
async with self._request("threads:read", thread_id) as request:
try:
async with reserve_artifact_write(request, thread_id, user_id=self._user_id):
for path in paths:
content = await await_drained(read_snapshot(request, self._user_id, thread_id, path, max_bytes=min(MAX_FILE_BYTES, remaining)))
digest = await asyncio.to_thread(hashlib.sha256, content)
artifacts.append((path, digest.hexdigest(), content))
remaining -= len(content)
except ConflictError as exc:
raise AgentRunError(503, "Worker artifacts are temporarily busy") from exc
return artifacts
async def start_batch_followup(self, *, thread_id, batch_id, input, context, config, execution_fence, goal_wakeup=None):
"""Native single follow-up; the existing SQL admission fences cancellation."""
from app.gateway.run_models import RunCreateRequest

View File

@ -0,0 +1,127 @@
"""Session-authorized task execution shared by coordination entry points."""
from hashlib import sha256
from pathlib import PurePosixPath
from typing import Annotated
from deerflow_extension_api.agent_runs import AgentRunError
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import Response
from pydantic import BaseModel, Field, StringConstraints
from app.gateway.authz import require_permission
from app.gateway.coordination_workers import worker_context, worker_error
from deerflow.persistence.coordination.model import TaskStatus
from deerflow.persistence.coordination.sql import TaskConflict, TaskNotFound
from deerflow.utils.file_io import await_drained
router = APIRouter(prefix="/api/coordination/tasks", tags=["coordination"])
RequestId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=200)]
TaskText = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=16384)]
class CreateWorkerTask(BaseModel):
request_id: RequestId
objective: TaskText
assistant_id: str = Field(default="lead_agent", min_length=1, max_length=128)
project_id: str | None = Field(default=None, min_length=1, max_length=64)
class WorkerInput(BaseModel):
request_id: RequestId
content: TaskText
class CancelTask(BaseModel):
expected_revision: int = Field(ge=0)
@router.post("")
@require_permission("runs", "create")
async def create_worker_task(body: CreateWorkerTask, request: Request):
service, owner, runs = worker_context(request)
async def dispatch():
task = await service.repo.create(owner, request_id=body.request_id, objective=body.objective, project_id=body.project_id)
return await service.dispatch(runs, owner, task["id"], body.assistant_id)
try:
return await await_drained(dispatch())
except (TaskConflict, TaskNotFound, AgentRunError) as exc:
raise worker_error(exc) from exc
@router.get("/{task_id}")
@require_permission("runs", "read")
async def get_worker_task(task_id: str, request: Request):
service, owner, _runs = worker_context(request)
try:
return await service.repo.view(owner, task_id)
except TaskNotFound as exc:
raise HTTPException(404, "Task not found") from exc
@router.post("/{task_id}/inputs")
@require_permission("runs", "create")
async def add_worker_input(task_id: str, body: WorkerInput, request: Request):
service, owner, runs = worker_context(request)
try:
item = await service.repo.append_input(owner, task_id, request_id="input:" + sha256(body.request_id.encode()).hexdigest(), content=body.content)
service.watch(runs, owner, task_id)
return item
except (TaskNotFound, TaskConflict) as exc:
raise worker_error(exc) from exc
@router.get("/{task_id}/inputs")
@require_permission("runs", "read")
async def list_worker_inputs(task_id: str, request: Request, after: int = Query(0, ge=0, le=2**63 - 1), limit: int = Query(100, ge=1, le=100)):
service, owner, _runs = worker_context(request)
try:
return await service.repo.list_worker_inputs(owner, task_id, after=after, limit=limit)
except TaskNotFound as exc:
raise HTTPException(404, "Task not found") from exc
@router.post("/{task_id}/cancel", responses={202: {"description": "Cancellation is still in progress"}})
@require_permission("runs", "cancel")
async def cancel_worker_task(task_id: str, body: CancelTask, request: Request, response: Response):
service, owner, runs = worker_context(request)
try:
task = await await_drained(service.cancel(runs, owner, task_id, body.expected_revision))
except (TaskConflict, TaskNotFound, AgentRunError) as exc:
raise worker_error(exc) from exc
if task["status"] == TaskStatus.CANCELLING:
response.status_code = 202
return task
@router.get("/{task_id}/artifacts")
@require_permission("threads", "read")
async def list_worker_artifacts(task_id: str, request: Request, limit: int = Query(100, ge=1, le=100), offset: int = Query(0, ge=0)):
service, owner, _runs = worker_context(request)
try:
return await service.repo.list_artifacts(owner, task_id, limit=limit, offset=offset)
except TaskNotFound as exc:
raise HTTPException(404, "Task not found") from exc
@router.get("/{task_id}/artifacts/{artifact_id}")
@require_permission("threads", "read")
async def download_worker_artifact(task_id: str, artifact_id: str, request: Request):
from app.gateway.routers.artifacts import _build_content_disposition
service, owner, _runs = worker_context(request)
try:
path, content = await service.repo.read_artifact(owner, task_id, artifact_id)
except TaskNotFound as exc:
raise HTTPException(404, "Artifact not found") from exc
return Response(
content,
media_type="application/octet-stream",
headers={
"Content-Disposition": _build_content_disposition("attachment", PurePosixPath(path).name),
"Cache-Control": "private, no-store",
"X-Content-Type-Options": "nosniff",
},
)

View File

@ -46,6 +46,19 @@ async def submit_run_input(thread_id: str, run_id: str, body: RunInputRequest, r
raise HTTPException(404, "Run not found")
if get_agent_runs(request) is None:
raise HTTPException(503, "Run inputs require an authenticated session")
from app.gateway.coordination_workers import managed_task, worker_context, worker_error
from deerflow.persistence.coordination.sql import TaskConflict, TaskNotFound
if task_id := await managed_task(request, thread_id):
service, owner, runs = worker_context(request)
if body.mode != "queue":
raise HTTPException(409, "Use queue mode to update a managed worker")
try:
item = await service.repo.append_input(owner, task_id, request_id=f"run-input:{run_id}:{body.request_id}", content=body.text)
service.watch(runs, owner, task_id)
return _worker_input_receipt(item, run_id, body.request_id)
except (TaskConflict, TaskNotFound) as exc:
raise worker_error(exc) from exc
row = await _store(request).get(run_id, user_id=user_id)
if row is None or row.get("run_inputs") is None:
raise HTTPException(409, "This Run has not enabled interactive inputs")
@ -63,10 +76,36 @@ async def submit_run_input(thread_id: str, run_id: str, body: RunInputRequest, r
return {**item, "target_run_id": run_id}
def _worker_input_receipt(item, run_id, request_id, consuming_run=None, cancelled=False):
return {
"id": request_id,
"target_run_id": run_id,
"message_id": str(uuid5(NAMESPACE_URL, f"deerflow-input:{run_id}:{request_id}")),
"mode": "queue",
"text": item["content"],
"created_at": item["created_at"],
"status": "applied" if consuming_run else "cancelled" if cancelled else "pending",
**({"run_id": consuming_run} if consuming_run else {}),
}
async def list_thread_inputs(thread_id: str, request: Request):
user_id = await get_current_user(request)
if user_id is None:
raise HTTPException(401, "Authentication required")
from app.gateway.coordination_workers import managed_task
from deerflow.persistence.coordination.sql import TaskNotFound
if task_id := await managed_task(request, thread_id):
try:
receipts, active, cancelled = await request.app.state.coordination_workers.repo.input_receipts(user_id, task_id)
except TaskNotFound as exc:
raise HTTPException(404, "Task not found") from exc
items = []
for item, consuming_run in receipts:
_, run, key = item["request_id"].split(":", 2)
items.append(_worker_input_receipt(item, run, key, consuming_run, cancelled))
return {"items": items, "active_run_id": active}
rows = await _store(request).list_input_runs(thread_id, user_id=user_id)
items = []
for row in reversed(rows):

View File

@ -2562,6 +2562,13 @@ async def start_run(
if not await thread_access_allowed():
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
from app.gateway.coordination_workers import direct_worker_run, managed_task
from deerflow.persistence.coordination.workers import THREAD_PREFIX
if thread_id.startswith(THREAD_PREFIX) and not getattr(request.state, "coordination_execution_id", None):
if task_id := await managed_task(request, thread_id):
return await direct_worker_run(request, task_id, body, idempotency_key)
owner_context_token = set_current_user(SimpleNamespace(id=owner_user_id)) if owner_user_id else None
try:
is_internal_caller = getattr(getattr(request, "state", None), "auth_source", None) == AUTH_SOURCE_INTERNAL

View File

@ -21,7 +21,11 @@ class BatchContinuations:
async def prepare(self, request):
from app.gateway.services import _CONTEXT_CONFIGURABLE_KEYS, _MODEL_SELECTION_CONTEXT_KEYS
from deerflow.persistence.coordination.workers import THREAD_PREFIX
workers = getattr(self.app.state, "coordination_workers", None)
if workers is not None and request.thread_id.startswith(THREAD_PREFIX) and await workers.repo.task_for_thread(request.thread_id):
raise AgentRunError(409, "A managed worker accepts follow-ups through its coordination task")
host = self.app.state.agent_runs_host
grant = host.capture_batch_authority(request.continuation_authority, user_id=request.user_id)
source = await request.continuation_authority.get(thread_id=request.thread_id, run_id=request.run_id)

View File

@ -29,3 +29,70 @@ prerequisites keep their state. A running caller can record cancellation
once it has confirmed that the subtree's executions have stopped. The
repository handles work state. Host admission, run control, and artifact
authorization belong to the execution integration.
## Managed workers
A signed-in caller can dispatch a task through the Gateway on a SQL-backed
installation. The following request starts a worker without a project. Pass
an optional `project_id` to create its thread in an existing project.
```http
POST /api/coordination/tasks
Content-Type: application/json
{"request_id":"review-27","objective":"Review the report","assistant_id":"lead_agent"}
```
The response includes the task, its assignment (agent and thread), and the
latest execution. Replaying the request reuses the worker. Read the same
view with `GET /api/coordination/tasks/{task_id}`. Add a follow-up with
`POST /api/coordination/tasks/{task_id}/inputs` and a JSON object containing
`request_id` and `content`. A text message sent to the worker thread uses
the same queue. Streaming and wait callers observe the run that consumes
their message. An interrupted graph requires an explicit resume command.
The task owner can poll `GET /api/coordination/tasks/{task_id}/inputs`
to read the inbox in sequence order. Pass the last seen `sequence` as
`after` and set `limit` up to 100. Rows include the content, creation
time, and source (`task_api`, `run_input`, or `worker_chat`).
An input is `pending` while it can still enter a Run, `admitted`
with its `run_id` once a Run has accepted the turn, or `not_admitted`
when the task has closed to further work.
The Gateway freezes each turn before admission. The SQL transaction that
creates the native run locks the task tree, verifies the execution epoch,
and records the run ID. A follow-up that arrives during a turn advances
the input sequence. When that run stops, the worker continues on its own
thread with the outstanding inputs. Each task has one assignee per epoch.
Workers can use the agent's allowed subagents for local delegation. Background
batches can run local work. A batch cannot schedule an independent follow-up
run on a managed thread; the task owns the next turn.
`POST /api/coordination/tasks/{task_id}/cancel` accepts `expected_revision`.
It fences the owned subtree before requesting run and native batch
cancellation. If confirmation takes longer than five seconds after the
fence is committed, the response is HTTP 202 with status `cancelling`.
The Gateway continues confirming in the background and publishes
`cancelled` once the runs have stopped and batch cancellation is requested.
Poll `GET /api/coordination/tasks/{task_id}` for the status.
A completed task links to the producing run's messages. For presented output
files, the Gateway stores the captured bytes with their SHA-256 hashes and
producing execution. The owner's task artifact URL continues to serve those
bytes after the source file changes. Capture uses the existing archive budgets
of 50 files, 50 MiB per file and 100 MiB per execution. The execution reports
`delivery_failed` if a presented file cannot be captured.
Use `GET /api/coordination/tasks/{task_id}/artifacts` to list the task's
deliveries, including files from earlier executions. The response lists
artifact IDs, source paths, hashes, sizes, producing `run_id` and `turn`,
and download URLs. Results are ordered by turn, then source path and ID.
Use `limit` (default and maximum 100) and `offset` (default 0) to page
through them. Files presented under the same path in different turns
have separate IDs and URLs. The task detail's `execution.artifacts` lists
deliveries produced by that execution.
Worker observation uses the initiating session's process-local run handle.
The task, frozen inputs, native run IDs and artifact contents survive a Gateway
restart. Automatic reauthorization and restart reconciliation are tracked in
the next stage of [RFC #8345](https://github.com/bytedance/deer-flow/issues/8345).

View File

@ -3,7 +3,7 @@
from datetime import datetime
from enum import StrEnum
from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Index, String, Text, UniqueConstraint
from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, ForeignKeyConstraint, Index, LargeBinary, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@ -52,3 +52,41 @@ class CoordinationInputRow(Base):
request_id: Mapped[str] = mapped_column(String(200))
content: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class CoordinationAssignmentRow(Base):
"""One active worker per task. Parallel work uses child tasks."""
__tablename__ = "coord_assignments"
task_id: Mapped[str] = mapped_column(ForeignKey("coord_tasks.id"), primary_key=True)
epoch: Mapped[int] = mapped_column(BigInteger, primary_key=True)
thread_id: Mapped[str] = mapped_column(String(64), unique=True)
assistant_id: Mapped[str] = mapped_column(String(128))
class CoordinationExecutionRow(Base):
__tablename__ = "coord_executions"
__table_args__ = (
ForeignKeyConstraint(["task_id", "epoch"], ["coord_assignments.task_id", "coord_assignments.epoch"]),
UniqueConstraint("task_id", "turn", name="uq_coord_execution_turn"),
UniqueConstraint("task_id", "resume_key", name="uq_coord_execution_resume"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
task_id: Mapped[str] = mapped_column(String(64))
epoch: Mapped[int] = mapped_column(BigInteger)
turn: Mapped[int] = mapped_column(BigInteger)
resume_key: Mapped[str | None] = mapped_column(String(200))
covered_seq: Mapped[int] = mapped_column(BigInteger)
frozen_input: Mapped[dict] = mapped_column(JSON)
status: Mapped[str] = mapped_column(String(32))
run_id: Mapped[str | None] = mapped_column(String(64), unique=True)
class CoordinationArtifactRow(Base):
__tablename__ = "coord_artifacts"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
execution_id: Mapped[str] = mapped_column(ForeignKey("coord_executions.id"), index=True)
path: Mapped[str] = mapped_column(Text)
sha256: Mapped[str] = mapped_column(String(64))
size: Mapped[int] = mapped_column(BigInteger)
content: Mapped[bytes] = mapped_column(LargeBinary)

View File

@ -176,21 +176,24 @@ class CoordinationTaskRepository:
async with self._write() as session:
task = await self._lock_tree(session, owner_id, task_id)
_check_revision(task, expected_revision)
if task.status != TaskStatus.QUEUED:
raise TaskConflict("Only a queued task can start")
if task.parent_id is not None:
parent = await self._lookup(session, owner_id, task.parent_id)
if parent.status != TaskStatus.RUNNING:
raise TaskConflict("Parent task has not started")
if task.depends_on:
query = select(CoordinationTaskRow.id).filter_by(owner_id=owner_id, status=TaskStatus.SUCCEEDED)
ready = set((await session.scalars(query.where(CoordinationTaskRow.id.in_(task.depends_on)))).all())
if ready != set(task.depends_on):
raise TaskConflict("Task dependencies have not succeeded")
task.status = TaskStatus.RUNNING
_touch(task)
await self._start_locked(session, task)
return _snapshot(task)
async def _start_locked(self, session: AsyncSession, task: CoordinationTaskRow) -> None:
if task.status != TaskStatus.QUEUED:
raise TaskConflict("Only a queued task can start")
if task.parent_id is not None:
parent = await self._lookup(session, task.owner_id, task.parent_id)
if parent.status != TaskStatus.RUNNING:
raise TaskConflict("Parent task has not started")
if task.depends_on:
query = select(CoordinationTaskRow.id).filter_by(owner_id=task.owner_id, status=TaskStatus.SUCCEEDED)
ready = set((await session.scalars(query.where(CoordinationTaskRow.id.in_(task.depends_on)))).all())
if ready != set(task.depends_on):
raise TaskConflict("Task dependencies have not succeeded")
task.status = TaskStatus.RUNNING
_touch(task)
async def finish(
self,
owner_id: str,
@ -211,15 +214,18 @@ class CoordinationTaskRepository:
_check_revision(task, expected_revision)
if task.latest_input_seq != covered_input_seq:
raise TaskConflict("Result does not cover the current task inputs")
children = (await session.scalars(select(CoordinationTaskRow.status).filter_by(parent_id=task_id, owner_id=owner_id))).all()
accepted = {TaskStatus.SUCCEEDED} if outcome == TaskStatus.SUCCEEDED else {TaskStatus.SUCCEEDED, TaskStatus.FAILED, TaskStatus.CANCELLED}
if any(status not in accepted for status in children):
raise TaskConflict("Child tasks have not settled for this outcome")
task.status = outcome
task.result_ref = result_ref
_touch(task)
await self._finish_locked(session, task, outcome, result_ref)
return _snapshot(task)
async def _finish_locked(self, session: AsyncSession, task: CoordinationTaskRow, outcome: TaskStatus, result_ref: str) -> None:
children = (await session.scalars(select(CoordinationTaskRow.status).filter_by(parent_id=task.id, owner_id=task.owner_id))).all()
accepted = {TaskStatus.SUCCEEDED} if outcome == TaskStatus.SUCCEEDED else {TaskStatus.SUCCEEDED, TaskStatus.FAILED, TaskStatus.CANCELLED}
if any(status not in accepted for status in children):
raise TaskConflict("Child tasks have not settled for this outcome")
task.status = outcome
task.result_ref = result_ref
_touch(task)
async def _subtree(self, session: AsyncSession, task: CoordinationTaskRow) -> list[CoordinationTaskRow]:
rows = (await session.scalars(select(CoordinationTaskRow).filter_by(owner_id=task.owner_id, root_id=task.root_id))).all()
children = defaultdict(list)
@ -255,8 +261,20 @@ class CoordinationTaskRepository:
task = await self._lock_tree(session, owner_id, task_id)
if task.status not in (TaskStatus.CANCELLING, TaskStatus.CANCELLED) or task.execution_epoch != execution_epoch:
raise TaskConflict("Cancellation no longer matches the task")
for row in await self._subtree(session, task):
rows = await self._subtree(session, task)
await self._confirm_stopped(session, rows)
for row in rows:
if row.status == TaskStatus.CANCELLING:
row.status = TaskStatus.CANCELLED
_touch(row)
return _snapshot(task)
async def _confirm_stopped(self, session, rows):
"""Close active execution records in the same transaction as their tasks."""
from sqlalchemy import update
from deerflow.persistence.coordination.model import CoordinationExecutionRow
ids = [row.id for row in rows if row.status == TaskStatus.CANCELLING]
if ids:
await session.execute(update(CoordinationExecutionRow).where(CoordinationExecutionRow.task_id.in_(ids), CoordinationExecutionRow.status.in_(("admitting", "running", "waiting_for_input"))).values(status="cancelled"))

View File

@ -0,0 +1,264 @@
"""Durable admission, cancellation and result ownership for managed workers."""
from uuid import uuid4
from sqlalchemy import select
from deerflow.persistence.coordination.model import CoordinationArtifactRow as Artifact
from deerflow.persistence.coordination.model import CoordinationAssignmentRow as Assignment
from deerflow.persistence.coordination.model import CoordinationExecutionRow as Execution
from deerflow.persistence.coordination.model import CoordinationInputRow, CoordinationTaskRow, TaskStatus
from deerflow.persistence.coordination.sql import CoordinationTaskRepository, TaskConflict, TaskNotFound
THREAD_PREFIX = "coord-worker-"
RUN_KEY_PREFIX = "coord-execution:"
INPUT_SOURCES = {"input": "task_api", "chat": "worker_chat", "run-input": "run_input"}
async def bind_admitted_worker(session, thread_id, owner_id, key, run_id):
"""Called inside native SQL run admission, under the same tree lock as cancellation."""
from deerflow.runtime.runs.manager import ConflictError
worker = await session.scalar(select(Assignment).where(Assignment.thread_id == thread_id))
if worker is None:
return # A pre-existing user-chosen thread ID may share the prefix.
task_id = worker.task_id
root_id = await session.scalar(select(CoordinationTaskRow.root_id).filter_by(id=task_id))
await session.execute(select(CoordinationTaskRow.id).filter_by(id=root_id).with_for_update())
task = await session.get(CoordinationTaskRow, task_id)
execution = await session.get(Execution, key[len(RUN_KEY_PREFIX) :]) if key and key.startswith(RUN_KEY_PREFIX) else None
if (
worker.epoch != task.execution_epoch
or task.owner_id != owner_id
or task.status != TaskStatus.RUNNING
or execution is None
or execution.task_id != task_id
or execution.epoch != task.execution_epoch
or execution.status != "admitting"
):
raise ConflictError("Managed worker admission no longer matches the task")
execution.run_id = run_id
execution.status = "running"
class CoordinationWorkerRepository(CoordinationTaskRepository):
async def task_for_thread(self, thread_id):
async with self._sessions() as session:
return await session.scalar(select(Assignment.task_id).filter_by(thread_id=thread_id))
@staticmethod
def _latest(task_id):
return select(Execution).filter_by(task_id=task_id).order_by(Execution.turn.desc()).limit(1)
@staticmethod
async def _artifact_refs(session, task_id, *, execution_id=None, limit=None, offset=0):
query = (
select(Artifact.id, Artifact.path, Artifact.sha256, Artifact.size, Execution.run_id, Execution.turn)
.join(Execution, Execution.id == Artifact.execution_id)
.where(Execution.task_id == task_id)
.order_by(Execution.turn, Artifact.path, Artifact.id)
)
if execution_id is not None:
query = query.where(Execution.id == execution_id)
rows = await session.execute(query.limit(limit).offset(offset))
return [
{
"id": row.id,
"source_path": row.path,
"sha256": row.sha256,
"size": row.size,
"run_id": row.run_id,
"turn": row.turn,
"url": f"/api/coordination/tasks/{task_id}/artifacts/{row.id}",
}
for row in rows
]
async def view(self, owner, task_id):
async with self._sessions() as session:
task = await self._lookup(session, owner, task_id)
assignment = await session.scalar(select(Assignment).filter_by(task_id=task_id).order_by(Assignment.epoch.desc()).limit(1))
execution = await session.scalar(self._latest(task_id))
result = None
if execution is not None:
result = execution.to_dict(exclude={"frozen_input", "resume_key"})
result["artifacts"] = await self._artifact_refs(session, task_id, execution_id=execution.id)
return {
"task": task.to_dict(),
"assignment": assignment.to_dict() if assignment else None,
"execution": result,
}
async def admission(self, owner, task_id, execution_id):
async with self._sessions() as session:
task = await self._lookup(session, owner, task_id)
execution = await session.get(Execution, execution_id)
if execution is None or execution.task_id != task_id or execution.epoch != task.execution_epoch or task.status != TaskStatus.RUNNING:
raise TaskConflict("Task no longer accepts this execution")
worker = await session.get(Assignment, (task_id, execution.epoch))
return worker.to_dict(), execution.frozen_input, task.project_id
async def list_artifacts(self, owner, task_id, *, limit, offset):
async with self._sessions() as session:
await self._lookup(session, owner, task_id)
return await self._artifact_refs(session, task_id, limit=limit, offset=offset)
async def read_artifact(self, owner, task_id, artifact_id):
async with self._sessions() as session:
query = select(Artifact).join(Execution, Execution.id == Artifact.execution_id).join(CoordinationTaskRow, CoordinationTaskRow.id == Execution.task_id)
artifact = await session.scalar(query.where(Artifact.id == artifact_id, Execution.task_id == task_id, CoordinationTaskRow.owner_id == owner))
if artifact is None:
raise TaskNotFound(artifact_id)
return artifact.path, artifact.content
async def prepare(self, owner, task_id, assistant_id=None):
"""Freeze a bounded turn before admission. Duplicate drivers get the same turn."""
async with self._write() as session:
task = await self._lock_tree(session, owner, task_id)
worker = await session.get(Assignment, (task_id, task.execution_epoch))
if worker is not None and assistant_id is not None and worker.assistant_id != assistant_id:
raise TaskConflict("Task already has a different worker")
if task.status == TaskStatus.QUEUED:
await self._start_locked(session, task)
if task.status != TaskStatus.RUNNING:
return None
if worker is None:
if assistant_id is None:
raise TaskConflict("Task has no assigned worker")
worker = Assignment(task_id=task_id, epoch=task.execution_epoch, assistant_id=assistant_id, thread_id=THREAD_PREFIX + uuid4().hex)
session.add(worker)
await session.flush()
execution = await session.scalar(self._latest(task_id))
if execution is None or (execution.status == "superseded" and execution.covered_seq < task.latest_input_seq):
previous = execution.covered_seq if execution else 0
if execution is None:
messages = [{"role": "user", "content": task.objective}]
elif execution.run_id is None:
messages = list(execution.frozen_input.get("messages", []))
else:
messages = []
inputs = await session.scalars(select(CoordinationInputRow).filter_by(task_id=task_id).where(CoordinationInputRow.sequence > previous).order_by(CoordinationInputRow.sequence).limit(8))
for item in inputs:
messages.append({"role": "user", "content": item.content})
previous = item.sequence
execution = Execution(
id=uuid4().hex,
task_id=task_id,
turn=execution.turn + 1 if execution else 1,
epoch=task.execution_epoch,
covered_seq=previous,
frozen_input={"messages": messages},
status="admitting",
)
session.add(execution)
return worker.to_dict(), execution.to_dict(), task.project_id
async def resume(self, owner, task_id, *, key, value):
async with self._write() as session:
task = await self._lock_tree(session, owner, task_id)
payload = {"command": {"resume": value}}
execution = await session.scalar(select(Execution).filter_by(task_id=task_id, resume_key=key))
if execution is not None:
if execution.frozen_input != payload:
raise TaskConflict("Request ID was already used with a different response")
else:
if task.status != TaskStatus.RUNNING:
raise TaskConflict("Task is closed")
previous = await session.scalar(self._latest(task_id))
if previous is None or previous.status != "waiting_for_input":
raise TaskConflict("Worker has no pending interrupt")
execution = Execution(
id=uuid4().hex,
task_id=task_id,
turn=previous.turn + 1,
epoch=task.execution_epoch,
covered_seq=previous.covered_seq,
frozen_input=payload,
status="admitting",
resume_key=key,
)
session.add(execution)
worker = await session.get(Assignment, (task_id, execution.epoch))
return worker.to_dict(), execution.to_dict(), task.project_id
async def settle(self, owner, task_id, execution_id, run_id, status, artifacts):
async with self._write() as session:
task = await self._lock_tree(session, owner, task_id)
execution = await session.get(Execution, execution_id)
if execution is None or execution.task_id != task_id or execution.run_id != run_id or execution.status not in ("running", "admitting"):
return
if task.status != TaskStatus.RUNNING or task.execution_epoch != execution.epoch:
return
for path, digest, content in artifacts:
session.add(Artifact(id=uuid4().hex, execution_id=execution.id, path=path, sha256=digest, size=len(content), content=content))
if status == "waiting_for_input":
execution.status = status
return
if execution.covered_seq < task.latest_input_seq:
execution.status = "superseded"
return
worker = await session.get(Assignment, (task_id, task.execution_epoch))
result_ref = f"/api/threads/{worker.thread_id}/runs/{run_id}/messages" if run_id else None
outcome = TaskStatus.SUCCEEDED if status == "success" else TaskStatus.FAILED
await self._finish_locked(session, task, outcome, result_ref or f"/api/coordination/tasks/{task_id}")
execution.status = status
@staticmethod
def _admitted_run(task_id, sequence):
return select(Execution.run_id).where(Execution.task_id == task_id, Execution.covered_seq >= sequence, Execution.run_id.is_not(None)).order_by(Execution.turn).limit(1)
async def list_worker_inputs(self, owner, task_id, *, after, limit):
async with self._sessions() as session:
task = await self._lookup(session, owner, task_id)
admitted = self._admitted_run(task_id, CoordinationInputRow.sequence).scalar_subquery()
query = select(CoordinationInputRow, admitted).where(CoordinationInputRow.task_id == task_id, CoordinationInputRow.sequence > after).order_by(CoordinationInputRow.sequence).limit(limit)
return [
{
"sequence": row.sequence,
"content": row.content,
"created_at": row.created_at,
"source": INPUT_SOURCES.get(row.request_id.partition(":")[0], "unknown"),
"status": "admitted" if run_id else "pending" if task.status in (TaskStatus.QUEUED, TaskStatus.RUNNING) else "not_admitted",
"run_id": run_id,
}
for row, run_id in (await session.execute(query)).all()
]
async def input_receipts(self, owner, task_id):
async with self._sessions() as session:
task = await self._lookup(session, owner, task_id)
consuming = self._admitted_run(task_id, CoordinationInputRow.sequence).scalar_subquery()
query = select(CoordinationInputRow, consuming).filter(CoordinationInputRow.task_id == task_id, CoordinationInputRow.request_id.startswith("run-input:")).order_by(CoordinationInputRow.sequence.desc()).limit(32)
results = [(row.to_dict(), run) for row, run in (await session.execute(query)).all() if row.request_id.count(":") >= 2]
execution = await session.scalar(self._latest(task_id))
active = execution.run_id if task.status == TaskStatus.RUNNING and execution and execution.status == "running" else None
return list(reversed(results)), active, task.status in (TaskStatus.CANCELLING, TaskStatus.CANCELLED)
async def run_for_input(self, owner, task_id, sequence):
async with self._sessions() as session:
task = await self._lookup(session, owner, task_id)
run_id = await session.scalar(self._admitted_run(task_id, sequence))
if run_id is None and task.status not in (TaskStatus.RUNNING, TaskStatus.QUEUED):
raise TaskConflict("Task stopped before this input could run")
return run_id
async def cancellations(self, owner, task_id):
async with self._write() as session:
task = await self._lock_tree(session, owner, task_id)
ids = [row.id for row in await self._subtree(session, task) if row.status == TaskStatus.CANCELLING]
rows = await session.execute(
select(Assignment.thread_id, Execution.run_id)
.join(Execution, (Assignment.task_id == Execution.task_id) & (Assignment.epoch == Execution.epoch))
.where(Assignment.task_id.in_(ids), Execution.run_id.is_not(None), Execution.status == "running")
)
return list(rows)
async def active_batches(self, owner, task_id):
from deerflow.persistence.subagent_batches.model import SubagentBatchRow
async with self._write() as session:
task = await self._lock_tree(session, owner, task_id)
ids = [row.id for row in await self._subtree(session, task) if row.status == TaskStatus.CANCELLING]
threads = select(Assignment.thread_id).where(Assignment.task_id.in_(ids))
query = select(SubagentBatchRow.thread_id, SubagentBatchRow.id).where(SubagentBatchRow.user_id == owner, SubagentBatchRow.thread_id.in_(threads), SubagentBatchRow.status.in_(("queued", "running", "paused")))
return (await session.execute(query)).all()

View File

@ -0,0 +1,56 @@
"""Bind coordination tasks to worker threads and native runs."""
import sqlalchemy as sa
from alembic import op
revision = "0057_coordination_workers"
down_revision = "0056_coordination_task_state"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("coord_assignments"):
op.create_table(
"coord_assignments",
sa.Column("task_id", sa.String(64), sa.ForeignKey("coord_tasks.id"), primary_key=True),
sa.Column("epoch", sa.BigInteger(), primary_key=True),
sa.Column("thread_id", sa.String(64), nullable=False, unique=True),
sa.Column("assistant_id", sa.String(128), nullable=False),
)
if not inspector.has_table("coord_executions"):
op.create_table(
"coord_executions",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("task_id", sa.String(64), nullable=False),
sa.Column("epoch", sa.BigInteger(), nullable=False),
sa.Column("turn", sa.BigInteger(), nullable=False),
sa.Column("resume_key", sa.String(200)),
sa.Column("covered_seq", sa.BigInteger(), nullable=False),
sa.Column("frozen_input", sa.JSON(), nullable=False),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("run_id", sa.String(64), unique=True),
sa.ForeignKeyConstraint(["task_id", "epoch"], ["coord_assignments.task_id", "coord_assignments.epoch"]),
sa.UniqueConstraint("task_id", "turn", name="uq_coord_execution_turn"),
sa.UniqueConstraint("task_id", "resume_key", name="uq_coord_execution_resume"),
)
if not inspector.has_table("coord_artifacts"):
op.create_table(
"coord_artifacts",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("execution_id", sa.String(64), sa.ForeignKey("coord_executions.id"), nullable=False),
sa.Column("path", sa.Text(), nullable=False),
sa.Column("sha256", sa.String(64), nullable=False),
sa.Column("size", sa.BigInteger(), nullable=False),
sa.Column("content", sa.LargeBinary(), nullable=False),
)
if "ix_coord_artifacts_execution_id" not in {idx["name"] for idx in inspector.get_indexes("coord_artifacts")}:
op.create_index("ix_coord_artifacts_execution_id", "coord_artifacts", ["execution_id"])
def downgrade() -> None:
for table in ("coord_artifacts", "coord_executions", "coord_assignments"):
if sa.inspect(op.get_bind()).has_table(table):
op.drop_table(table)

View File

@ -24,7 +24,7 @@ from deerflow.persistence.channel_connections.model import (
from deerflow.persistence.channel_leases.model import ChannelLeaseRow
from deerflow.persistence.channel_runtime_configs.model import ChannelRuntimeConfigRow
from deerflow.persistence.channel_thread_bindings.model import ChannelThreadBindingRow
from deerflow.persistence.coordination.model import CoordinationInputRow, CoordinationTaskRow
from deerflow.persistence.coordination.model import CoordinationArtifactRow, CoordinationAssignmentRow, CoordinationExecutionRow, CoordinationInputRow, CoordinationTaskRow
from deerflow.persistence.extension_leases.model import ExtensionExecutionLeaseRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.login_throttle.model import LoginThrottleRow
@ -62,6 +62,9 @@ __all__ = [
"ChannelOAuthStateRow",
"ChannelRuntimeConfigRow",
"ChannelThreadBindingRow",
"CoordinationArtifactRow",
"CoordinationAssignmentRow",
"CoordinationExecutionRow",
"CoordinationInputRow",
"CoordinationTaskRow",
"FeedbackRow",

View File

@ -1200,6 +1200,11 @@ class RunRepository(RunStore):
if existing is not None:
raise RunIdempotencyConflict(self._row_to_dict(existing))
from deerflow.persistence.coordination.workers import THREAD_PREFIX, bind_admitted_worker
if thread_id.startswith(THREAD_PREFIX) and values["operation_kind"] == "run":
await bind_admitted_worker(session, thread_id, resolved_user_id, idempotency_key, run_id)
if multitask_strategy in ("interrupt", "rollback"):
stmt = (
select(RunRow)

View File

@ -773,14 +773,13 @@ def test_timeout_runs_inside_the_machine(installed) -> None:
assert call["timeout"] > 3
@posix_only
def test_timed_out_command_keeps_partial_output_and_exit_124(installed) -> None:
_, fake = installed
fake.local_shell = True
# Simulate a remote timeout after the command has already produced output.
fake.exec_override = lambda *_: _Result(124, "before")
output = _box(fake).execute_command("echo before; sleep 30; echo after", timeout=1)
assert output.startswith("before")
assert "after" not in output
assert output.endswith("Exit Code: 124")
assert fake.execs[-1]["argv"] == ["timeout", "-k", "5", "1", "bash", "-lc", "echo before; sleep 30; echo after"]
assert output == "before\nExit Code: 124"
@posix_only

View File

@ -175,13 +175,13 @@ async def test_coordination_migration_matches_model_and_downgrades(database):
engine, _, schema = database
config = bootstrap._get_alembic_config(engine, postgres_schema=schema)
await asyncio.to_thread(command.upgrade, config, "0055_thread_shares")
await asyncio.to_thread(command.upgrade, config, "0056_coordination_task_state")
await asyncio.to_thread(command.upgrade, config, "0057_coordination_workers")
async with engine.connect() as conn:
def diff(sync):
context = MigrationContext.configure(
sync,
opts={"compare_type": True, "include_object": lambda _, name, kind, *_args: kind != "table" or name in {"coord_tasks", "coord_inputs"}},
opts={"compare_type": True, "include_object": lambda _, name, kind, *_args: kind != "table" or name in {"coord_tasks", "coord_inputs", "coord_assignments", "coord_executions", "coord_artifacts"}},
)
return compare_metadata(context, Base.metadata)
@ -193,6 +193,56 @@ async def test_coordination_migration_matches_model_and_downgrades(database):
async with engine.connect() as conn:
tables = await conn.run_sync(lambda sync: set(inspect(sync).get_table_names()))
assert "threads_meta" in tables
assert not {"coord_tasks", "coord_inputs"} & tables
await asyncio.to_thread(command.upgrade, config, "0056_coordination_task_state")
assert not {"coord_tasks", "coord_inputs", "coord_assignments", "coord_executions", "coord_artifacts"} & tables
await asyncio.to_thread(command.upgrade, config, "0057_coordination_workers")
assert (await repo.create("alice", request_id="root", objective="After re-upgrade"))["status"] == TaskStatus.QUEUED
@pytest.mark.asyncio
async def test_worker_admission_is_atomic_with_cancellation_and_replay(database):
from deerflow.persistence.coordination.workers import RUN_KEY_PREFIX, CoordinationWorkerRepository
from deerflow.persistence.run import RunRepository
from deerflow.runtime.runs.manager import ConflictError
engine, dialect, schema = database
await bootstrap.bootstrap_schema(engine, backend=dialect, postgres_schema=schema)
sessions = async_sessionmaker(engine)
workers = CoordinationWorkerRepository(sessions)
runs = RunRepository(sessions)
task = await workers.create("alice", request_id="dispatch", objective="Review")
await workers.append_input("alice", task["id"], request_id="input:follow-up", content="Check the result")
first, retry = await asyncio.gather(*(workers.prepare("alice", task["id"], "lead_agent") for _ in range(2)))
assert first == retry
worker, execution, project = first
assert project is None
kwargs = dict(thread_id=worker["thread_id"], owner_worker_id="gateway", user_id="alice", lease_expires_at=None)
key = RUN_KEY_PREFIX + execution["id"]
with pytest.raises(ConflictError):
await runs.create_thread_operation_atomic("unmanaged", **kwargs, idempotency_key="client-key")
with pytest.raises(TaskNotFound):
await workers.view("bob", task["id"])
with pytest.raises(TaskNotFound):
await workers.list_artifacts("bob", task["id"], limit=1, offset=0)
with pytest.raises(TaskNotFound):
await workers.list_worker_inputs("bob", task["id"], after=0, limit=1)
(pending,) = await workers.list_worker_inputs("alice", task["id"], after=0, limit=1)
assert (pending["source"], pending["status"], pending["run_id"]) == ("task_api", "pending", None)
assert await workers.list_artifacts("alice", task["id"], limit=1, offset=0) == []
current = await workers.get("alice", task["id"])
admitted, cancelled = await asyncio.gather(
runs.create_thread_operation_atomic("native", **kwargs, idempotency_key=key),
workers.request_cancel("alice", task["id"], expected_revision=current["revision"]),
return_exceptions=True,
)
assert cancelled["status"] == TaskStatus.CANCELLING
stopped = await workers.cancellations("alice", task["id"])
(receipt,) = await workers.list_worker_inputs("alice", task["id"], after=0, limit=1)
if isinstance(admitted, ConflictError):
assert stopped == [] and await runs.get("native", user_id="alice") is None
assert (receipt["status"], receipt["run_id"]) == ("not_admitted", None)
else:
assert not isinstance(admitted, BaseException) and stopped == [(worker["thread_id"], "native")]
assert (await workers.view("alice", task["id"]))["execution"]["run_id"] == "native"
assert (receipt["status"], receipt["run_id"]) == ("admitted", "native")
with pytest.raises(ConflictError):
await runs.create_thread_operation_atomic("late", **kwargs, idempotency_key=RUN_KEY_PREFIX + uuid4().hex)

View File

@ -161,7 +161,9 @@ async def runtime(host, monkeypatch, tmp_path):
prompt = state["messages"][-1].content
if prompt == "wait":
entered.set()
await asyncio.Event().wait()
await getattr(request.app.state, "release_worker", asyncio.Event()).wait()
if present := getattr(request.app.state, "present_worker_report", None):
await present(prompt, runtime)
if prompt == "approve":
prompt = interrupt({"question": "Continue?"})
return {"messages": [AIMessage(content=f"answer:{prompt}")]}
@ -776,3 +778,208 @@ async def test_restored_delegated_handle_is_plugin_scoped_with_extension_origin(
assert runs._origin == {"kind": "extension", "namespace": "workflows"}
with pytest.raises(AgentRunError, match="already bound"):
runs.for_plugin("other")
@pytest_asyncio.fixture
async def worker_gateway(runtime, tmp_path):
import asyncio
from httpx import ASGITransport, AsyncClient
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.coordination_workers import CoordinationWorkers
from app.gateway.routers import coordination, thread_runs
from deerflow.persistence.bootstrap import bootstrap_schema
from deerflow.persistence.coordination.workers import CoordinationWorkerRepository
from deerflow.persistence.run import RunRepository
from deerflow.runtime import RunManager
from deerflow.runtime.user_context import reset_current_user, set_current_user
runs, state, request, entered, _ = runtime
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'workers.db'}")
await bootstrap_schema(engine, backend="sqlite")
factory = async_sessionmaker(engine)
state.run_store = RunRepository(factory)
await state.run_manager.shutdown()
state.run_manager = RunManager(store=state.run_store)
workers = state.coordination_workers = CoordinationWorkers(CoordinationWorkerRepository(factory))
state.release_worker = asyncio.Event()
request.app.include_router(coordination.router)
request.app.include_router(thread_runs.router)
async def authenticated(scope, receive, send):
token = set_current_user(request.state.user)
try:
await request.app({**scope, "state": dict(request.scope["state"])}, receive, send)
finally:
reset_current_user(token)
try:
async with AsyncClient(transport=ASGITransport(app=authenticated), base_url="http://gateway") as client:
yield client, runs, state, entered, str(request.state.user.id)
finally:
await workers.close()
await state.run_manager.shutdown()
await engine.dispose()
async def _wait_for_worker(client, task_id, section, field, value):
import asyncio
async with asyncio.timeout(12):
while True:
response = await client.get(f"/api/coordination/tasks/{task_id}")
response.raise_for_status()
view = response.json()
if view[section][field] == value:
return view
await asyncio.sleep(0.05)
@pytest.mark.asyncio
@pytest.mark.parametrize("followup_report", [None, b"revised report"], ids=["text-followup", "same-name-file"])
async def test_managed_worker_routes_followups_and_retains_the_producing_runs_file(worker_gateway, followup_report):
import asyncio
from uuid import uuid4
from langchain_core.messages import AIMessage, ToolMessage
from langgraph.types import Command
from deerflow.config.paths import get_paths
from deerflow.persistence.coordination.sql import TaskNotFound
client, runs, state, entered, owner = worker_gateway
async def present_report(prompt, runtime):
assert runtime.context["subagent_enabled"] is True
report = b"original report" if prompt == "wait" else followup_report
if report is not None:
outputs = get_paths().sandbox_outputs_dir(runtime.context["thread_id"], user_id=owner)
await asyncio.to_thread(outputs.mkdir, parents=True, exist_ok=True)
await asyncio.to_thread((outputs / "report.txt").write_bytes, report)
journal = runtime.context["__run_journal"]
journal._remember_current_run_tool_calls(AIMessage(content="", tool_calls=[{"id": "present", "name": "present_files", "args": {}}]), caller="lead_agent")
journal.on_tool_end(Command(update={"artifacts": ["/mnt/user-data/outputs/report.txt"], "messages": [ToolMessage("Presented", tool_call_id="present")]}), run_id=uuid4())
state.present_worker_report = present_report
url = "/api/coordination/tasks"
body = {"request_id": "dispatch", "objective": "wait"}
first = (await client.post(url, json=body)).raise_for_status().json()
task_id, thread, run = first["task"]["id"], first["assignment"]["thread_id"], first["execution"]["run_id"]
assert first["task"]["project_id"] is None and (await client.post(url, json=body)).json()["execution"]["run_id"] == run
await asyncio.wait_for(entered.wait(), 10)
thread_url = f"/api/threads/{thread}"
inbox_url = f"{url}/{task_id}/inputs"
(await client.post(inbox_url, json={"request_id": "task-input", "content": "task follow-up"})).raise_for_status()
followup = {"request_id": "fix", "mode": "queue", "text": "new requirement"}
receipt = (await client.post(f"{thread_url}/runs/{run}/inputs", json=followup)).raise_for_status().json()
assert receipt["status"] == "pending"
assert (await client.post(f"{thread_url}/runs/{run}/inputs", json=followup)).json() == receipt
direct = asyncio.create_task(client.post(f"{thread_url}/runs", json={"input": {"messages": [{"type": "human", "content": [{"type": "text", "text": "final instruction"}]}]}}))
try:
await _wait_for_worker(client, task_id, "task", "latest_input_seq", 3)
pending = (await client.get(inbox_url)).raise_for_status().json()
assert [(item["sequence"], item["source"], item["content"], item["status"]) for item in pending] == [
(1, "task_api", "task follow-up", "pending"),
(2, "run_input", "new requirement", "pending"),
(3, "worker_chat", "final instruction", "pending"),
]
assert (await client.get(inbox_url, params={"after": 1, "limit": 1})).raise_for_status().json() == pending[1:2]
assert (await client.get(f"{url}/missing-task/inputs")).status_code == 404
finally:
state.release_worker.set()
(await direct).raise_for_status()
result = await _wait_for_worker(client, task_id, "task", "status", "succeeded")
final_run = result["execution"]["run_id"]
assert final_run != run and result["task"]["result_ref"] == f"{thread_url}/runs/{final_run}/messages"
assert (await direct).raise_for_status().json()["run_id"] == final_run
assert (await runs.get_state(thread_id=thread))["values"]["messages"][-1]["content"] == "answer:final instruction"
assert (await client.get(f"{thread_url}/inputs")).json()["items"][0]["run_id"] == final_run
admitted = (await client.get(inbox_url)).raise_for_status().json()
assert [(item["run_id"], item["status"]) for item in admitted] == [(final_run, "admitted")] * 3
artifact_list = f"{url}/{task_id}/artifacts"
artifacts = (await client.get(artifact_list)).raise_for_status().json()
expected_versions = [(run, 1, b"original report")]
if followup_report:
expected_versions.append((final_run, 2, followup_report))
assert [(item["run_id"], item["turn"]) for item in artifacts] == [(producer, turn) for producer, turn, _ in expected_versions]
assert {item["source_path"] for item in artifacts} == {"/mnt/user-data/outputs/report.txt"}
assert len({item["url"] for item in artifacts}) == len(expected_versions)
assert result["execution"]["artifacts"] == (artifacts[1:] if followup_report else [])
assert (await client.get(artifact_list, params={"limit": 1, "offset": 1})).raise_for_status().json() == artifacts[1:]
assert (await client.get(f"{url}/missing-task/artifacts")).status_code == 404
await asyncio.to_thread((get_paths().sandbox_outputs_dir(thread, user_id=owner) / "report.txt").unlink)
for artifact, (_, _, content) in zip(artifacts, expected_versions, strict=True):
assert (await client.get(artifact["url"])).raise_for_status().content == content
with pytest.raises(TaskNotFound):
await state.coordination_workers.repo.read_artifact("another-user", task_id, artifacts[0]["id"])
@pytest.mark.asyncio
async def test_managed_worker_requires_explicit_resume_and_fences_cancellation(worker_gateway):
import asyncio
client, runs, _, entered, _ = worker_gateway
url = "/api/coordination/tasks"
paused = (await client.post(url, json={"request_id": "pause", "objective": "approve"})).raise_for_status().json()
task_id = paused["task"]["id"]
await _wait_for_worker(client, task_id, "execution", "status", "waiting_for_input")
target = f"/api/threads/{paused['assignment']['thread_id']}/runs"
reply = {"json": {"command": {"resume": "confirmed"}}, "headers": {"Idempotency-Key": "reply"}}
answered = (await client.post(target, **reply)).raise_for_status().json()
result = await _wait_for_worker(client, task_id, "task", "status", "succeeded")
assert result["task"]["result_ref"].endswith(f"/{answered['run_id']}/messages")
assert (await client.post(target, **reply)).raise_for_status().json()["run_id"] == answered["run_id"]
other = (await client.post(url, json={"request_id": "cancel", "objective": "wait"})).raise_for_status().json()
await asyncio.wait_for(entered.wait(), 10)
task_id, thread = other["task"]["id"], other["assignment"]["thread_id"]
cancelled = (await client.post(f"{url}/{task_id}/cancel", json={"expected_revision": other["task"]["revision"]})).raise_for_status().json()
assert cancelled["status"] == "cancelled"
assert (await runs.get(thread_id=thread, run_id=other["execution"]["run_id"])).status == "interrupted"
assert (await client.post(f"/api/threads/{thread}/runs", json={"input": {"messages": [{"role": "user", "content": "too late"}]}})).status_code == 409
@pytest.mark.asyncio
async def test_managed_worker_cancel_returns_cancelling_and_finishes_in_background(worker_gateway, monkeypatch):
import asyncio
from app.gateway import coordination_workers
client, runs, _, entered, _ = worker_gateway
started, release = asyncio.Event(), asyncio.Event()
calls = []
request_stop = type(runs).cancel
async def paused_stop(self, *, thread_id, run_id):
calls.append(run_id)
started.set()
await release.wait()
if len(calls) == 1:
raise AgentRunError(503, "Run is temporarily unavailable")
await request_stop(self, thread_id=thread_id, run_id=run_id)
monkeypatch.setattr(type(runs), "cancel", paused_stop)
monkeypatch.setattr(coordination_workers, "CANCEL_RESPONSE_TIMEOUT", 0.05, raising=False)
url = "/api/coordination/tasks"
first = (await client.post(url, json={"objective": "wait", "request_id": "background-cancel"})).raise_for_status().json()
task_id, run_id = first["task"]["id"], first["execution"]["run_id"]
await asyncio.wait_for(entered.wait(), 10)
cancel_url, body = f"{url}/{task_id}/cancel", {"expected_revision": first["task"]["revision"]}
pending = asyncio.create_task(client.post(cancel_url, json=body))
try:
await asyncio.wait_for(started.wait(), 5)
done, _ = await asyncio.wait({pending}, timeout=3)
assert pending in done, "Cancellation response waited for the blocked run"
response = pending.result()
assert response.status_code == 202 and response.json()["status"] == "cancelling"
assert (await client.post(cancel_url, json=body)).json()["status"] == "cancelling"
assert calls == [run_id]
assert (await client.post(f"{url}/{task_id}/inputs", json={"request_id": "late", "content": "too late"})).status_code == 409
finally:
release.set()
await pending
final = await _wait_for_worker(client, task_id, "task", "status", "cancelled")
assert calls == [run_id, run_id]
assert final["execution"]["status"] == "cancelled"
assert (await runs.get(thread_id=first["assignment"]["thread_id"], run_id=run_id)).status == "interrupted"

View File

@ -44,9 +44,10 @@ pytestmark = pytest.mark.asyncio
async def test_0047_remains_in_the_single_migration_chain():
assert bootstrap._get_head_revision() == "0056_coordination_task_state"
assert bootstrap._get_head_revision() == "0057_coordination_workers"
script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR))
assert script.get_heads() == ["0056_coordination_task_state"]
assert script.get_heads() == ["0057_coordination_workers"]
assert script.get_revision("0057_coordination_workers").down_revision == "0056_coordination_task_state"
assert script.get_revision("0056_coordination_task_state").down_revision == "0055_thread_shares"
assert script.get_revision("0055_thread_shares").down_revision == "0054_project_memory"
assert script.get_revision("0054_project_memory").down_revision == "0053_tool_approvals"