mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
feat(coordination): run tasks on managed worker threads (#8387)
* feat(coordination): run tasks on managed worker threads * fix(coordination): expose task artifact history * test(boxd): make partial timeout output check deterministic * fix(coordination): expose task inbox and bound cancellation waits
This commit is contained in:
parent
b4b489d7f2
commit
50d04ddf71
@ -31,6 +31,7 @@ from app.gateway.routers import (
|
||||
channel_connections,
|
||||
channels,
|
||||
console,
|
||||
coordination,
|
||||
features,
|
||||
feedback,
|
||||
github_webhooks,
|
||||
@ -1358,6 +1359,7 @@ This gateway provides runtime endpoints for agent runs plus custom endpoints for
|
||||
|
||||
# Durable MCP tasks are scoped to their owning thread.
|
||||
app.include_router(mcp_tasks.router)
|
||||
app.include_router(coordination.router)
|
||||
app.include_router(subagent_batches.router)
|
||||
|
||||
# Memory API is mounted at /api/memory
|
||||
|
||||
238
backend/app/gateway/coordination_workers.py
Normal file
238
backend/app/gateway/coordination_workers.py
Normal file
@ -0,0 +1,238 @@
|
||||
"""Gateway execution of durable coordination tasks using caller-bound AgentRuns."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from hashlib import sha256
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow_extension_api.agent_runs import AgentRunError
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.gateway.deps import get_agent_runs
|
||||
from deerflow.persistence.coordination.model import TaskStatus
|
||||
from deerflow.persistence.coordination.sql import TaskConflict, TaskNotFound
|
||||
from deerflow.persistence.coordination.workers import THREAD_PREFIX
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
CANCEL_RESPONSE_TIMEOUT = 5
|
||||
|
||||
|
||||
class CoordinationWorkers:
|
||||
def __init__(self, repository):
|
||||
self.repo = repository
|
||||
self._drivers = {}
|
||||
self._wakeups = set()
|
||||
self._cancellations = {}
|
||||
|
||||
async def close(self):
|
||||
jobs = [*self._drivers.values(), *self._cancellations.values()]
|
||||
for job in jobs:
|
||||
job.cancel()
|
||||
await asyncio.gather(*jobs, return_exceptions=True)
|
||||
|
||||
async def dispatch(self, runs, owner, task_id, assistant_id):
|
||||
turn = await self.repo.prepare(owner, task_id, assistant_id)
|
||||
if turn is not None:
|
||||
await self._admit(runs, owner, task_id, turn)
|
||||
self.watch(runs, owner, task_id)
|
||||
return await self.repo.view(owner, task_id)
|
||||
|
||||
async def _admit(self, runs, owner, task_id, turn):
|
||||
worker, execution, project = turn
|
||||
thread = worker["thread_id"]
|
||||
if execution["run_id"] is None:
|
||||
try:
|
||||
await runs.create_thread(thread_id=thread, assistant_id=worker["assistant_id"], project_id=project)
|
||||
admitted = await runs.start_coordination_worker(task_id=task_id, execution_id=execution["id"])
|
||||
return admitted.run_id
|
||||
except AgentRunError as exc:
|
||||
if exc.status_code in (400, 403, 404, 422):
|
||||
await self.repo.settle(owner, task_id, execution["id"], None, "admission_failed", [])
|
||||
raise
|
||||
return execution["run_id"]
|
||||
|
||||
def watch(self, runs, owner, task_id):
|
||||
key = (owner, task_id)
|
||||
if (existing := self._drivers.get(key)) is not None and not existing.done():
|
||||
self._wakeups.add(key)
|
||||
return existing
|
||||
|
||||
async def drive():
|
||||
delay = 1
|
||||
while True:
|
||||
self._wakeups.discard(key)
|
||||
try:
|
||||
await self._drive(runs, owner, task_id)
|
||||
except AgentRunError as exc:
|
||||
if exc.status_code < 500:
|
||||
raise
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(30, delay * 2)
|
||||
continue
|
||||
if key not in self._wakeups:
|
||||
return
|
||||
|
||||
task = asyncio.create_task(drive())
|
||||
self._drivers[key] = task
|
||||
|
||||
def finished(job):
|
||||
if self._drivers.get(key) is job:
|
||||
self._drivers.pop(key)
|
||||
self._wakeups.discard(key)
|
||||
if not job.cancelled() and (error := job.exception()) is not None:
|
||||
logger.warning("Coordination worker observation stopped for task %s (%s)", task_id, type(error).__name__)
|
||||
|
||||
task.add_done_callback(finished)
|
||||
return task
|
||||
|
||||
async def _drive(self, runs, owner, task_id):
|
||||
while (turn := await self.repo.prepare(owner, task_id)) is not None:
|
||||
worker, execution, _project = turn
|
||||
if execution["status"] == "waiting_for_input":
|
||||
return
|
||||
if execution["run_id"] is None:
|
||||
await self._admit(runs, owner, task_id, turn)
|
||||
continue
|
||||
thread = worker["thread_id"]
|
||||
try:
|
||||
run = await runs.wait_coordination_worker(thread_id=thread, run_id=execution["run_id"], timeout=60)
|
||||
except TimeoutError:
|
||||
continue
|
||||
artifacts = []
|
||||
status = run.status
|
||||
if status == "success" and not run.stop_reason:
|
||||
try:
|
||||
artifacts = await runs.coordination_artifacts(thread_id=thread, run_id=run.run_id)
|
||||
except AgentRunError as exc:
|
||||
if exc.status_code >= 500:
|
||||
raise
|
||||
status = "delivery_failed"
|
||||
elif status == "success":
|
||||
status = "stopped"
|
||||
await self.repo.settle(owner, task_id, execution["id"], run.run_id, status, artifacts)
|
||||
|
||||
async def cancel(self, runs, owner, task_id, revision):
|
||||
task = await self.repo.request_cancel(owner, task_id, expected_revision=revision)
|
||||
if task["status"] == TaskStatus.CANCELLED:
|
||||
return task
|
||||
key = (owner, task_id)
|
||||
job = self._cancellations.get(key)
|
||||
if job is None or job.done():
|
||||
job = asyncio.create_task(self._finish_cancellation(runs, owner, task_id, task["execution_epoch"]))
|
||||
self._cancellations[key] = job
|
||||
|
||||
def finished(completed):
|
||||
if self._cancellations.get(key) is completed:
|
||||
self._cancellations.pop(key)
|
||||
if not completed.cancelled() and (error := completed.exception()) is not None:
|
||||
logger.warning("Coordination cancellation stopped for task %s (%s)", task_id, type(error).__name__)
|
||||
|
||||
job.add_done_callback(finished)
|
||||
# asyncio.wait leaves the tracked job running after the response deadline.
|
||||
done, _ = await asyncio.wait({job}, timeout=CANCEL_RESPONSE_TIMEOUT)
|
||||
return job.result() if done else task
|
||||
|
||||
async def _finish_cancellation(self, runs, owner, task_id, epoch):
|
||||
delay = 1
|
||||
while True:
|
||||
try:
|
||||
await self._stop_subtree(runs, owner, task_id)
|
||||
return await self.repo.confirm_cancel(owner, task_id, execution_epoch=epoch)
|
||||
except AgentRunError as exc:
|
||||
if exc.status_code < 500:
|
||||
raise
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(30, delay * 2)
|
||||
|
||||
async def _stop_subtree(self, runs, owner, task_id):
|
||||
active = await self.repo.cancellations(owner, task_id)
|
||||
for thread, run_id in active:
|
||||
run = await runs.get(thread_id=thread, run_id=run_id)
|
||||
if run.status in ("pending", "running"):
|
||||
try:
|
||||
await runs.cancel(thread_id=thread, run_id=run_id)
|
||||
except AgentRunError as exc:
|
||||
if exc.status_code != 409 or (await runs.get(thread_id=thread, run_id=run_id)).status in ("pending", "running"):
|
||||
raise
|
||||
for thread, run_id in active:
|
||||
while True:
|
||||
try:
|
||||
await runs.wait_coordination_worker(thread_id=thread, run_id=run_id, timeout=60)
|
||||
break
|
||||
except TimeoutError:
|
||||
continue
|
||||
for batch in await self.repo.active_batches(owner, task_id):
|
||||
await runs.cancel_worker_batch(thread_id=batch.thread_id, batch_id=batch.id)
|
||||
|
||||
|
||||
def worker_context(request):
|
||||
service = getattr(request.app.state, "coordination_workers", None)
|
||||
if service is None:
|
||||
raise HTTPException(503, "Coordination workers require SQL persistence")
|
||||
runs = get_agent_runs(request)
|
||||
if runs is None:
|
||||
raise HTTPException(403, "Coordination workers require a user session")
|
||||
return service, str(request.state.user.id), runs.for_plugin("native-coordination")
|
||||
|
||||
|
||||
def worker_error(exc):
|
||||
if isinstance(exc, TaskNotFound):
|
||||
return HTTPException(404, "Task not found")
|
||||
if isinstance(exc, AgentRunError):
|
||||
return HTTPException(exc.status_code, str(exc) if exc.status_code < 500 else "Worker host unavailable")
|
||||
return HTTPException(409, str(exc))
|
||||
|
||||
|
||||
async def managed_task(request, thread_id):
|
||||
if not thread_id.startswith(THREAD_PREFIX):
|
||||
return None
|
||||
service = getattr(request.app.state, "coordination_workers", None)
|
||||
return await service.repo.task_for_thread(thread_id) if service is not None else None
|
||||
|
||||
|
||||
async def direct_worker_run(request, task_id, body, key):
|
||||
from app.gateway.deps import get_run_manager
|
||||
|
||||
service, owner, runs = worker_context(request)
|
||||
if getattr(request.state, "run_origin", None) or body.checkpoint or body.checkpoint_id or body.conversation_references or body.multitask_strategy != "reject":
|
||||
raise HTTPException(409, "Use a task input to continue this managed worker")
|
||||
try:
|
||||
if body.command is not None:
|
||||
if set(body.command) != {"resume"} or body.command["resume"] is None or body.input:
|
||||
raise HTTPException(422, "An explicit interrupt response is required")
|
||||
view = await service.repo.view(owner, task_id)
|
||||
current = view["execution"]
|
||||
if current and current["status"] == "running" and current["run_id"]:
|
||||
native = await runs.get(thread_id=view["assignment"]["thread_id"], run_id=current["run_id"])
|
||||
if native.status == "waiting_for_input":
|
||||
await service.repo.settle(owner, task_id, current["id"], native.run_id, native.status, [])
|
||||
turn = await service.repo.resume(owner, task_id, key=key or uuid4().hex, value=body.command["resume"])
|
||||
run_id = await service._admit(runs, owner, task_id, turn)
|
||||
service.watch(runs, owner, task_id)
|
||||
else:
|
||||
messages = (body.input or {}).get("messages")
|
||||
if not isinstance(messages, list) or len(messages) != 1 or not isinstance(messages[0], dict):
|
||||
raise HTTPException(422, "Managed workers accept one text message")
|
||||
message = messages[0]
|
||||
content = message.get("content")
|
||||
if isinstance(content, list) and all(isinstance(part, dict) and part.get("type") == "text" and isinstance(part.get("text"), str) for part in content):
|
||||
content = "".join(part["text"] for part in content)
|
||||
if message.get("role", message.get("type")) not in ("user", "human") or not isinstance(content, str) or not 0 < len(content.strip()) <= 16384:
|
||||
raise HTTPException(422, "Managed workers accept one text message")
|
||||
identifier = key or message.get("id") or uuid4().hex
|
||||
if not isinstance(identifier, str):
|
||||
raise HTTPException(422, "Message ID must be a string")
|
||||
row = await service.repo.append_input(owner, task_id, request_id="chat:" + sha256(identifier.encode()).hexdigest(), content=content)
|
||||
driver = service.watch(runs, owner, task_id)
|
||||
# Streaming and wait endpoints must observe the Run that consumed this input.
|
||||
while not (run_id := await service.repo.run_for_input(owner, task_id, row["sequence"])):
|
||||
if driver.done():
|
||||
driver.result()
|
||||
raise HTTPException(409, "Input queued; the worker is waiting for an explicit interrupt response")
|
||||
await asyncio.sleep(0.25)
|
||||
record = await get_run_manager(request).get(run_id, user_id=owner)
|
||||
if record is None:
|
||||
raise HTTPException(503, "Worker run is unavailable")
|
||||
return record
|
||||
except (TaskNotFound, TaskConflict, AgentRunError) as exc:
|
||||
raise worker_error(exc) from exc
|
||||
@ -963,10 +963,14 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
|
||||
stack.callback(reset_login_throttle_store)
|
||||
|
||||
if sf is not None:
|
||||
from app.gateway.coordination_workers import CoordinationWorkers
|
||||
from deerflow.persistence.coordination.workers import CoordinationWorkerRepository
|
||||
from deerflow.persistence.feedback import FeedbackRepository
|
||||
from deerflow.persistence.personal_access_tokens import PersonalAccessTokenRepository
|
||||
from deerflow.persistence.run import RunRepository
|
||||
|
||||
app.state.coordination_workers = CoordinationWorkers(CoordinationWorkerRepository(sf))
|
||||
stack.push_async_callback(app.state.coordination_workers.close)
|
||||
app.state.run_store = RunRepository(sf)
|
||||
app.state.feedback_repo = FeedbackRepository(sf)
|
||||
from app.gateway.auth.pat import PAT_LAST_USED_WRITE_INTERVAL_SECONDS
|
||||
@ -975,6 +979,7 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
|
||||
else:
|
||||
from deerflow.runtime.runs.store.memory import MemoryRunStore
|
||||
|
||||
app.state.coordination_workers = None
|
||||
app.state.run_store = MemoryRunStore()
|
||||
app.state.feedback_repo = None
|
||||
# Memory backend has no durable PAT store, so Bearer credentials
|
||||
|
||||
@ -305,11 +305,11 @@ class _BoundAgentRuns:
|
||||
|
||||
return await asyncio.to_thread(snapshot)
|
||||
|
||||
async def create_thread(self, *, assistant_id="lead_agent", thread_id=None, metadata=None) -> str:
|
||||
async def create_thread(self, *, assistant_id="lead_agent", thread_id=None, metadata=None, project_id=None) -> str:
|
||||
from app.gateway.routers.threads import ThreadCreateRequest, create_thread
|
||||
|
||||
async with self._request("threads:write") as request:
|
||||
response = await create_thread(ThreadCreateRequest(assistant_id=assistant_id, thread_id=thread_id, metadata=_json_object(metadata) or {}), request)
|
||||
response = await create_thread(ThreadCreateRequest(assistant_id=assistant_id, thread_id=thread_id, project_id=project_id, metadata=_json_object(metadata) or {}), request)
|
||||
return response.thread_id
|
||||
|
||||
async def _start(self, *, thread_id, input=None, resume=None, context=None, idempotency_key=None, execution_fence=None, artifact_imports=None):
|
||||
@ -350,6 +350,93 @@ class _BoundAgentRuns:
|
||||
async def start(self, *, thread_id, input, context=None, idempotency_key=None, execution_fence=None, artifact_imports=None) -> AgentRun:
|
||||
return await self._start(thread_id=thread_id, input=input, context=context, idempotency_key=idempotency_key, execution_fence=execution_fence, artifact_imports=artifact_imports)
|
||||
|
||||
async def start_coordination_worker(self, *, task_id, execution_id):
|
||||
from app.gateway.run_models import RunCreateRequest
|
||||
from app.gateway.services import start_run
|
||||
from deerflow.persistence.coordination.sql import TaskConflict, TaskNotFound
|
||||
from deerflow.persistence.coordination.workers import RUN_KEY_PREFIX
|
||||
from deerflow.persistence.thread_meta.base import THREAD_PROJECT_METADATA_KEY
|
||||
|
||||
async with self._request("runs:create") as request:
|
||||
try:
|
||||
worker, frozen, project_id = await request.app.state.coordination_workers.repo.admission(self._user_id, task_id, execution_id)
|
||||
except TaskNotFound as exc:
|
||||
raise AgentRunError(404, "Worker task unavailable") from exc
|
||||
except TaskConflict as exc:
|
||||
raise AgentRunError(409, str(exc)) from exc
|
||||
thread_id = worker["thread_id"]
|
||||
async with self._request("runs:create", thread_id) as request:
|
||||
record = await request.app.state.thread_store.get(thread_id, user_id=self._user_id)
|
||||
if record is None or record.get("assistant_id") != worker["assistant_id"] or (record.get("metadata") or {}).get(THREAD_PROJECT_METADATA_KEY) != project_id:
|
||||
raise AgentRunError(409, "Worker thread binding changed")
|
||||
body = RunCreateRequest(
|
||||
assistant_id=worker["assistant_id"],
|
||||
input=None if "command" in frozen else frozen,
|
||||
command=frozen.get("command"),
|
||||
context={"subagent_enabled": True},
|
||||
on_disconnect="continue",
|
||||
)
|
||||
request.state.coordination_execution_id = execution_id
|
||||
record = await start_run(body, thread_id, request, require_existing_thread=True, idempotency_key=RUN_KEY_PREFIX + execution_id)
|
||||
return _view(record)
|
||||
|
||||
async def wait_coordination_worker(self, *, thread_id, run_id, timeout=60):
|
||||
"""Observe the committed terminal status, written after the delivery receipt."""
|
||||
from app.gateway.deps import get_run_store
|
||||
|
||||
async with asyncio.timeout(timeout):
|
||||
delay = 0.25
|
||||
while True:
|
||||
async with self._request("runs:read", thread_id) as request:
|
||||
row = await get_run_store(request).get(run_id, user_id=self._user_id)
|
||||
if row is None or row["thread_id"] != thread_id:
|
||||
raise AgentRunError(404, "Worker run unavailable")
|
||||
if row["status"] not in ("pending", "running"):
|
||||
return AgentRun(thread_id, run_id, row["status"], row.get("assistant_id"), row.get("stop_reason"))
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(delay * 2, 4)
|
||||
|
||||
async def cancel_worker_batch(self, *, thread_id, batch_id):
|
||||
from app.gateway.deps import get_subagent_batch_service
|
||||
|
||||
async with self._request("runs:cancel", thread_id) as request:
|
||||
await get_subagent_batch_service(request).cancel_batch(batch_id=batch_id, user_id=self._user_id, continuation_authority=self)
|
||||
|
||||
async def coordination_artifacts(self, *, thread_id, run_id):
|
||||
from app.gateway.artifact_archive import MAX_FILE_BYTES, MAX_FILES, MAX_TOTAL_BYTES
|
||||
from app.gateway.deps import get_run_event_store
|
||||
from app.gateway.extension_artifacts import read_snapshot
|
||||
from app.gateway.routers.artifacts import reserve_artifact_write
|
||||
from deerflow.constants import DELIVERY_PRESENTATION_TOOL_NAMES
|
||||
from deerflow.runtime.runs.manager import ConflictError
|
||||
from deerflow.utils.file_io import await_drained
|
||||
|
||||
async with self._request("runs:read", thread_id) as request:
|
||||
events = await get_run_event_store(request).list_events(thread_id, run_id, event_types=["run.delivery"], limit=2, user_id=self._user_id)
|
||||
if len(events) != 1:
|
||||
raise AgentRunError(409, "Run delivery receipt is unavailable")
|
||||
by_tool = (events[0].get("content") or {}).get("by_tool")
|
||||
if not isinstance(by_tool, dict) or any(not isinstance(paths, list) or any(not isinstance(path, str) for path in paths) for paths in by_tool.values()):
|
||||
raise AgentRunError(409, "Invalid run delivery receipt")
|
||||
paths = dict.fromkeys(path for tool in DELIVERY_PRESENTATION_TOOL_NAMES for path in by_tool.get(tool, []))
|
||||
if not paths:
|
||||
return []
|
||||
if len(paths) > MAX_FILES:
|
||||
raise AgentRunError(422, "Too many task artifacts")
|
||||
artifacts = []
|
||||
remaining = MAX_TOTAL_BYTES
|
||||
async with self._request("threads:read", thread_id) as request:
|
||||
try:
|
||||
async with reserve_artifact_write(request, thread_id, user_id=self._user_id):
|
||||
for path in paths:
|
||||
content = await await_drained(read_snapshot(request, self._user_id, thread_id, path, max_bytes=min(MAX_FILE_BYTES, remaining)))
|
||||
digest = await asyncio.to_thread(hashlib.sha256, content)
|
||||
artifacts.append((path, digest.hexdigest(), content))
|
||||
remaining -= len(content)
|
||||
except ConflictError as exc:
|
||||
raise AgentRunError(503, "Worker artifacts are temporarily busy") from exc
|
||||
return artifacts
|
||||
|
||||
async def start_batch_followup(self, *, thread_id, batch_id, input, context, config, execution_fence, goal_wakeup=None):
|
||||
"""Native single follow-up; the existing SQL admission fences cancellation."""
|
||||
from app.gateway.run_models import RunCreateRequest
|
||||
|
||||
127
backend/app/gateway/routers/coordination.py
Normal file
127
backend/app/gateway/routers/coordination.py
Normal file
@ -0,0 +1,127 @@
|
||||
"""Session-authorized task execution shared by coordination entry points."""
|
||||
|
||||
from hashlib import sha256
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Annotated
|
||||
|
||||
from deerflow_extension_api.agent_runs import AgentRunError
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, Field, StringConstraints
|
||||
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.coordination_workers import worker_context, worker_error
|
||||
from deerflow.persistence.coordination.model import TaskStatus
|
||||
from deerflow.persistence.coordination.sql import TaskConflict, TaskNotFound
|
||||
from deerflow.utils.file_io import await_drained
|
||||
|
||||
router = APIRouter(prefix="/api/coordination/tasks", tags=["coordination"])
|
||||
RequestId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=200)]
|
||||
TaskText = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=16384)]
|
||||
|
||||
|
||||
class CreateWorkerTask(BaseModel):
|
||||
request_id: RequestId
|
||||
objective: TaskText
|
||||
assistant_id: str = Field(default="lead_agent", min_length=1, max_length=128)
|
||||
project_id: str | None = Field(default=None, min_length=1, max_length=64)
|
||||
|
||||
|
||||
class WorkerInput(BaseModel):
|
||||
request_id: RequestId
|
||||
content: TaskText
|
||||
|
||||
|
||||
class CancelTask(BaseModel):
|
||||
expected_revision: int = Field(ge=0)
|
||||
|
||||
|
||||
@router.post("")
|
||||
@require_permission("runs", "create")
|
||||
async def create_worker_task(body: CreateWorkerTask, request: Request):
|
||||
service, owner, runs = worker_context(request)
|
||||
|
||||
async def dispatch():
|
||||
task = await service.repo.create(owner, request_id=body.request_id, objective=body.objective, project_id=body.project_id)
|
||||
return await service.dispatch(runs, owner, task["id"], body.assistant_id)
|
||||
|
||||
try:
|
||||
return await await_drained(dispatch())
|
||||
except (TaskConflict, TaskNotFound, AgentRunError) as exc:
|
||||
raise worker_error(exc) from exc
|
||||
|
||||
|
||||
@router.get("/{task_id}")
|
||||
@require_permission("runs", "read")
|
||||
async def get_worker_task(task_id: str, request: Request):
|
||||
service, owner, _runs = worker_context(request)
|
||||
try:
|
||||
return await service.repo.view(owner, task_id)
|
||||
except TaskNotFound as exc:
|
||||
raise HTTPException(404, "Task not found") from exc
|
||||
|
||||
|
||||
@router.post("/{task_id}/inputs")
|
||||
@require_permission("runs", "create")
|
||||
async def add_worker_input(task_id: str, body: WorkerInput, request: Request):
|
||||
service, owner, runs = worker_context(request)
|
||||
try:
|
||||
item = await service.repo.append_input(owner, task_id, request_id="input:" + sha256(body.request_id.encode()).hexdigest(), content=body.content)
|
||||
service.watch(runs, owner, task_id)
|
||||
return item
|
||||
except (TaskNotFound, TaskConflict) as exc:
|
||||
raise worker_error(exc) from exc
|
||||
|
||||
|
||||
@router.get("/{task_id}/inputs")
|
||||
@require_permission("runs", "read")
|
||||
async def list_worker_inputs(task_id: str, request: Request, after: int = Query(0, ge=0, le=2**63 - 1), limit: int = Query(100, ge=1, le=100)):
|
||||
service, owner, _runs = worker_context(request)
|
||||
try:
|
||||
return await service.repo.list_worker_inputs(owner, task_id, after=after, limit=limit)
|
||||
except TaskNotFound as exc:
|
||||
raise HTTPException(404, "Task not found") from exc
|
||||
|
||||
|
||||
@router.post("/{task_id}/cancel", responses={202: {"description": "Cancellation is still in progress"}})
|
||||
@require_permission("runs", "cancel")
|
||||
async def cancel_worker_task(task_id: str, body: CancelTask, request: Request, response: Response):
|
||||
service, owner, runs = worker_context(request)
|
||||
try:
|
||||
task = await await_drained(service.cancel(runs, owner, task_id, body.expected_revision))
|
||||
except (TaskConflict, TaskNotFound, AgentRunError) as exc:
|
||||
raise worker_error(exc) from exc
|
||||
if task["status"] == TaskStatus.CANCELLING:
|
||||
response.status_code = 202
|
||||
return task
|
||||
|
||||
|
||||
@router.get("/{task_id}/artifacts")
|
||||
@require_permission("threads", "read")
|
||||
async def list_worker_artifacts(task_id: str, request: Request, limit: int = Query(100, ge=1, le=100), offset: int = Query(0, ge=0)):
|
||||
service, owner, _runs = worker_context(request)
|
||||
try:
|
||||
return await service.repo.list_artifacts(owner, task_id, limit=limit, offset=offset)
|
||||
except TaskNotFound as exc:
|
||||
raise HTTPException(404, "Task not found") from exc
|
||||
|
||||
|
||||
@router.get("/{task_id}/artifacts/{artifact_id}")
|
||||
@require_permission("threads", "read")
|
||||
async def download_worker_artifact(task_id: str, artifact_id: str, request: Request):
|
||||
from app.gateway.routers.artifacts import _build_content_disposition
|
||||
|
||||
service, owner, _runs = worker_context(request)
|
||||
try:
|
||||
path, content = await service.repo.read_artifact(owner, task_id, artifact_id)
|
||||
except TaskNotFound as exc:
|
||||
raise HTTPException(404, "Artifact not found") from exc
|
||||
return Response(
|
||||
content,
|
||||
media_type="application/octet-stream",
|
||||
headers={
|
||||
"Content-Disposition": _build_content_disposition("attachment", PurePosixPath(path).name),
|
||||
"Cache-Control": "private, no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
@ -46,6 +46,19 @@ async def submit_run_input(thread_id: str, run_id: str, body: RunInputRequest, r
|
||||
raise HTTPException(404, "Run not found")
|
||||
if get_agent_runs(request) is None:
|
||||
raise HTTPException(503, "Run inputs require an authenticated session")
|
||||
from app.gateway.coordination_workers import managed_task, worker_context, worker_error
|
||||
from deerflow.persistence.coordination.sql import TaskConflict, TaskNotFound
|
||||
|
||||
if task_id := await managed_task(request, thread_id):
|
||||
service, owner, runs = worker_context(request)
|
||||
if body.mode != "queue":
|
||||
raise HTTPException(409, "Use queue mode to update a managed worker")
|
||||
try:
|
||||
item = await service.repo.append_input(owner, task_id, request_id=f"run-input:{run_id}:{body.request_id}", content=body.text)
|
||||
service.watch(runs, owner, task_id)
|
||||
return _worker_input_receipt(item, run_id, body.request_id)
|
||||
except (TaskConflict, TaskNotFound) as exc:
|
||||
raise worker_error(exc) from exc
|
||||
row = await _store(request).get(run_id, user_id=user_id)
|
||||
if row is None or row.get("run_inputs") is None:
|
||||
raise HTTPException(409, "This Run has not enabled interactive inputs")
|
||||
@ -63,10 +76,36 @@ async def submit_run_input(thread_id: str, run_id: str, body: RunInputRequest, r
|
||||
return {**item, "target_run_id": run_id}
|
||||
|
||||
|
||||
def _worker_input_receipt(item, run_id, request_id, consuming_run=None, cancelled=False):
|
||||
return {
|
||||
"id": request_id,
|
||||
"target_run_id": run_id,
|
||||
"message_id": str(uuid5(NAMESPACE_URL, f"deerflow-input:{run_id}:{request_id}")),
|
||||
"mode": "queue",
|
||||
"text": item["content"],
|
||||
"created_at": item["created_at"],
|
||||
"status": "applied" if consuming_run else "cancelled" if cancelled else "pending",
|
||||
**({"run_id": consuming_run} if consuming_run else {}),
|
||||
}
|
||||
|
||||
|
||||
async def list_thread_inputs(thread_id: str, request: Request):
|
||||
user_id = await get_current_user(request)
|
||||
if user_id is None:
|
||||
raise HTTPException(401, "Authentication required")
|
||||
from app.gateway.coordination_workers import managed_task
|
||||
from deerflow.persistence.coordination.sql import TaskNotFound
|
||||
|
||||
if task_id := await managed_task(request, thread_id):
|
||||
try:
|
||||
receipts, active, cancelled = await request.app.state.coordination_workers.repo.input_receipts(user_id, task_id)
|
||||
except TaskNotFound as exc:
|
||||
raise HTTPException(404, "Task not found") from exc
|
||||
items = []
|
||||
for item, consuming_run in receipts:
|
||||
_, run, key = item["request_id"].split(":", 2)
|
||||
items.append(_worker_input_receipt(item, run, key, consuming_run, cancelled))
|
||||
return {"items": items, "active_run_id": active}
|
||||
rows = await _store(request).list_input_runs(thread_id, user_id=user_id)
|
||||
items = []
|
||||
for row in reversed(rows):
|
||||
|
||||
@ -2562,6 +2562,13 @@ async def start_run(
|
||||
if not await thread_access_allowed():
|
||||
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
|
||||
|
||||
from app.gateway.coordination_workers import direct_worker_run, managed_task
|
||||
from deerflow.persistence.coordination.workers import THREAD_PREFIX
|
||||
|
||||
if thread_id.startswith(THREAD_PREFIX) and not getattr(request.state, "coordination_execution_id", None):
|
||||
if task_id := await managed_task(request, thread_id):
|
||||
return await direct_worker_run(request, task_id, body, idempotency_key)
|
||||
|
||||
owner_context_token = set_current_user(SimpleNamespace(id=owner_user_id)) if owner_user_id else None
|
||||
try:
|
||||
is_internal_caller = getattr(getattr(request, "state", None), "auth_source", None) == AUTH_SOURCE_INTERNAL
|
||||
|
||||
@ -21,7 +21,11 @@ class BatchContinuations:
|
||||
|
||||
async def prepare(self, request):
|
||||
from app.gateway.services import _CONTEXT_CONFIGURABLE_KEYS, _MODEL_SELECTION_CONTEXT_KEYS
|
||||
from deerflow.persistence.coordination.workers import THREAD_PREFIX
|
||||
|
||||
workers = getattr(self.app.state, "coordination_workers", None)
|
||||
if workers is not None and request.thread_id.startswith(THREAD_PREFIX) and await workers.repo.task_for_thread(request.thread_id):
|
||||
raise AgentRunError(409, "A managed worker accepts follow-ups through its coordination task")
|
||||
host = self.app.state.agent_runs_host
|
||||
grant = host.capture_batch_authority(request.continuation_authority, user_id=request.user_id)
|
||||
source = await request.continuation_authority.get(thread_id=request.thread_id, run_id=request.run_id)
|
||||
|
||||
@ -29,3 +29,70 @@ prerequisites keep their state. A running caller can record cancellation
|
||||
once it has confirmed that the subtree's executions have stopped. The
|
||||
repository handles work state. Host admission, run control, and artifact
|
||||
authorization belong to the execution integration.
|
||||
|
||||
## Managed workers
|
||||
|
||||
A signed-in caller can dispatch a task through the Gateway on a SQL-backed
|
||||
installation. The following request starts a worker without a project. Pass
|
||||
an optional `project_id` to create its thread in an existing project.
|
||||
|
||||
```http
|
||||
POST /api/coordination/tasks
|
||||
Content-Type: application/json
|
||||
|
||||
{"request_id":"review-27","objective":"Review the report","assistant_id":"lead_agent"}
|
||||
```
|
||||
|
||||
The response includes the task, its assignment (agent and thread), and the
|
||||
latest execution. Replaying the request reuses the worker. Read the same
|
||||
view with `GET /api/coordination/tasks/{task_id}`. Add a follow-up with
|
||||
`POST /api/coordination/tasks/{task_id}/inputs` and a JSON object containing
|
||||
`request_id` and `content`. A text message sent to the worker thread uses
|
||||
the same queue. Streaming and wait callers observe the run that consumes
|
||||
their message. An interrupted graph requires an explicit resume command.
|
||||
|
||||
The task owner can poll `GET /api/coordination/tasks/{task_id}/inputs`
|
||||
to read the inbox in sequence order. Pass the last seen `sequence` as
|
||||
`after` and set `limit` up to 100. Rows include the content, creation
|
||||
time, and source (`task_api`, `run_input`, or `worker_chat`).
|
||||
An input is `pending` while it can still enter a Run, `admitted`
|
||||
with its `run_id` once a Run has accepted the turn, or `not_admitted`
|
||||
when the task has closed to further work.
|
||||
|
||||
The Gateway freezes each turn before admission. The SQL transaction that
|
||||
creates the native run locks the task tree, verifies the execution epoch,
|
||||
and records the run ID. A follow-up that arrives during a turn advances
|
||||
the input sequence. When that run stops, the worker continues on its own
|
||||
thread with the outstanding inputs. Each task has one assignee per epoch.
|
||||
Workers can use the agent's allowed subagents for local delegation. Background
|
||||
batches can run local work. A batch cannot schedule an independent follow-up
|
||||
run on a managed thread; the task owns the next turn.
|
||||
|
||||
`POST /api/coordination/tasks/{task_id}/cancel` accepts `expected_revision`.
|
||||
It fences the owned subtree before requesting run and native batch
|
||||
cancellation. If confirmation takes longer than five seconds after the
|
||||
fence is committed, the response is HTTP 202 with status `cancelling`.
|
||||
The Gateway continues confirming in the background and publishes
|
||||
`cancelled` once the runs have stopped and batch cancellation is requested.
|
||||
Poll `GET /api/coordination/tasks/{task_id}` for the status.
|
||||
|
||||
A completed task links to the producing run's messages. For presented output
|
||||
files, the Gateway stores the captured bytes with their SHA-256 hashes and
|
||||
producing execution. The owner's task artifact URL continues to serve those
|
||||
bytes after the source file changes. Capture uses the existing archive budgets
|
||||
of 50 files, 50 MiB per file and 100 MiB per execution. The execution reports
|
||||
`delivery_failed` if a presented file cannot be captured.
|
||||
|
||||
Use `GET /api/coordination/tasks/{task_id}/artifacts` to list the task's
|
||||
deliveries, including files from earlier executions. The response lists
|
||||
artifact IDs, source paths, hashes, sizes, producing `run_id` and `turn`,
|
||||
and download URLs. Results are ordered by turn, then source path and ID.
|
||||
Use `limit` (default and maximum 100) and `offset` (default 0) to page
|
||||
through them. Files presented under the same path in different turns
|
||||
have separate IDs and URLs. The task detail's `execution.artifacts` lists
|
||||
deliveries produced by that execution.
|
||||
|
||||
Worker observation uses the initiating session's process-local run handle.
|
||||
The task, frozen inputs, native run IDs and artifact contents survive a Gateway
|
||||
restart. Automatic reauthorization and restart reconciliation are tracked in
|
||||
the next stage of [RFC #8345](https://github.com/bytedance/deer-flow/issues/8345).
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
|
||||
from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Index, String, Text, UniqueConstraint
|
||||
from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, ForeignKeyConstraint, Index, LargeBinary, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@ -52,3 +52,41 @@ class CoordinationInputRow(Base):
|
||||
request_id: Mapped[str] = mapped_column(String(200))
|
||||
content: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class CoordinationAssignmentRow(Base):
|
||||
"""One active worker per task. Parallel work uses child tasks."""
|
||||
|
||||
__tablename__ = "coord_assignments"
|
||||
task_id: Mapped[str] = mapped_column(ForeignKey("coord_tasks.id"), primary_key=True)
|
||||
epoch: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
assistant_id: Mapped[str] = mapped_column(String(128))
|
||||
|
||||
|
||||
class CoordinationExecutionRow(Base):
|
||||
__tablename__ = "coord_executions"
|
||||
__table_args__ = (
|
||||
ForeignKeyConstraint(["task_id", "epoch"], ["coord_assignments.task_id", "coord_assignments.epoch"]),
|
||||
UniqueConstraint("task_id", "turn", name="uq_coord_execution_turn"),
|
||||
UniqueConstraint("task_id", "resume_key", name="uq_coord_execution_resume"),
|
||||
)
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
task_id: Mapped[str] = mapped_column(String(64))
|
||||
epoch: Mapped[int] = mapped_column(BigInteger)
|
||||
turn: Mapped[int] = mapped_column(BigInteger)
|
||||
resume_key: Mapped[str | None] = mapped_column(String(200))
|
||||
covered_seq: Mapped[int] = mapped_column(BigInteger)
|
||||
frozen_input: Mapped[dict] = mapped_column(JSON)
|
||||
status: Mapped[str] = mapped_column(String(32))
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), unique=True)
|
||||
|
||||
|
||||
class CoordinationArtifactRow(Base):
|
||||
__tablename__ = "coord_artifacts"
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
execution_id: Mapped[str] = mapped_column(ForeignKey("coord_executions.id"), index=True)
|
||||
path: Mapped[str] = mapped_column(Text)
|
||||
sha256: Mapped[str] = mapped_column(String(64))
|
||||
size: Mapped[int] = mapped_column(BigInteger)
|
||||
content: Mapped[bytes] = mapped_column(LargeBinary)
|
||||
|
||||
@ -176,21 +176,24 @@ class CoordinationTaskRepository:
|
||||
async with self._write() as session:
|
||||
task = await self._lock_tree(session, owner_id, task_id)
|
||||
_check_revision(task, expected_revision)
|
||||
if task.status != TaskStatus.QUEUED:
|
||||
raise TaskConflict("Only a queued task can start")
|
||||
if task.parent_id is not None:
|
||||
parent = await self._lookup(session, owner_id, task.parent_id)
|
||||
if parent.status != TaskStatus.RUNNING:
|
||||
raise TaskConflict("Parent task has not started")
|
||||
if task.depends_on:
|
||||
query = select(CoordinationTaskRow.id).filter_by(owner_id=owner_id, status=TaskStatus.SUCCEEDED)
|
||||
ready = set((await session.scalars(query.where(CoordinationTaskRow.id.in_(task.depends_on)))).all())
|
||||
if ready != set(task.depends_on):
|
||||
raise TaskConflict("Task dependencies have not succeeded")
|
||||
task.status = TaskStatus.RUNNING
|
||||
_touch(task)
|
||||
await self._start_locked(session, task)
|
||||
return _snapshot(task)
|
||||
|
||||
async def _start_locked(self, session: AsyncSession, task: CoordinationTaskRow) -> None:
|
||||
if task.status != TaskStatus.QUEUED:
|
||||
raise TaskConflict("Only a queued task can start")
|
||||
if task.parent_id is not None:
|
||||
parent = await self._lookup(session, task.owner_id, task.parent_id)
|
||||
if parent.status != TaskStatus.RUNNING:
|
||||
raise TaskConflict("Parent task has not started")
|
||||
if task.depends_on:
|
||||
query = select(CoordinationTaskRow.id).filter_by(owner_id=task.owner_id, status=TaskStatus.SUCCEEDED)
|
||||
ready = set((await session.scalars(query.where(CoordinationTaskRow.id.in_(task.depends_on)))).all())
|
||||
if ready != set(task.depends_on):
|
||||
raise TaskConflict("Task dependencies have not succeeded")
|
||||
task.status = TaskStatus.RUNNING
|
||||
_touch(task)
|
||||
|
||||
async def finish(
|
||||
self,
|
||||
owner_id: str,
|
||||
@ -211,15 +214,18 @@ class CoordinationTaskRepository:
|
||||
_check_revision(task, expected_revision)
|
||||
if task.latest_input_seq != covered_input_seq:
|
||||
raise TaskConflict("Result does not cover the current task inputs")
|
||||
children = (await session.scalars(select(CoordinationTaskRow.status).filter_by(parent_id=task_id, owner_id=owner_id))).all()
|
||||
accepted = {TaskStatus.SUCCEEDED} if outcome == TaskStatus.SUCCEEDED else {TaskStatus.SUCCEEDED, TaskStatus.FAILED, TaskStatus.CANCELLED}
|
||||
if any(status not in accepted for status in children):
|
||||
raise TaskConflict("Child tasks have not settled for this outcome")
|
||||
task.status = outcome
|
||||
task.result_ref = result_ref
|
||||
_touch(task)
|
||||
await self._finish_locked(session, task, outcome, result_ref)
|
||||
return _snapshot(task)
|
||||
|
||||
async def _finish_locked(self, session: AsyncSession, task: CoordinationTaskRow, outcome: TaskStatus, result_ref: str) -> None:
|
||||
children = (await session.scalars(select(CoordinationTaskRow.status).filter_by(parent_id=task.id, owner_id=task.owner_id))).all()
|
||||
accepted = {TaskStatus.SUCCEEDED} if outcome == TaskStatus.SUCCEEDED else {TaskStatus.SUCCEEDED, TaskStatus.FAILED, TaskStatus.CANCELLED}
|
||||
if any(status not in accepted for status in children):
|
||||
raise TaskConflict("Child tasks have not settled for this outcome")
|
||||
task.status = outcome
|
||||
task.result_ref = result_ref
|
||||
_touch(task)
|
||||
|
||||
async def _subtree(self, session: AsyncSession, task: CoordinationTaskRow) -> list[CoordinationTaskRow]:
|
||||
rows = (await session.scalars(select(CoordinationTaskRow).filter_by(owner_id=task.owner_id, root_id=task.root_id))).all()
|
||||
children = defaultdict(list)
|
||||
@ -255,8 +261,20 @@ class CoordinationTaskRepository:
|
||||
task = await self._lock_tree(session, owner_id, task_id)
|
||||
if task.status not in (TaskStatus.CANCELLING, TaskStatus.CANCELLED) or task.execution_epoch != execution_epoch:
|
||||
raise TaskConflict("Cancellation no longer matches the task")
|
||||
for row in await self._subtree(session, task):
|
||||
rows = await self._subtree(session, task)
|
||||
await self._confirm_stopped(session, rows)
|
||||
for row in rows:
|
||||
if row.status == TaskStatus.CANCELLING:
|
||||
row.status = TaskStatus.CANCELLED
|
||||
_touch(row)
|
||||
return _snapshot(task)
|
||||
|
||||
async def _confirm_stopped(self, session, rows):
|
||||
"""Close active execution records in the same transaction as their tasks."""
|
||||
from sqlalchemy import update
|
||||
|
||||
from deerflow.persistence.coordination.model import CoordinationExecutionRow
|
||||
|
||||
ids = [row.id for row in rows if row.status == TaskStatus.CANCELLING]
|
||||
if ids:
|
||||
await session.execute(update(CoordinationExecutionRow).where(CoordinationExecutionRow.task_id.in_(ids), CoordinationExecutionRow.status.in_(("admitting", "running", "waiting_for_input"))).values(status="cancelled"))
|
||||
|
||||
@ -0,0 +1,264 @@
|
||||
"""Durable admission, cancellation and result ownership for managed workers."""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from deerflow.persistence.coordination.model import CoordinationArtifactRow as Artifact
|
||||
from deerflow.persistence.coordination.model import CoordinationAssignmentRow as Assignment
|
||||
from deerflow.persistence.coordination.model import CoordinationExecutionRow as Execution
|
||||
from deerflow.persistence.coordination.model import CoordinationInputRow, CoordinationTaskRow, TaskStatus
|
||||
from deerflow.persistence.coordination.sql import CoordinationTaskRepository, TaskConflict, TaskNotFound
|
||||
|
||||
THREAD_PREFIX = "coord-worker-"
|
||||
RUN_KEY_PREFIX = "coord-execution:"
|
||||
INPUT_SOURCES = {"input": "task_api", "chat": "worker_chat", "run-input": "run_input"}
|
||||
|
||||
|
||||
async def bind_admitted_worker(session, thread_id, owner_id, key, run_id):
|
||||
"""Called inside native SQL run admission, under the same tree lock as cancellation."""
|
||||
from deerflow.runtime.runs.manager import ConflictError
|
||||
|
||||
worker = await session.scalar(select(Assignment).where(Assignment.thread_id == thread_id))
|
||||
if worker is None:
|
||||
return # A pre-existing user-chosen thread ID may share the prefix.
|
||||
task_id = worker.task_id
|
||||
root_id = await session.scalar(select(CoordinationTaskRow.root_id).filter_by(id=task_id))
|
||||
await session.execute(select(CoordinationTaskRow.id).filter_by(id=root_id).with_for_update())
|
||||
task = await session.get(CoordinationTaskRow, task_id)
|
||||
execution = await session.get(Execution, key[len(RUN_KEY_PREFIX) :]) if key and key.startswith(RUN_KEY_PREFIX) else None
|
||||
if (
|
||||
worker.epoch != task.execution_epoch
|
||||
or task.owner_id != owner_id
|
||||
or task.status != TaskStatus.RUNNING
|
||||
or execution is None
|
||||
or execution.task_id != task_id
|
||||
or execution.epoch != task.execution_epoch
|
||||
or execution.status != "admitting"
|
||||
):
|
||||
raise ConflictError("Managed worker admission no longer matches the task")
|
||||
execution.run_id = run_id
|
||||
execution.status = "running"
|
||||
|
||||
|
||||
class CoordinationWorkerRepository(CoordinationTaskRepository):
|
||||
async def task_for_thread(self, thread_id):
|
||||
async with self._sessions() as session:
|
||||
return await session.scalar(select(Assignment.task_id).filter_by(thread_id=thread_id))
|
||||
|
||||
@staticmethod
|
||||
def _latest(task_id):
|
||||
return select(Execution).filter_by(task_id=task_id).order_by(Execution.turn.desc()).limit(1)
|
||||
|
||||
@staticmethod
|
||||
async def _artifact_refs(session, task_id, *, execution_id=None, limit=None, offset=0):
|
||||
query = (
|
||||
select(Artifact.id, Artifact.path, Artifact.sha256, Artifact.size, Execution.run_id, Execution.turn)
|
||||
.join(Execution, Execution.id == Artifact.execution_id)
|
||||
.where(Execution.task_id == task_id)
|
||||
.order_by(Execution.turn, Artifact.path, Artifact.id)
|
||||
)
|
||||
if execution_id is not None:
|
||||
query = query.where(Execution.id == execution_id)
|
||||
rows = await session.execute(query.limit(limit).offset(offset))
|
||||
return [
|
||||
{
|
||||
"id": row.id,
|
||||
"source_path": row.path,
|
||||
"sha256": row.sha256,
|
||||
"size": row.size,
|
||||
"run_id": row.run_id,
|
||||
"turn": row.turn,
|
||||
"url": f"/api/coordination/tasks/{task_id}/artifacts/{row.id}",
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
async def view(self, owner, task_id):
|
||||
async with self._sessions() as session:
|
||||
task = await self._lookup(session, owner, task_id)
|
||||
assignment = await session.scalar(select(Assignment).filter_by(task_id=task_id).order_by(Assignment.epoch.desc()).limit(1))
|
||||
execution = await session.scalar(self._latest(task_id))
|
||||
result = None
|
||||
if execution is not None:
|
||||
result = execution.to_dict(exclude={"frozen_input", "resume_key"})
|
||||
result["artifacts"] = await self._artifact_refs(session, task_id, execution_id=execution.id)
|
||||
return {
|
||||
"task": task.to_dict(),
|
||||
"assignment": assignment.to_dict() if assignment else None,
|
||||
"execution": result,
|
||||
}
|
||||
|
||||
async def admission(self, owner, task_id, execution_id):
|
||||
async with self._sessions() as session:
|
||||
task = await self._lookup(session, owner, task_id)
|
||||
execution = await session.get(Execution, execution_id)
|
||||
if execution is None or execution.task_id != task_id or execution.epoch != task.execution_epoch or task.status != TaskStatus.RUNNING:
|
||||
raise TaskConflict("Task no longer accepts this execution")
|
||||
worker = await session.get(Assignment, (task_id, execution.epoch))
|
||||
return worker.to_dict(), execution.frozen_input, task.project_id
|
||||
|
||||
async def list_artifacts(self, owner, task_id, *, limit, offset):
|
||||
async with self._sessions() as session:
|
||||
await self._lookup(session, owner, task_id)
|
||||
return await self._artifact_refs(session, task_id, limit=limit, offset=offset)
|
||||
|
||||
async def read_artifact(self, owner, task_id, artifact_id):
|
||||
async with self._sessions() as session:
|
||||
query = select(Artifact).join(Execution, Execution.id == Artifact.execution_id).join(CoordinationTaskRow, CoordinationTaskRow.id == Execution.task_id)
|
||||
artifact = await session.scalar(query.where(Artifact.id == artifact_id, Execution.task_id == task_id, CoordinationTaskRow.owner_id == owner))
|
||||
if artifact is None:
|
||||
raise TaskNotFound(artifact_id)
|
||||
return artifact.path, artifact.content
|
||||
|
||||
async def prepare(self, owner, task_id, assistant_id=None):
|
||||
"""Freeze a bounded turn before admission. Duplicate drivers get the same turn."""
|
||||
async with self._write() as session:
|
||||
task = await self._lock_tree(session, owner, task_id)
|
||||
worker = await session.get(Assignment, (task_id, task.execution_epoch))
|
||||
if worker is not None and assistant_id is not None and worker.assistant_id != assistant_id:
|
||||
raise TaskConflict("Task already has a different worker")
|
||||
if task.status == TaskStatus.QUEUED:
|
||||
await self._start_locked(session, task)
|
||||
if task.status != TaskStatus.RUNNING:
|
||||
return None
|
||||
if worker is None:
|
||||
if assistant_id is None:
|
||||
raise TaskConflict("Task has no assigned worker")
|
||||
worker = Assignment(task_id=task_id, epoch=task.execution_epoch, assistant_id=assistant_id, thread_id=THREAD_PREFIX + uuid4().hex)
|
||||
session.add(worker)
|
||||
await session.flush()
|
||||
execution = await session.scalar(self._latest(task_id))
|
||||
if execution is None or (execution.status == "superseded" and execution.covered_seq < task.latest_input_seq):
|
||||
previous = execution.covered_seq if execution else 0
|
||||
if execution is None:
|
||||
messages = [{"role": "user", "content": task.objective}]
|
||||
elif execution.run_id is None:
|
||||
messages = list(execution.frozen_input.get("messages", []))
|
||||
else:
|
||||
messages = []
|
||||
inputs = await session.scalars(select(CoordinationInputRow).filter_by(task_id=task_id).where(CoordinationInputRow.sequence > previous).order_by(CoordinationInputRow.sequence).limit(8))
|
||||
for item in inputs:
|
||||
messages.append({"role": "user", "content": item.content})
|
||||
previous = item.sequence
|
||||
execution = Execution(
|
||||
id=uuid4().hex,
|
||||
task_id=task_id,
|
||||
turn=execution.turn + 1 if execution else 1,
|
||||
epoch=task.execution_epoch,
|
||||
covered_seq=previous,
|
||||
frozen_input={"messages": messages},
|
||||
status="admitting",
|
||||
)
|
||||
session.add(execution)
|
||||
return worker.to_dict(), execution.to_dict(), task.project_id
|
||||
|
||||
async def resume(self, owner, task_id, *, key, value):
|
||||
async with self._write() as session:
|
||||
task = await self._lock_tree(session, owner, task_id)
|
||||
payload = {"command": {"resume": value}}
|
||||
execution = await session.scalar(select(Execution).filter_by(task_id=task_id, resume_key=key))
|
||||
if execution is not None:
|
||||
if execution.frozen_input != payload:
|
||||
raise TaskConflict("Request ID was already used with a different response")
|
||||
else:
|
||||
if task.status != TaskStatus.RUNNING:
|
||||
raise TaskConflict("Task is closed")
|
||||
previous = await session.scalar(self._latest(task_id))
|
||||
if previous is None or previous.status != "waiting_for_input":
|
||||
raise TaskConflict("Worker has no pending interrupt")
|
||||
execution = Execution(
|
||||
id=uuid4().hex,
|
||||
task_id=task_id,
|
||||
turn=previous.turn + 1,
|
||||
epoch=task.execution_epoch,
|
||||
covered_seq=previous.covered_seq,
|
||||
frozen_input=payload,
|
||||
status="admitting",
|
||||
resume_key=key,
|
||||
)
|
||||
session.add(execution)
|
||||
worker = await session.get(Assignment, (task_id, execution.epoch))
|
||||
return worker.to_dict(), execution.to_dict(), task.project_id
|
||||
|
||||
async def settle(self, owner, task_id, execution_id, run_id, status, artifacts):
|
||||
async with self._write() as session:
|
||||
task = await self._lock_tree(session, owner, task_id)
|
||||
execution = await session.get(Execution, execution_id)
|
||||
if execution is None or execution.task_id != task_id or execution.run_id != run_id or execution.status not in ("running", "admitting"):
|
||||
return
|
||||
if task.status != TaskStatus.RUNNING or task.execution_epoch != execution.epoch:
|
||||
return
|
||||
for path, digest, content in artifacts:
|
||||
session.add(Artifact(id=uuid4().hex, execution_id=execution.id, path=path, sha256=digest, size=len(content), content=content))
|
||||
if status == "waiting_for_input":
|
||||
execution.status = status
|
||||
return
|
||||
if execution.covered_seq < task.latest_input_seq:
|
||||
execution.status = "superseded"
|
||||
return
|
||||
worker = await session.get(Assignment, (task_id, task.execution_epoch))
|
||||
result_ref = f"/api/threads/{worker.thread_id}/runs/{run_id}/messages" if run_id else None
|
||||
outcome = TaskStatus.SUCCEEDED if status == "success" else TaskStatus.FAILED
|
||||
await self._finish_locked(session, task, outcome, result_ref or f"/api/coordination/tasks/{task_id}")
|
||||
execution.status = status
|
||||
|
||||
@staticmethod
|
||||
def _admitted_run(task_id, sequence):
|
||||
return select(Execution.run_id).where(Execution.task_id == task_id, Execution.covered_seq >= sequence, Execution.run_id.is_not(None)).order_by(Execution.turn).limit(1)
|
||||
|
||||
async def list_worker_inputs(self, owner, task_id, *, after, limit):
|
||||
async with self._sessions() as session:
|
||||
task = await self._lookup(session, owner, task_id)
|
||||
admitted = self._admitted_run(task_id, CoordinationInputRow.sequence).scalar_subquery()
|
||||
query = select(CoordinationInputRow, admitted).where(CoordinationInputRow.task_id == task_id, CoordinationInputRow.sequence > after).order_by(CoordinationInputRow.sequence).limit(limit)
|
||||
return [
|
||||
{
|
||||
"sequence": row.sequence,
|
||||
"content": row.content,
|
||||
"created_at": row.created_at,
|
||||
"source": INPUT_SOURCES.get(row.request_id.partition(":")[0], "unknown"),
|
||||
"status": "admitted" if run_id else "pending" if task.status in (TaskStatus.QUEUED, TaskStatus.RUNNING) else "not_admitted",
|
||||
"run_id": run_id,
|
||||
}
|
||||
for row, run_id in (await session.execute(query)).all()
|
||||
]
|
||||
|
||||
async def input_receipts(self, owner, task_id):
|
||||
async with self._sessions() as session:
|
||||
task = await self._lookup(session, owner, task_id)
|
||||
consuming = self._admitted_run(task_id, CoordinationInputRow.sequence).scalar_subquery()
|
||||
query = select(CoordinationInputRow, consuming).filter(CoordinationInputRow.task_id == task_id, CoordinationInputRow.request_id.startswith("run-input:")).order_by(CoordinationInputRow.sequence.desc()).limit(32)
|
||||
results = [(row.to_dict(), run) for row, run in (await session.execute(query)).all() if row.request_id.count(":") >= 2]
|
||||
execution = await session.scalar(self._latest(task_id))
|
||||
active = execution.run_id if task.status == TaskStatus.RUNNING and execution and execution.status == "running" else None
|
||||
return list(reversed(results)), active, task.status in (TaskStatus.CANCELLING, TaskStatus.CANCELLED)
|
||||
|
||||
async def run_for_input(self, owner, task_id, sequence):
|
||||
async with self._sessions() as session:
|
||||
task = await self._lookup(session, owner, task_id)
|
||||
run_id = await session.scalar(self._admitted_run(task_id, sequence))
|
||||
if run_id is None and task.status not in (TaskStatus.RUNNING, TaskStatus.QUEUED):
|
||||
raise TaskConflict("Task stopped before this input could run")
|
||||
return run_id
|
||||
|
||||
async def cancellations(self, owner, task_id):
|
||||
async with self._write() as session:
|
||||
task = await self._lock_tree(session, owner, task_id)
|
||||
ids = [row.id for row in await self._subtree(session, task) if row.status == TaskStatus.CANCELLING]
|
||||
rows = await session.execute(
|
||||
select(Assignment.thread_id, Execution.run_id)
|
||||
.join(Execution, (Assignment.task_id == Execution.task_id) & (Assignment.epoch == Execution.epoch))
|
||||
.where(Assignment.task_id.in_(ids), Execution.run_id.is_not(None), Execution.status == "running")
|
||||
)
|
||||
return list(rows)
|
||||
|
||||
async def active_batches(self, owner, task_id):
|
||||
from deerflow.persistence.subagent_batches.model import SubagentBatchRow
|
||||
|
||||
async with self._write() as session:
|
||||
task = await self._lock_tree(session, owner, task_id)
|
||||
ids = [row.id for row in await self._subtree(session, task) if row.status == TaskStatus.CANCELLING]
|
||||
threads = select(Assignment.thread_id).where(Assignment.task_id.in_(ids))
|
||||
query = select(SubagentBatchRow.thread_id, SubagentBatchRow.id).where(SubagentBatchRow.user_id == owner, SubagentBatchRow.thread_id.in_(threads), SubagentBatchRow.status.in_(("queued", "running", "paused")))
|
||||
return (await session.execute(query)).all()
|
||||
@ -0,0 +1,56 @@
|
||||
"""Bind coordination tasks to worker threads and native runs."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0057_coordination_workers"
|
||||
down_revision = "0056_coordination_task_state"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table("coord_assignments"):
|
||||
op.create_table(
|
||||
"coord_assignments",
|
||||
sa.Column("task_id", sa.String(64), sa.ForeignKey("coord_tasks.id"), primary_key=True),
|
||||
sa.Column("epoch", sa.BigInteger(), primary_key=True),
|
||||
sa.Column("thread_id", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("assistant_id", sa.String(128), nullable=False),
|
||||
)
|
||||
if not inspector.has_table("coord_executions"):
|
||||
op.create_table(
|
||||
"coord_executions",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("task_id", sa.String(64), nullable=False),
|
||||
sa.Column("epoch", sa.BigInteger(), nullable=False),
|
||||
sa.Column("turn", sa.BigInteger(), nullable=False),
|
||||
sa.Column("resume_key", sa.String(200)),
|
||||
sa.Column("covered_seq", sa.BigInteger(), nullable=False),
|
||||
sa.Column("frozen_input", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(32), nullable=False),
|
||||
sa.Column("run_id", sa.String(64), unique=True),
|
||||
sa.ForeignKeyConstraint(["task_id", "epoch"], ["coord_assignments.task_id", "coord_assignments.epoch"]),
|
||||
sa.UniqueConstraint("task_id", "turn", name="uq_coord_execution_turn"),
|
||||
sa.UniqueConstraint("task_id", "resume_key", name="uq_coord_execution_resume"),
|
||||
)
|
||||
|
||||
if not inspector.has_table("coord_artifacts"):
|
||||
op.create_table(
|
||||
"coord_artifacts",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("execution_id", sa.String(64), sa.ForeignKey("coord_executions.id"), nullable=False),
|
||||
sa.Column("path", sa.Text(), nullable=False),
|
||||
sa.Column("sha256", sa.String(64), nullable=False),
|
||||
sa.Column("size", sa.BigInteger(), nullable=False),
|
||||
sa.Column("content", sa.LargeBinary(), nullable=False),
|
||||
)
|
||||
if "ix_coord_artifacts_execution_id" not in {idx["name"] for idx in inspector.get_indexes("coord_artifacts")}:
|
||||
op.create_index("ix_coord_artifacts_execution_id", "coord_artifacts", ["execution_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for table in ("coord_artifacts", "coord_executions", "coord_assignments"):
|
||||
if sa.inspect(op.get_bind()).has_table(table):
|
||||
op.drop_table(table)
|
||||
@ -24,7 +24,7 @@ from deerflow.persistence.channel_connections.model import (
|
||||
from deerflow.persistence.channel_leases.model import ChannelLeaseRow
|
||||
from deerflow.persistence.channel_runtime_configs.model import ChannelRuntimeConfigRow
|
||||
from deerflow.persistence.channel_thread_bindings.model import ChannelThreadBindingRow
|
||||
from deerflow.persistence.coordination.model import CoordinationInputRow, CoordinationTaskRow
|
||||
from deerflow.persistence.coordination.model import CoordinationArtifactRow, CoordinationAssignmentRow, CoordinationExecutionRow, CoordinationInputRow, CoordinationTaskRow
|
||||
from deerflow.persistence.extension_leases.model import ExtensionExecutionLeaseRow
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.login_throttle.model import LoginThrottleRow
|
||||
@ -62,6 +62,9 @@ __all__ = [
|
||||
"ChannelOAuthStateRow",
|
||||
"ChannelRuntimeConfigRow",
|
||||
"ChannelThreadBindingRow",
|
||||
"CoordinationArtifactRow",
|
||||
"CoordinationAssignmentRow",
|
||||
"CoordinationExecutionRow",
|
||||
"CoordinationInputRow",
|
||||
"CoordinationTaskRow",
|
||||
"FeedbackRow",
|
||||
|
||||
@ -1200,6 +1200,11 @@ class RunRepository(RunStore):
|
||||
if existing is not None:
|
||||
raise RunIdempotencyConflict(self._row_to_dict(existing))
|
||||
|
||||
from deerflow.persistence.coordination.workers import THREAD_PREFIX, bind_admitted_worker
|
||||
|
||||
if thread_id.startswith(THREAD_PREFIX) and values["operation_kind"] == "run":
|
||||
await bind_admitted_worker(session, thread_id, resolved_user_id, idempotency_key, run_id)
|
||||
|
||||
if multitask_strategy in ("interrupt", "rollback"):
|
||||
stmt = (
|
||||
select(RunRow)
|
||||
|
||||
@ -773,14 +773,13 @@ def test_timeout_runs_inside_the_machine(installed) -> None:
|
||||
assert call["timeout"] > 3
|
||||
|
||||
|
||||
@posix_only
|
||||
def test_timed_out_command_keeps_partial_output_and_exit_124(installed) -> None:
|
||||
_, fake = installed
|
||||
fake.local_shell = True
|
||||
# Simulate a remote timeout after the command has already produced output.
|
||||
fake.exec_override = lambda *_: _Result(124, "before")
|
||||
output = _box(fake).execute_command("echo before; sleep 30; echo after", timeout=1)
|
||||
assert output.startswith("before")
|
||||
assert "after" not in output
|
||||
assert output.endswith("Exit Code: 124")
|
||||
assert fake.execs[-1]["argv"] == ["timeout", "-k", "5", "1", "bash", "-lc", "echo before; sleep 30; echo after"]
|
||||
assert output == "before\nExit Code: 124"
|
||||
|
||||
|
||||
@posix_only
|
||||
|
||||
@ -175,13 +175,13 @@ async def test_coordination_migration_matches_model_and_downgrades(database):
|
||||
engine, _, schema = database
|
||||
config = bootstrap._get_alembic_config(engine, postgres_schema=schema)
|
||||
await asyncio.to_thread(command.upgrade, config, "0055_thread_shares")
|
||||
await asyncio.to_thread(command.upgrade, config, "0056_coordination_task_state")
|
||||
await asyncio.to_thread(command.upgrade, config, "0057_coordination_workers")
|
||||
async with engine.connect() as conn:
|
||||
|
||||
def diff(sync):
|
||||
context = MigrationContext.configure(
|
||||
sync,
|
||||
opts={"compare_type": True, "include_object": lambda _, name, kind, *_args: kind != "table" or name in {"coord_tasks", "coord_inputs"}},
|
||||
opts={"compare_type": True, "include_object": lambda _, name, kind, *_args: kind != "table" or name in {"coord_tasks", "coord_inputs", "coord_assignments", "coord_executions", "coord_artifacts"}},
|
||||
)
|
||||
return compare_metadata(context, Base.metadata)
|
||||
|
||||
@ -193,6 +193,56 @@ async def test_coordination_migration_matches_model_and_downgrades(database):
|
||||
async with engine.connect() as conn:
|
||||
tables = await conn.run_sync(lambda sync: set(inspect(sync).get_table_names()))
|
||||
assert "threads_meta" in tables
|
||||
assert not {"coord_tasks", "coord_inputs"} & tables
|
||||
await asyncio.to_thread(command.upgrade, config, "0056_coordination_task_state")
|
||||
assert not {"coord_tasks", "coord_inputs", "coord_assignments", "coord_executions", "coord_artifacts"} & tables
|
||||
await asyncio.to_thread(command.upgrade, config, "0057_coordination_workers")
|
||||
assert (await repo.create("alice", request_id="root", objective="After re-upgrade"))["status"] == TaskStatus.QUEUED
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_admission_is_atomic_with_cancellation_and_replay(database):
|
||||
from deerflow.persistence.coordination.workers import RUN_KEY_PREFIX, CoordinationWorkerRepository
|
||||
from deerflow.persistence.run import RunRepository
|
||||
from deerflow.runtime.runs.manager import ConflictError
|
||||
|
||||
engine, dialect, schema = database
|
||||
await bootstrap.bootstrap_schema(engine, backend=dialect, postgres_schema=schema)
|
||||
sessions = async_sessionmaker(engine)
|
||||
workers = CoordinationWorkerRepository(sessions)
|
||||
runs = RunRepository(sessions)
|
||||
task = await workers.create("alice", request_id="dispatch", objective="Review")
|
||||
await workers.append_input("alice", task["id"], request_id="input:follow-up", content="Check the result")
|
||||
first, retry = await asyncio.gather(*(workers.prepare("alice", task["id"], "lead_agent") for _ in range(2)))
|
||||
assert first == retry
|
||||
worker, execution, project = first
|
||||
assert project is None
|
||||
kwargs = dict(thread_id=worker["thread_id"], owner_worker_id="gateway", user_id="alice", lease_expires_at=None)
|
||||
key = RUN_KEY_PREFIX + execution["id"]
|
||||
with pytest.raises(ConflictError):
|
||||
await runs.create_thread_operation_atomic("unmanaged", **kwargs, idempotency_key="client-key")
|
||||
with pytest.raises(TaskNotFound):
|
||||
await workers.view("bob", task["id"])
|
||||
with pytest.raises(TaskNotFound):
|
||||
await workers.list_artifacts("bob", task["id"], limit=1, offset=0)
|
||||
with pytest.raises(TaskNotFound):
|
||||
await workers.list_worker_inputs("bob", task["id"], after=0, limit=1)
|
||||
(pending,) = await workers.list_worker_inputs("alice", task["id"], after=0, limit=1)
|
||||
assert (pending["source"], pending["status"], pending["run_id"]) == ("task_api", "pending", None)
|
||||
assert await workers.list_artifacts("alice", task["id"], limit=1, offset=0) == []
|
||||
current = await workers.get("alice", task["id"])
|
||||
admitted, cancelled = await asyncio.gather(
|
||||
runs.create_thread_operation_atomic("native", **kwargs, idempotency_key=key),
|
||||
workers.request_cancel("alice", task["id"], expected_revision=current["revision"]),
|
||||
return_exceptions=True,
|
||||
)
|
||||
assert cancelled["status"] == TaskStatus.CANCELLING
|
||||
stopped = await workers.cancellations("alice", task["id"])
|
||||
(receipt,) = await workers.list_worker_inputs("alice", task["id"], after=0, limit=1)
|
||||
if isinstance(admitted, ConflictError):
|
||||
assert stopped == [] and await runs.get("native", user_id="alice") is None
|
||||
assert (receipt["status"], receipt["run_id"]) == ("not_admitted", None)
|
||||
else:
|
||||
assert not isinstance(admitted, BaseException) and stopped == [(worker["thread_id"], "native")]
|
||||
assert (await workers.view("alice", task["id"]))["execution"]["run_id"] == "native"
|
||||
assert (receipt["status"], receipt["run_id"]) == ("admitted", "native")
|
||||
with pytest.raises(ConflictError):
|
||||
await runs.create_thread_operation_atomic("late", **kwargs, idempotency_key=RUN_KEY_PREFIX + uuid4().hex)
|
||||
|
||||
@ -161,7 +161,9 @@ async def runtime(host, monkeypatch, tmp_path):
|
||||
prompt = state["messages"][-1].content
|
||||
if prompt == "wait":
|
||||
entered.set()
|
||||
await asyncio.Event().wait()
|
||||
await getattr(request.app.state, "release_worker", asyncio.Event()).wait()
|
||||
if present := getattr(request.app.state, "present_worker_report", None):
|
||||
await present(prompt, runtime)
|
||||
if prompt == "approve":
|
||||
prompt = interrupt({"question": "Continue?"})
|
||||
return {"messages": [AIMessage(content=f"answer:{prompt}")]}
|
||||
@ -776,3 +778,208 @@ async def test_restored_delegated_handle_is_plugin_scoped_with_extension_origin(
|
||||
assert runs._origin == {"kind": "extension", "namespace": "workflows"}
|
||||
with pytest.raises(AgentRunError, match="already bound"):
|
||||
runs.for_plugin("other")
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def worker_gateway(runtime, tmp_path):
|
||||
import asyncio
|
||||
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway.coordination_workers import CoordinationWorkers
|
||||
from app.gateway.routers import coordination, thread_runs
|
||||
from deerflow.persistence.bootstrap import bootstrap_schema
|
||||
from deerflow.persistence.coordination.workers import CoordinationWorkerRepository
|
||||
from deerflow.persistence.run import RunRepository
|
||||
from deerflow.runtime import RunManager
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
|
||||
runs, state, request, entered, _ = runtime
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'workers.db'}")
|
||||
await bootstrap_schema(engine, backend="sqlite")
|
||||
factory = async_sessionmaker(engine)
|
||||
state.run_store = RunRepository(factory)
|
||||
await state.run_manager.shutdown()
|
||||
state.run_manager = RunManager(store=state.run_store)
|
||||
workers = state.coordination_workers = CoordinationWorkers(CoordinationWorkerRepository(factory))
|
||||
state.release_worker = asyncio.Event()
|
||||
request.app.include_router(coordination.router)
|
||||
request.app.include_router(thread_runs.router)
|
||||
|
||||
async def authenticated(scope, receive, send):
|
||||
token = set_current_user(request.state.user)
|
||||
try:
|
||||
await request.app({**scope, "state": dict(request.scope["state"])}, receive, send)
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
|
||||
try:
|
||||
async with AsyncClient(transport=ASGITransport(app=authenticated), base_url="http://gateway") as client:
|
||||
yield client, runs, state, entered, str(request.state.user.id)
|
||||
finally:
|
||||
await workers.close()
|
||||
await state.run_manager.shutdown()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _wait_for_worker(client, task_id, section, field, value):
|
||||
import asyncio
|
||||
|
||||
async with asyncio.timeout(12):
|
||||
while True:
|
||||
response = await client.get(f"/api/coordination/tasks/{task_id}")
|
||||
response.raise_for_status()
|
||||
view = response.json()
|
||||
if view[section][field] == value:
|
||||
return view
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("followup_report", [None, b"revised report"], ids=["text-followup", "same-name-file"])
|
||||
async def test_managed_worker_routes_followups_and_retains_the_producing_runs_file(worker_gateway, followup_report):
|
||||
import asyncio
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
from langgraph.types import Command
|
||||
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.persistence.coordination.sql import TaskNotFound
|
||||
|
||||
client, runs, state, entered, owner = worker_gateway
|
||||
|
||||
async def present_report(prompt, runtime):
|
||||
assert runtime.context["subagent_enabled"] is True
|
||||
report = b"original report" if prompt == "wait" else followup_report
|
||||
if report is not None:
|
||||
outputs = get_paths().sandbox_outputs_dir(runtime.context["thread_id"], user_id=owner)
|
||||
await asyncio.to_thread(outputs.mkdir, parents=True, exist_ok=True)
|
||||
await asyncio.to_thread((outputs / "report.txt").write_bytes, report)
|
||||
journal = runtime.context["__run_journal"]
|
||||
journal._remember_current_run_tool_calls(AIMessage(content="", tool_calls=[{"id": "present", "name": "present_files", "args": {}}]), caller="lead_agent")
|
||||
journal.on_tool_end(Command(update={"artifacts": ["/mnt/user-data/outputs/report.txt"], "messages": [ToolMessage("Presented", tool_call_id="present")]}), run_id=uuid4())
|
||||
|
||||
state.present_worker_report = present_report
|
||||
url = "/api/coordination/tasks"
|
||||
body = {"request_id": "dispatch", "objective": "wait"}
|
||||
first = (await client.post(url, json=body)).raise_for_status().json()
|
||||
task_id, thread, run = first["task"]["id"], first["assignment"]["thread_id"], first["execution"]["run_id"]
|
||||
assert first["task"]["project_id"] is None and (await client.post(url, json=body)).json()["execution"]["run_id"] == run
|
||||
await asyncio.wait_for(entered.wait(), 10)
|
||||
thread_url = f"/api/threads/{thread}"
|
||||
inbox_url = f"{url}/{task_id}/inputs"
|
||||
(await client.post(inbox_url, json={"request_id": "task-input", "content": "task follow-up"})).raise_for_status()
|
||||
followup = {"request_id": "fix", "mode": "queue", "text": "new requirement"}
|
||||
receipt = (await client.post(f"{thread_url}/runs/{run}/inputs", json=followup)).raise_for_status().json()
|
||||
assert receipt["status"] == "pending"
|
||||
assert (await client.post(f"{thread_url}/runs/{run}/inputs", json=followup)).json() == receipt
|
||||
direct = asyncio.create_task(client.post(f"{thread_url}/runs", json={"input": {"messages": [{"type": "human", "content": [{"type": "text", "text": "final instruction"}]}]}}))
|
||||
try:
|
||||
await _wait_for_worker(client, task_id, "task", "latest_input_seq", 3)
|
||||
pending = (await client.get(inbox_url)).raise_for_status().json()
|
||||
assert [(item["sequence"], item["source"], item["content"], item["status"]) for item in pending] == [
|
||||
(1, "task_api", "task follow-up", "pending"),
|
||||
(2, "run_input", "new requirement", "pending"),
|
||||
(3, "worker_chat", "final instruction", "pending"),
|
||||
]
|
||||
assert (await client.get(inbox_url, params={"after": 1, "limit": 1})).raise_for_status().json() == pending[1:2]
|
||||
assert (await client.get(f"{url}/missing-task/inputs")).status_code == 404
|
||||
finally:
|
||||
state.release_worker.set()
|
||||
(await direct).raise_for_status()
|
||||
result = await _wait_for_worker(client, task_id, "task", "status", "succeeded")
|
||||
final_run = result["execution"]["run_id"]
|
||||
assert final_run != run and result["task"]["result_ref"] == f"{thread_url}/runs/{final_run}/messages"
|
||||
assert (await direct).raise_for_status().json()["run_id"] == final_run
|
||||
assert (await runs.get_state(thread_id=thread))["values"]["messages"][-1]["content"] == "answer:final instruction"
|
||||
assert (await client.get(f"{thread_url}/inputs")).json()["items"][0]["run_id"] == final_run
|
||||
admitted = (await client.get(inbox_url)).raise_for_status().json()
|
||||
assert [(item["run_id"], item["status"]) for item in admitted] == [(final_run, "admitted")] * 3
|
||||
artifact_list = f"{url}/{task_id}/artifacts"
|
||||
artifacts = (await client.get(artifact_list)).raise_for_status().json()
|
||||
expected_versions = [(run, 1, b"original report")]
|
||||
if followup_report:
|
||||
expected_versions.append((final_run, 2, followup_report))
|
||||
assert [(item["run_id"], item["turn"]) for item in artifacts] == [(producer, turn) for producer, turn, _ in expected_versions]
|
||||
assert {item["source_path"] for item in artifacts} == {"/mnt/user-data/outputs/report.txt"}
|
||||
assert len({item["url"] for item in artifacts}) == len(expected_versions)
|
||||
assert result["execution"]["artifacts"] == (artifacts[1:] if followup_report else [])
|
||||
assert (await client.get(artifact_list, params={"limit": 1, "offset": 1})).raise_for_status().json() == artifacts[1:]
|
||||
assert (await client.get(f"{url}/missing-task/artifacts")).status_code == 404
|
||||
await asyncio.to_thread((get_paths().sandbox_outputs_dir(thread, user_id=owner) / "report.txt").unlink)
|
||||
for artifact, (_, _, content) in zip(artifacts, expected_versions, strict=True):
|
||||
assert (await client.get(artifact["url"])).raise_for_status().content == content
|
||||
with pytest.raises(TaskNotFound):
|
||||
await state.coordination_workers.repo.read_artifact("another-user", task_id, artifacts[0]["id"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_managed_worker_requires_explicit_resume_and_fences_cancellation(worker_gateway):
|
||||
import asyncio
|
||||
|
||||
client, runs, _, entered, _ = worker_gateway
|
||||
url = "/api/coordination/tasks"
|
||||
paused = (await client.post(url, json={"request_id": "pause", "objective": "approve"})).raise_for_status().json()
|
||||
task_id = paused["task"]["id"]
|
||||
await _wait_for_worker(client, task_id, "execution", "status", "waiting_for_input")
|
||||
target = f"/api/threads/{paused['assignment']['thread_id']}/runs"
|
||||
reply = {"json": {"command": {"resume": "confirmed"}}, "headers": {"Idempotency-Key": "reply"}}
|
||||
answered = (await client.post(target, **reply)).raise_for_status().json()
|
||||
result = await _wait_for_worker(client, task_id, "task", "status", "succeeded")
|
||||
assert result["task"]["result_ref"].endswith(f"/{answered['run_id']}/messages")
|
||||
assert (await client.post(target, **reply)).raise_for_status().json()["run_id"] == answered["run_id"]
|
||||
|
||||
other = (await client.post(url, json={"request_id": "cancel", "objective": "wait"})).raise_for_status().json()
|
||||
await asyncio.wait_for(entered.wait(), 10)
|
||||
task_id, thread = other["task"]["id"], other["assignment"]["thread_id"]
|
||||
cancelled = (await client.post(f"{url}/{task_id}/cancel", json={"expected_revision": other["task"]["revision"]})).raise_for_status().json()
|
||||
assert cancelled["status"] == "cancelled"
|
||||
assert (await runs.get(thread_id=thread, run_id=other["execution"]["run_id"])).status == "interrupted"
|
||||
assert (await client.post(f"/api/threads/{thread}/runs", json={"input": {"messages": [{"role": "user", "content": "too late"}]}})).status_code == 409
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_managed_worker_cancel_returns_cancelling_and_finishes_in_background(worker_gateway, monkeypatch):
|
||||
import asyncio
|
||||
|
||||
from app.gateway import coordination_workers
|
||||
|
||||
client, runs, _, entered, _ = worker_gateway
|
||||
started, release = asyncio.Event(), asyncio.Event()
|
||||
calls = []
|
||||
request_stop = type(runs).cancel
|
||||
|
||||
async def paused_stop(self, *, thread_id, run_id):
|
||||
calls.append(run_id)
|
||||
started.set()
|
||||
await release.wait()
|
||||
if len(calls) == 1:
|
||||
raise AgentRunError(503, "Run is temporarily unavailable")
|
||||
await request_stop(self, thread_id=thread_id, run_id=run_id)
|
||||
|
||||
monkeypatch.setattr(type(runs), "cancel", paused_stop)
|
||||
monkeypatch.setattr(coordination_workers, "CANCEL_RESPONSE_TIMEOUT", 0.05, raising=False)
|
||||
url = "/api/coordination/tasks"
|
||||
first = (await client.post(url, json={"objective": "wait", "request_id": "background-cancel"})).raise_for_status().json()
|
||||
task_id, run_id = first["task"]["id"], first["execution"]["run_id"]
|
||||
await asyncio.wait_for(entered.wait(), 10)
|
||||
cancel_url, body = f"{url}/{task_id}/cancel", {"expected_revision": first["task"]["revision"]}
|
||||
pending = asyncio.create_task(client.post(cancel_url, json=body))
|
||||
try:
|
||||
await asyncio.wait_for(started.wait(), 5)
|
||||
done, _ = await asyncio.wait({pending}, timeout=3)
|
||||
assert pending in done, "Cancellation response waited for the blocked run"
|
||||
response = pending.result()
|
||||
assert response.status_code == 202 and response.json()["status"] == "cancelling"
|
||||
assert (await client.post(cancel_url, json=body)).json()["status"] == "cancelling"
|
||||
assert calls == [run_id]
|
||||
assert (await client.post(f"{url}/{task_id}/inputs", json={"request_id": "late", "content": "too late"})).status_code == 409
|
||||
finally:
|
||||
release.set()
|
||||
await pending
|
||||
final = await _wait_for_worker(client, task_id, "task", "status", "cancelled")
|
||||
assert calls == [run_id, run_id]
|
||||
assert final["execution"]["status"] == "cancelled"
|
||||
assert (await runs.get(thread_id=first["assignment"]["thread_id"], run_id=run_id)).status == "interrupted"
|
||||
|
||||
@ -44,9 +44,10 @@ pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
async def test_0047_remains_in_the_single_migration_chain():
|
||||
assert bootstrap._get_head_revision() == "0056_coordination_task_state"
|
||||
assert bootstrap._get_head_revision() == "0057_coordination_workers"
|
||||
script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR))
|
||||
assert script.get_heads() == ["0056_coordination_task_state"]
|
||||
assert script.get_heads() == ["0057_coordination_workers"]
|
||||
assert script.get_revision("0057_coordination_workers").down_revision == "0056_coordination_task_state"
|
||||
assert script.get_revision("0056_coordination_task_state").down_revision == "0055_thread_shares"
|
||||
assert script.get_revision("0055_thread_shares").down_revision == "0054_project_memory"
|
||||
assert script.get_revision("0054_project_memory").down_revision == "0053_tool_approvals"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user