From 50d04ddf715677f2500b384db254025192f8179f Mon Sep 17 00:00:00 2001 From: Wenchao An Date: Fri, 20 Nov 2026 06:26:29 +0800 Subject: [PATCH] feat(coordination): run tasks on managed worker threads (#8387) * feat(coordination): run tasks on managed worker threads * fix(coordination): expose task artifact history * test(boxd): make partial timeout output check deterministic * fix(coordination): expose task inbox and bound cancellation waits --- backend/app/gateway/app.py | 2 + backend/app/gateway/coordination_workers.py | 238 ++++++++++++++++ backend/app/gateway/deps.py | 5 + backend/app/gateway/extension_agent_runs.py | 91 +++++- backend/app/gateway/routers/coordination.py | 127 +++++++++ backend/app/gateway/run_inputs.py | 39 +++ backend/app/gateway/services.py | 7 + backend/app/subagent_batches/continuation.py | 4 + backend/docs/task-coordination-state.md | 67 +++++ .../persistence/coordination/model.py | 40 ++- .../deerflow/persistence/coordination/sql.py | 60 ++-- .../persistence/coordination/workers.py | 264 ++++++++++++++++++ .../versions/0057_coordination_workers.py | 56 ++++ .../deerflow/persistence/models/__init__.py | 5 +- .../harness/deerflow/persistence/run/sql.py | 5 + backend/tests/test_boxd_sandbox_provider.py | 9 +- .../test_coordination_task_repository.py | 58 +++- backend/tests/test_extension_agent_runs.py | 209 +++++++++++++- .../test_migration_0047_mcp_oauth_tokens.py | 5 +- 19 files changed, 1254 insertions(+), 37 deletions(-) create mode 100644 backend/app/gateway/coordination_workers.py create mode 100644 backend/app/gateway/routers/coordination.py create mode 100644 backend/packages/harness/deerflow/persistence/coordination/workers.py create mode 100644 backend/packages/harness/deerflow/persistence/migrations/versions/0057_coordination_workers.py diff --git a/backend/app/gateway/app.py b/backend/app/gateway/app.py index 1320b62ef1..e5348c1895 100644 --- a/backend/app/gateway/app.py +++ b/backend/app/gateway/app.py @@ -31,6 +31,7 @@ from app.gateway.routers import ( channel_connections, channels, console, + coordination, features, feedback, github_webhooks, @@ -1358,6 +1359,7 @@ This gateway provides runtime endpoints for agent runs plus custom endpoints for # Durable MCP tasks are scoped to their owning thread. app.include_router(mcp_tasks.router) + app.include_router(coordination.router) app.include_router(subagent_batches.router) # Memory API is mounted at /api/memory diff --git a/backend/app/gateway/coordination_workers.py b/backend/app/gateway/coordination_workers.py new file mode 100644 index 0000000000..b203e0f2f9 --- /dev/null +++ b/backend/app/gateway/coordination_workers.py @@ -0,0 +1,238 @@ +"""Gateway execution of durable coordination tasks using caller-bound AgentRuns.""" + +import asyncio +import logging +from hashlib import sha256 +from uuid import uuid4 + +from deerflow_extension_api.agent_runs import AgentRunError +from fastapi import HTTPException + +from app.gateway.deps import get_agent_runs +from deerflow.persistence.coordination.model import TaskStatus +from deerflow.persistence.coordination.sql import TaskConflict, TaskNotFound +from deerflow.persistence.coordination.workers import THREAD_PREFIX + +logger = logging.getLogger(__name__) +CANCEL_RESPONSE_TIMEOUT = 5 + + +class CoordinationWorkers: + def __init__(self, repository): + self.repo = repository + self._drivers = {} + self._wakeups = set() + self._cancellations = {} + + async def close(self): + jobs = [*self._drivers.values(), *self._cancellations.values()] + for job in jobs: + job.cancel() + await asyncio.gather(*jobs, return_exceptions=True) + + async def dispatch(self, runs, owner, task_id, assistant_id): + turn = await self.repo.prepare(owner, task_id, assistant_id) + if turn is not None: + await self._admit(runs, owner, task_id, turn) + self.watch(runs, owner, task_id) + return await self.repo.view(owner, task_id) + + async def _admit(self, runs, owner, task_id, turn): + worker, execution, project = turn + thread = worker["thread_id"] + if execution["run_id"] is None: + try: + await runs.create_thread(thread_id=thread, assistant_id=worker["assistant_id"], project_id=project) + admitted = await runs.start_coordination_worker(task_id=task_id, execution_id=execution["id"]) + return admitted.run_id + except AgentRunError as exc: + if exc.status_code in (400, 403, 404, 422): + await self.repo.settle(owner, task_id, execution["id"], None, "admission_failed", []) + raise + return execution["run_id"] + + def watch(self, runs, owner, task_id): + key = (owner, task_id) + if (existing := self._drivers.get(key)) is not None and not existing.done(): + self._wakeups.add(key) + return existing + + async def drive(): + delay = 1 + while True: + self._wakeups.discard(key) + try: + await self._drive(runs, owner, task_id) + except AgentRunError as exc: + if exc.status_code < 500: + raise + await asyncio.sleep(delay) + delay = min(30, delay * 2) + continue + if key not in self._wakeups: + return + + task = asyncio.create_task(drive()) + self._drivers[key] = task + + def finished(job): + if self._drivers.get(key) is job: + self._drivers.pop(key) + self._wakeups.discard(key) + if not job.cancelled() and (error := job.exception()) is not None: + logger.warning("Coordination worker observation stopped for task %s (%s)", task_id, type(error).__name__) + + task.add_done_callback(finished) + return task + + async def _drive(self, runs, owner, task_id): + while (turn := await self.repo.prepare(owner, task_id)) is not None: + worker, execution, _project = turn + if execution["status"] == "waiting_for_input": + return + if execution["run_id"] is None: + await self._admit(runs, owner, task_id, turn) + continue + thread = worker["thread_id"] + try: + run = await runs.wait_coordination_worker(thread_id=thread, run_id=execution["run_id"], timeout=60) + except TimeoutError: + continue + artifacts = [] + status = run.status + if status == "success" and not run.stop_reason: + try: + artifacts = await runs.coordination_artifacts(thread_id=thread, run_id=run.run_id) + except AgentRunError as exc: + if exc.status_code >= 500: + raise + status = "delivery_failed" + elif status == "success": + status = "stopped" + await self.repo.settle(owner, task_id, execution["id"], run.run_id, status, artifacts) + + async def cancel(self, runs, owner, task_id, revision): + task = await self.repo.request_cancel(owner, task_id, expected_revision=revision) + if task["status"] == TaskStatus.CANCELLED: + return task + key = (owner, task_id) + job = self._cancellations.get(key) + if job is None or job.done(): + job = asyncio.create_task(self._finish_cancellation(runs, owner, task_id, task["execution_epoch"])) + self._cancellations[key] = job + + def finished(completed): + if self._cancellations.get(key) is completed: + self._cancellations.pop(key) + if not completed.cancelled() and (error := completed.exception()) is not None: + logger.warning("Coordination cancellation stopped for task %s (%s)", task_id, type(error).__name__) + + job.add_done_callback(finished) + # asyncio.wait leaves the tracked job running after the response deadline. + done, _ = await asyncio.wait({job}, timeout=CANCEL_RESPONSE_TIMEOUT) + return job.result() if done else task + + async def _finish_cancellation(self, runs, owner, task_id, epoch): + delay = 1 + while True: + try: + await self._stop_subtree(runs, owner, task_id) + return await self.repo.confirm_cancel(owner, task_id, execution_epoch=epoch) + except AgentRunError as exc: + if exc.status_code < 500: + raise + await asyncio.sleep(delay) + delay = min(30, delay * 2) + + async def _stop_subtree(self, runs, owner, task_id): + active = await self.repo.cancellations(owner, task_id) + for thread, run_id in active: + run = await runs.get(thread_id=thread, run_id=run_id) + if run.status in ("pending", "running"): + try: + await runs.cancel(thread_id=thread, run_id=run_id) + except AgentRunError as exc: + if exc.status_code != 409 or (await runs.get(thread_id=thread, run_id=run_id)).status in ("pending", "running"): + raise + for thread, run_id in active: + while True: + try: + await runs.wait_coordination_worker(thread_id=thread, run_id=run_id, timeout=60) + break + except TimeoutError: + continue + for batch in await self.repo.active_batches(owner, task_id): + await runs.cancel_worker_batch(thread_id=batch.thread_id, batch_id=batch.id) + + +def worker_context(request): + service = getattr(request.app.state, "coordination_workers", None) + if service is None: + raise HTTPException(503, "Coordination workers require SQL persistence") + runs = get_agent_runs(request) + if runs is None: + raise HTTPException(403, "Coordination workers require a user session") + return service, str(request.state.user.id), runs.for_plugin("native-coordination") + + +def worker_error(exc): + if isinstance(exc, TaskNotFound): + return HTTPException(404, "Task not found") + if isinstance(exc, AgentRunError): + return HTTPException(exc.status_code, str(exc) if exc.status_code < 500 else "Worker host unavailable") + return HTTPException(409, str(exc)) + + +async def managed_task(request, thread_id): + if not thread_id.startswith(THREAD_PREFIX): + return None + service = getattr(request.app.state, "coordination_workers", None) + return await service.repo.task_for_thread(thread_id) if service is not None else None + + +async def direct_worker_run(request, task_id, body, key): + from app.gateway.deps import get_run_manager + + service, owner, runs = worker_context(request) + if getattr(request.state, "run_origin", None) or body.checkpoint or body.checkpoint_id or body.conversation_references or body.multitask_strategy != "reject": + raise HTTPException(409, "Use a task input to continue this managed worker") + try: + if body.command is not None: + if set(body.command) != {"resume"} or body.command["resume"] is None or body.input: + raise HTTPException(422, "An explicit interrupt response is required") + view = await service.repo.view(owner, task_id) + current = view["execution"] + if current and current["status"] == "running" and current["run_id"]: + native = await runs.get(thread_id=view["assignment"]["thread_id"], run_id=current["run_id"]) + if native.status == "waiting_for_input": + await service.repo.settle(owner, task_id, current["id"], native.run_id, native.status, []) + turn = await service.repo.resume(owner, task_id, key=key or uuid4().hex, value=body.command["resume"]) + run_id = await service._admit(runs, owner, task_id, turn) + service.watch(runs, owner, task_id) + else: + messages = (body.input or {}).get("messages") + if not isinstance(messages, list) or len(messages) != 1 or not isinstance(messages[0], dict): + raise HTTPException(422, "Managed workers accept one text message") + message = messages[0] + content = message.get("content") + if isinstance(content, list) and all(isinstance(part, dict) and part.get("type") == "text" and isinstance(part.get("text"), str) for part in content): + content = "".join(part["text"] for part in content) + if message.get("role", message.get("type")) not in ("user", "human") or not isinstance(content, str) or not 0 < len(content.strip()) <= 16384: + raise HTTPException(422, "Managed workers accept one text message") + identifier = key or message.get("id") or uuid4().hex + if not isinstance(identifier, str): + raise HTTPException(422, "Message ID must be a string") + row = await service.repo.append_input(owner, task_id, request_id="chat:" + sha256(identifier.encode()).hexdigest(), content=content) + driver = service.watch(runs, owner, task_id) + # Streaming and wait endpoints must observe the Run that consumed this input. + while not (run_id := await service.repo.run_for_input(owner, task_id, row["sequence"])): + if driver.done(): + driver.result() + raise HTTPException(409, "Input queued; the worker is waiting for an explicit interrupt response") + await asyncio.sleep(0.25) + record = await get_run_manager(request).get(run_id, user_id=owner) + if record is None: + raise HTTPException(503, "Worker run is unavailable") + return record + except (TaskNotFound, TaskConflict, AgentRunError) as exc: + raise worker_error(exc) from exc diff --git a/backend/app/gateway/deps.py b/backend/app/gateway/deps.py index ee8c1ab53d..57e45c3da4 100644 --- a/backend/app/gateway/deps.py +++ b/backend/app/gateway/deps.py @@ -963,10 +963,14 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen stack.callback(reset_login_throttle_store) if sf is not None: + from app.gateway.coordination_workers import CoordinationWorkers + from deerflow.persistence.coordination.workers import CoordinationWorkerRepository from deerflow.persistence.feedback import FeedbackRepository from deerflow.persistence.personal_access_tokens import PersonalAccessTokenRepository from deerflow.persistence.run import RunRepository + app.state.coordination_workers = CoordinationWorkers(CoordinationWorkerRepository(sf)) + stack.push_async_callback(app.state.coordination_workers.close) app.state.run_store = RunRepository(sf) app.state.feedback_repo = FeedbackRepository(sf) from app.gateway.auth.pat import PAT_LAST_USED_WRITE_INTERVAL_SECONDS @@ -975,6 +979,7 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen else: from deerflow.runtime.runs.store.memory import MemoryRunStore + app.state.coordination_workers = None app.state.run_store = MemoryRunStore() app.state.feedback_repo = None # Memory backend has no durable PAT store, so Bearer credentials diff --git a/backend/app/gateway/extension_agent_runs.py b/backend/app/gateway/extension_agent_runs.py index 28fb29d11a..36ad5afe7d 100644 --- a/backend/app/gateway/extension_agent_runs.py +++ b/backend/app/gateway/extension_agent_runs.py @@ -305,11 +305,11 @@ class _BoundAgentRuns: return await asyncio.to_thread(snapshot) - async def create_thread(self, *, assistant_id="lead_agent", thread_id=None, metadata=None) -> str: + async def create_thread(self, *, assistant_id="lead_agent", thread_id=None, metadata=None, project_id=None) -> str: from app.gateway.routers.threads import ThreadCreateRequest, create_thread async with self._request("threads:write") as request: - response = await create_thread(ThreadCreateRequest(assistant_id=assistant_id, thread_id=thread_id, metadata=_json_object(metadata) or {}), request) + response = await create_thread(ThreadCreateRequest(assistant_id=assistant_id, thread_id=thread_id, project_id=project_id, metadata=_json_object(metadata) or {}), request) return response.thread_id async def _start(self, *, thread_id, input=None, resume=None, context=None, idempotency_key=None, execution_fence=None, artifact_imports=None): @@ -350,6 +350,93 @@ class _BoundAgentRuns: async def start(self, *, thread_id, input, context=None, idempotency_key=None, execution_fence=None, artifact_imports=None) -> AgentRun: return await self._start(thread_id=thread_id, input=input, context=context, idempotency_key=idempotency_key, execution_fence=execution_fence, artifact_imports=artifact_imports) + async def start_coordination_worker(self, *, task_id, execution_id): + from app.gateway.run_models import RunCreateRequest + from app.gateway.services import start_run + from deerflow.persistence.coordination.sql import TaskConflict, TaskNotFound + from deerflow.persistence.coordination.workers import RUN_KEY_PREFIX + from deerflow.persistence.thread_meta.base import THREAD_PROJECT_METADATA_KEY + + async with self._request("runs:create") as request: + try: + worker, frozen, project_id = await request.app.state.coordination_workers.repo.admission(self._user_id, task_id, execution_id) + except TaskNotFound as exc: + raise AgentRunError(404, "Worker task unavailable") from exc + except TaskConflict as exc: + raise AgentRunError(409, str(exc)) from exc + thread_id = worker["thread_id"] + async with self._request("runs:create", thread_id) as request: + record = await request.app.state.thread_store.get(thread_id, user_id=self._user_id) + if record is None or record.get("assistant_id") != worker["assistant_id"] or (record.get("metadata") or {}).get(THREAD_PROJECT_METADATA_KEY) != project_id: + raise AgentRunError(409, "Worker thread binding changed") + body = RunCreateRequest( + assistant_id=worker["assistant_id"], + input=None if "command" in frozen else frozen, + command=frozen.get("command"), + context={"subagent_enabled": True}, + on_disconnect="continue", + ) + request.state.coordination_execution_id = execution_id + record = await start_run(body, thread_id, request, require_existing_thread=True, idempotency_key=RUN_KEY_PREFIX + execution_id) + return _view(record) + + async def wait_coordination_worker(self, *, thread_id, run_id, timeout=60): + """Observe the committed terminal status, written after the delivery receipt.""" + from app.gateway.deps import get_run_store + + async with asyncio.timeout(timeout): + delay = 0.25 + while True: + async with self._request("runs:read", thread_id) as request: + row = await get_run_store(request).get(run_id, user_id=self._user_id) + if row is None or row["thread_id"] != thread_id: + raise AgentRunError(404, "Worker run unavailable") + if row["status"] not in ("pending", "running"): + return AgentRun(thread_id, run_id, row["status"], row.get("assistant_id"), row.get("stop_reason")) + await asyncio.sleep(delay) + delay = min(delay * 2, 4) + + async def cancel_worker_batch(self, *, thread_id, batch_id): + from app.gateway.deps import get_subagent_batch_service + + async with self._request("runs:cancel", thread_id) as request: + await get_subagent_batch_service(request).cancel_batch(batch_id=batch_id, user_id=self._user_id, continuation_authority=self) + + async def coordination_artifacts(self, *, thread_id, run_id): + from app.gateway.artifact_archive import MAX_FILE_BYTES, MAX_FILES, MAX_TOTAL_BYTES + from app.gateway.deps import get_run_event_store + from app.gateway.extension_artifacts import read_snapshot + from app.gateway.routers.artifacts import reserve_artifact_write + from deerflow.constants import DELIVERY_PRESENTATION_TOOL_NAMES + from deerflow.runtime.runs.manager import ConflictError + from deerflow.utils.file_io import await_drained + + async with self._request("runs:read", thread_id) as request: + events = await get_run_event_store(request).list_events(thread_id, run_id, event_types=["run.delivery"], limit=2, user_id=self._user_id) + if len(events) != 1: + raise AgentRunError(409, "Run delivery receipt is unavailable") + by_tool = (events[0].get("content") or {}).get("by_tool") + if not isinstance(by_tool, dict) or any(not isinstance(paths, list) or any(not isinstance(path, str) for path in paths) for paths in by_tool.values()): + raise AgentRunError(409, "Invalid run delivery receipt") + paths = dict.fromkeys(path for tool in DELIVERY_PRESENTATION_TOOL_NAMES for path in by_tool.get(tool, [])) + if not paths: + return [] + if len(paths) > MAX_FILES: + raise AgentRunError(422, "Too many task artifacts") + artifacts = [] + remaining = MAX_TOTAL_BYTES + async with self._request("threads:read", thread_id) as request: + try: + async with reserve_artifact_write(request, thread_id, user_id=self._user_id): + for path in paths: + content = await await_drained(read_snapshot(request, self._user_id, thread_id, path, max_bytes=min(MAX_FILE_BYTES, remaining))) + digest = await asyncio.to_thread(hashlib.sha256, content) + artifacts.append((path, digest.hexdigest(), content)) + remaining -= len(content) + except ConflictError as exc: + raise AgentRunError(503, "Worker artifacts are temporarily busy") from exc + return artifacts + async def start_batch_followup(self, *, thread_id, batch_id, input, context, config, execution_fence, goal_wakeup=None): """Native single follow-up; the existing SQL admission fences cancellation.""" from app.gateway.run_models import RunCreateRequest diff --git a/backend/app/gateway/routers/coordination.py b/backend/app/gateway/routers/coordination.py new file mode 100644 index 0000000000..c81f7ae847 --- /dev/null +++ b/backend/app/gateway/routers/coordination.py @@ -0,0 +1,127 @@ +"""Session-authorized task execution shared by coordination entry points.""" + +from hashlib import sha256 +from pathlib import PurePosixPath +from typing import Annotated + +from deerflow_extension_api.agent_runs import AgentRunError +from fastapi import APIRouter, HTTPException, Query, Request +from fastapi.responses import Response +from pydantic import BaseModel, Field, StringConstraints + +from app.gateway.authz import require_permission +from app.gateway.coordination_workers import worker_context, worker_error +from deerflow.persistence.coordination.model import TaskStatus +from deerflow.persistence.coordination.sql import TaskConflict, TaskNotFound +from deerflow.utils.file_io import await_drained + +router = APIRouter(prefix="/api/coordination/tasks", tags=["coordination"]) +RequestId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=200)] +TaskText = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=16384)] + + +class CreateWorkerTask(BaseModel): + request_id: RequestId + objective: TaskText + assistant_id: str = Field(default="lead_agent", min_length=1, max_length=128) + project_id: str | None = Field(default=None, min_length=1, max_length=64) + + +class WorkerInput(BaseModel): + request_id: RequestId + content: TaskText + + +class CancelTask(BaseModel): + expected_revision: int = Field(ge=0) + + +@router.post("") +@require_permission("runs", "create") +async def create_worker_task(body: CreateWorkerTask, request: Request): + service, owner, runs = worker_context(request) + + async def dispatch(): + task = await service.repo.create(owner, request_id=body.request_id, objective=body.objective, project_id=body.project_id) + return await service.dispatch(runs, owner, task["id"], body.assistant_id) + + try: + return await await_drained(dispatch()) + except (TaskConflict, TaskNotFound, AgentRunError) as exc: + raise worker_error(exc) from exc + + +@router.get("/{task_id}") +@require_permission("runs", "read") +async def get_worker_task(task_id: str, request: Request): + service, owner, _runs = worker_context(request) + try: + return await service.repo.view(owner, task_id) + except TaskNotFound as exc: + raise HTTPException(404, "Task not found") from exc + + +@router.post("/{task_id}/inputs") +@require_permission("runs", "create") +async def add_worker_input(task_id: str, body: WorkerInput, request: Request): + service, owner, runs = worker_context(request) + try: + item = await service.repo.append_input(owner, task_id, request_id="input:" + sha256(body.request_id.encode()).hexdigest(), content=body.content) + service.watch(runs, owner, task_id) + return item + except (TaskNotFound, TaskConflict) as exc: + raise worker_error(exc) from exc + + +@router.get("/{task_id}/inputs") +@require_permission("runs", "read") +async def list_worker_inputs(task_id: str, request: Request, after: int = Query(0, ge=0, le=2**63 - 1), limit: int = Query(100, ge=1, le=100)): + service, owner, _runs = worker_context(request) + try: + return await service.repo.list_worker_inputs(owner, task_id, after=after, limit=limit) + except TaskNotFound as exc: + raise HTTPException(404, "Task not found") from exc + + +@router.post("/{task_id}/cancel", responses={202: {"description": "Cancellation is still in progress"}}) +@require_permission("runs", "cancel") +async def cancel_worker_task(task_id: str, body: CancelTask, request: Request, response: Response): + service, owner, runs = worker_context(request) + try: + task = await await_drained(service.cancel(runs, owner, task_id, body.expected_revision)) + except (TaskConflict, TaskNotFound, AgentRunError) as exc: + raise worker_error(exc) from exc + if task["status"] == TaskStatus.CANCELLING: + response.status_code = 202 + return task + + +@router.get("/{task_id}/artifacts") +@require_permission("threads", "read") +async def list_worker_artifacts(task_id: str, request: Request, limit: int = Query(100, ge=1, le=100), offset: int = Query(0, ge=0)): + service, owner, _runs = worker_context(request) + try: + return await service.repo.list_artifacts(owner, task_id, limit=limit, offset=offset) + except TaskNotFound as exc: + raise HTTPException(404, "Task not found") from exc + + +@router.get("/{task_id}/artifacts/{artifact_id}") +@require_permission("threads", "read") +async def download_worker_artifact(task_id: str, artifact_id: str, request: Request): + from app.gateway.routers.artifacts import _build_content_disposition + + service, owner, _runs = worker_context(request) + try: + path, content = await service.repo.read_artifact(owner, task_id, artifact_id) + except TaskNotFound as exc: + raise HTTPException(404, "Artifact not found") from exc + return Response( + content, + media_type="application/octet-stream", + headers={ + "Content-Disposition": _build_content_disposition("attachment", PurePosixPath(path).name), + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + }, + ) diff --git a/backend/app/gateway/run_inputs.py b/backend/app/gateway/run_inputs.py index 236a9450ff..e66daf7248 100644 --- a/backend/app/gateway/run_inputs.py +++ b/backend/app/gateway/run_inputs.py @@ -46,6 +46,19 @@ async def submit_run_input(thread_id: str, run_id: str, body: RunInputRequest, r raise HTTPException(404, "Run not found") if get_agent_runs(request) is None: raise HTTPException(503, "Run inputs require an authenticated session") + from app.gateway.coordination_workers import managed_task, worker_context, worker_error + from deerflow.persistence.coordination.sql import TaskConflict, TaskNotFound + + if task_id := await managed_task(request, thread_id): + service, owner, runs = worker_context(request) + if body.mode != "queue": + raise HTTPException(409, "Use queue mode to update a managed worker") + try: + item = await service.repo.append_input(owner, task_id, request_id=f"run-input:{run_id}:{body.request_id}", content=body.text) + service.watch(runs, owner, task_id) + return _worker_input_receipt(item, run_id, body.request_id) + except (TaskConflict, TaskNotFound) as exc: + raise worker_error(exc) from exc row = await _store(request).get(run_id, user_id=user_id) if row is None or row.get("run_inputs") is None: raise HTTPException(409, "This Run has not enabled interactive inputs") @@ -63,10 +76,36 @@ async def submit_run_input(thread_id: str, run_id: str, body: RunInputRequest, r return {**item, "target_run_id": run_id} +def _worker_input_receipt(item, run_id, request_id, consuming_run=None, cancelled=False): + return { + "id": request_id, + "target_run_id": run_id, + "message_id": str(uuid5(NAMESPACE_URL, f"deerflow-input:{run_id}:{request_id}")), + "mode": "queue", + "text": item["content"], + "created_at": item["created_at"], + "status": "applied" if consuming_run else "cancelled" if cancelled else "pending", + **({"run_id": consuming_run} if consuming_run else {}), + } + + async def list_thread_inputs(thread_id: str, request: Request): user_id = await get_current_user(request) if user_id is None: raise HTTPException(401, "Authentication required") + from app.gateway.coordination_workers import managed_task + from deerflow.persistence.coordination.sql import TaskNotFound + + if task_id := await managed_task(request, thread_id): + try: + receipts, active, cancelled = await request.app.state.coordination_workers.repo.input_receipts(user_id, task_id) + except TaskNotFound as exc: + raise HTTPException(404, "Task not found") from exc + items = [] + for item, consuming_run in receipts: + _, run, key = item["request_id"].split(":", 2) + items.append(_worker_input_receipt(item, run, key, consuming_run, cancelled)) + return {"items": items, "active_run_id": active} rows = await _store(request).list_input_runs(thread_id, user_id=user_id) items = [] for row in reversed(rows): diff --git a/backend/app/gateway/services.py b/backend/app/gateway/services.py index 541fa1a6de..1777dce7d5 100644 --- a/backend/app/gateway/services.py +++ b/backend/app/gateway/services.py @@ -2562,6 +2562,13 @@ async def start_run( if not await thread_access_allowed(): raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found") + from app.gateway.coordination_workers import direct_worker_run, managed_task + from deerflow.persistence.coordination.workers import THREAD_PREFIX + + if thread_id.startswith(THREAD_PREFIX) and not getattr(request.state, "coordination_execution_id", None): + if task_id := await managed_task(request, thread_id): + return await direct_worker_run(request, task_id, body, idempotency_key) + owner_context_token = set_current_user(SimpleNamespace(id=owner_user_id)) if owner_user_id else None try: is_internal_caller = getattr(getattr(request, "state", None), "auth_source", None) == AUTH_SOURCE_INTERNAL diff --git a/backend/app/subagent_batches/continuation.py b/backend/app/subagent_batches/continuation.py index 59afbdffa6..e613b03f7c 100644 --- a/backend/app/subagent_batches/continuation.py +++ b/backend/app/subagent_batches/continuation.py @@ -21,7 +21,11 @@ class BatchContinuations: async def prepare(self, request): from app.gateway.services import _CONTEXT_CONFIGURABLE_KEYS, _MODEL_SELECTION_CONTEXT_KEYS + from deerflow.persistence.coordination.workers import THREAD_PREFIX + workers = getattr(self.app.state, "coordination_workers", None) + if workers is not None and request.thread_id.startswith(THREAD_PREFIX) and await workers.repo.task_for_thread(request.thread_id): + raise AgentRunError(409, "A managed worker accepts follow-ups through its coordination task") host = self.app.state.agent_runs_host grant = host.capture_batch_authority(request.continuation_authority, user_id=request.user_id) source = await request.continuation_authority.get(thread_id=request.thread_id, run_id=request.run_id) diff --git a/backend/docs/task-coordination-state.md b/backend/docs/task-coordination-state.md index 1835e22388..8a210f2f02 100644 --- a/backend/docs/task-coordination-state.md +++ b/backend/docs/task-coordination-state.md @@ -29,3 +29,70 @@ prerequisites keep their state. A running caller can record cancellation once it has confirmed that the subtree's executions have stopped. The repository handles work state. Host admission, run control, and artifact authorization belong to the execution integration. + +## Managed workers + +A signed-in caller can dispatch a task through the Gateway on a SQL-backed +installation. The following request starts a worker without a project. Pass +an optional `project_id` to create its thread in an existing project. + +```http +POST /api/coordination/tasks +Content-Type: application/json + +{"request_id":"review-27","objective":"Review the report","assistant_id":"lead_agent"} +``` + +The response includes the task, its assignment (agent and thread), and the +latest execution. Replaying the request reuses the worker. Read the same +view with `GET /api/coordination/tasks/{task_id}`. Add a follow-up with +`POST /api/coordination/tasks/{task_id}/inputs` and a JSON object containing +`request_id` and `content`. A text message sent to the worker thread uses +the same queue. Streaming and wait callers observe the run that consumes +their message. An interrupted graph requires an explicit resume command. + +The task owner can poll `GET /api/coordination/tasks/{task_id}/inputs` +to read the inbox in sequence order. Pass the last seen `sequence` as +`after` and set `limit` up to 100. Rows include the content, creation +time, and source (`task_api`, `run_input`, or `worker_chat`). +An input is `pending` while it can still enter a Run, `admitted` +with its `run_id` once a Run has accepted the turn, or `not_admitted` +when the task has closed to further work. + +The Gateway freezes each turn before admission. The SQL transaction that +creates the native run locks the task tree, verifies the execution epoch, +and records the run ID. A follow-up that arrives during a turn advances +the input sequence. When that run stops, the worker continues on its own +thread with the outstanding inputs. Each task has one assignee per epoch. +Workers can use the agent's allowed subagents for local delegation. Background +batches can run local work. A batch cannot schedule an independent follow-up +run on a managed thread; the task owns the next turn. + +`POST /api/coordination/tasks/{task_id}/cancel` accepts `expected_revision`. +It fences the owned subtree before requesting run and native batch +cancellation. If confirmation takes longer than five seconds after the +fence is committed, the response is HTTP 202 with status `cancelling`. +The Gateway continues confirming in the background and publishes +`cancelled` once the runs have stopped and batch cancellation is requested. +Poll `GET /api/coordination/tasks/{task_id}` for the status. + +A completed task links to the producing run's messages. For presented output +files, the Gateway stores the captured bytes with their SHA-256 hashes and +producing execution. The owner's task artifact URL continues to serve those +bytes after the source file changes. Capture uses the existing archive budgets +of 50 files, 50 MiB per file and 100 MiB per execution. The execution reports +`delivery_failed` if a presented file cannot be captured. + +Use `GET /api/coordination/tasks/{task_id}/artifacts` to list the task's +deliveries, including files from earlier executions. The response lists +artifact IDs, source paths, hashes, sizes, producing `run_id` and `turn`, +and download URLs. Results are ordered by turn, then source path and ID. +Use `limit` (default and maximum 100) and `offset` (default 0) to page +through them. Files presented under the same path in different turns +have separate IDs and URLs. The task detail's `execution.artifacts` lists +deliveries produced by that execution. + +Worker observation uses the initiating session's process-local run handle. +The task, frozen inputs, native run IDs and artifact contents survive a Gateway +restart. Automatic reauthorization and restart reconciliation are tracked in +the next stage of [RFC #8345](https://github.com/bytedance/deer-flow/issues/8345). diff --git a/backend/packages/harness/deerflow/persistence/coordination/model.py b/backend/packages/harness/deerflow/persistence/coordination/model.py index 3b508f2f2e..2af92c6c64 100644 --- a/backend/packages/harness/deerflow/persistence/coordination/model.py +++ b/backend/packages/harness/deerflow/persistence/coordination/model.py @@ -3,7 +3,7 @@ from datetime import datetime from enum import StrEnum -from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Index, String, Text, UniqueConstraint +from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, ForeignKeyConstraint, Index, LargeBinary, String, Text, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column from deerflow.persistence.base import Base @@ -52,3 +52,41 @@ class CoordinationInputRow(Base): request_id: Mapped[str] = mapped_column(String(200)) content: Mapped[str] = mapped_column(Text) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class CoordinationAssignmentRow(Base): + """One active worker per task. Parallel work uses child tasks.""" + + __tablename__ = "coord_assignments" + task_id: Mapped[str] = mapped_column(ForeignKey("coord_tasks.id"), primary_key=True) + epoch: Mapped[int] = mapped_column(BigInteger, primary_key=True) + thread_id: Mapped[str] = mapped_column(String(64), unique=True) + assistant_id: Mapped[str] = mapped_column(String(128)) + + +class CoordinationExecutionRow(Base): + __tablename__ = "coord_executions" + __table_args__ = ( + ForeignKeyConstraint(["task_id", "epoch"], ["coord_assignments.task_id", "coord_assignments.epoch"]), + UniqueConstraint("task_id", "turn", name="uq_coord_execution_turn"), + UniqueConstraint("task_id", "resume_key", name="uq_coord_execution_resume"), + ) + id: Mapped[str] = mapped_column(String(64), primary_key=True) + task_id: Mapped[str] = mapped_column(String(64)) + epoch: Mapped[int] = mapped_column(BigInteger) + turn: Mapped[int] = mapped_column(BigInteger) + resume_key: Mapped[str | None] = mapped_column(String(200)) + covered_seq: Mapped[int] = mapped_column(BigInteger) + frozen_input: Mapped[dict] = mapped_column(JSON) + status: Mapped[str] = mapped_column(String(32)) + run_id: Mapped[str | None] = mapped_column(String(64), unique=True) + + +class CoordinationArtifactRow(Base): + __tablename__ = "coord_artifacts" + id: Mapped[str] = mapped_column(String(64), primary_key=True) + execution_id: Mapped[str] = mapped_column(ForeignKey("coord_executions.id"), index=True) + path: Mapped[str] = mapped_column(Text) + sha256: Mapped[str] = mapped_column(String(64)) + size: Mapped[int] = mapped_column(BigInteger) + content: Mapped[bytes] = mapped_column(LargeBinary) diff --git a/backend/packages/harness/deerflow/persistence/coordination/sql.py b/backend/packages/harness/deerflow/persistence/coordination/sql.py index 027e70d7de..59c079deef 100644 --- a/backend/packages/harness/deerflow/persistence/coordination/sql.py +++ b/backend/packages/harness/deerflow/persistence/coordination/sql.py @@ -176,21 +176,24 @@ class CoordinationTaskRepository: async with self._write() as session: task = await self._lock_tree(session, owner_id, task_id) _check_revision(task, expected_revision) - if task.status != TaskStatus.QUEUED: - raise TaskConflict("Only a queued task can start") - if task.parent_id is not None: - parent = await self._lookup(session, owner_id, task.parent_id) - if parent.status != TaskStatus.RUNNING: - raise TaskConflict("Parent task has not started") - if task.depends_on: - query = select(CoordinationTaskRow.id).filter_by(owner_id=owner_id, status=TaskStatus.SUCCEEDED) - ready = set((await session.scalars(query.where(CoordinationTaskRow.id.in_(task.depends_on)))).all()) - if ready != set(task.depends_on): - raise TaskConflict("Task dependencies have not succeeded") - task.status = TaskStatus.RUNNING - _touch(task) + await self._start_locked(session, task) return _snapshot(task) + async def _start_locked(self, session: AsyncSession, task: CoordinationTaskRow) -> None: + if task.status != TaskStatus.QUEUED: + raise TaskConflict("Only a queued task can start") + if task.parent_id is not None: + parent = await self._lookup(session, task.owner_id, task.parent_id) + if parent.status != TaskStatus.RUNNING: + raise TaskConflict("Parent task has not started") + if task.depends_on: + query = select(CoordinationTaskRow.id).filter_by(owner_id=task.owner_id, status=TaskStatus.SUCCEEDED) + ready = set((await session.scalars(query.where(CoordinationTaskRow.id.in_(task.depends_on)))).all()) + if ready != set(task.depends_on): + raise TaskConflict("Task dependencies have not succeeded") + task.status = TaskStatus.RUNNING + _touch(task) + async def finish( self, owner_id: str, @@ -211,15 +214,18 @@ class CoordinationTaskRepository: _check_revision(task, expected_revision) if task.latest_input_seq != covered_input_seq: raise TaskConflict("Result does not cover the current task inputs") - children = (await session.scalars(select(CoordinationTaskRow.status).filter_by(parent_id=task_id, owner_id=owner_id))).all() - accepted = {TaskStatus.SUCCEEDED} if outcome == TaskStatus.SUCCEEDED else {TaskStatus.SUCCEEDED, TaskStatus.FAILED, TaskStatus.CANCELLED} - if any(status not in accepted for status in children): - raise TaskConflict("Child tasks have not settled for this outcome") - task.status = outcome - task.result_ref = result_ref - _touch(task) + await self._finish_locked(session, task, outcome, result_ref) return _snapshot(task) + async def _finish_locked(self, session: AsyncSession, task: CoordinationTaskRow, outcome: TaskStatus, result_ref: str) -> None: + children = (await session.scalars(select(CoordinationTaskRow.status).filter_by(parent_id=task.id, owner_id=task.owner_id))).all() + accepted = {TaskStatus.SUCCEEDED} if outcome == TaskStatus.SUCCEEDED else {TaskStatus.SUCCEEDED, TaskStatus.FAILED, TaskStatus.CANCELLED} + if any(status not in accepted for status in children): + raise TaskConflict("Child tasks have not settled for this outcome") + task.status = outcome + task.result_ref = result_ref + _touch(task) + async def _subtree(self, session: AsyncSession, task: CoordinationTaskRow) -> list[CoordinationTaskRow]: rows = (await session.scalars(select(CoordinationTaskRow).filter_by(owner_id=task.owner_id, root_id=task.root_id))).all() children = defaultdict(list) @@ -255,8 +261,20 @@ class CoordinationTaskRepository: task = await self._lock_tree(session, owner_id, task_id) if task.status not in (TaskStatus.CANCELLING, TaskStatus.CANCELLED) or task.execution_epoch != execution_epoch: raise TaskConflict("Cancellation no longer matches the task") - for row in await self._subtree(session, task): + rows = await self._subtree(session, task) + await self._confirm_stopped(session, rows) + for row in rows: if row.status == TaskStatus.CANCELLING: row.status = TaskStatus.CANCELLED _touch(row) return _snapshot(task) + + async def _confirm_stopped(self, session, rows): + """Close active execution records in the same transaction as their tasks.""" + from sqlalchemy import update + + from deerflow.persistence.coordination.model import CoordinationExecutionRow + + ids = [row.id for row in rows if row.status == TaskStatus.CANCELLING] + if ids: + await session.execute(update(CoordinationExecutionRow).where(CoordinationExecutionRow.task_id.in_(ids), CoordinationExecutionRow.status.in_(("admitting", "running", "waiting_for_input"))).values(status="cancelled")) diff --git a/backend/packages/harness/deerflow/persistence/coordination/workers.py b/backend/packages/harness/deerflow/persistence/coordination/workers.py new file mode 100644 index 0000000000..6806459d5b --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/coordination/workers.py @@ -0,0 +1,264 @@ +"""Durable admission, cancellation and result ownership for managed workers.""" + +from uuid import uuid4 + +from sqlalchemy import select + +from deerflow.persistence.coordination.model import CoordinationArtifactRow as Artifact +from deerflow.persistence.coordination.model import CoordinationAssignmentRow as Assignment +from deerflow.persistence.coordination.model import CoordinationExecutionRow as Execution +from deerflow.persistence.coordination.model import CoordinationInputRow, CoordinationTaskRow, TaskStatus +from deerflow.persistence.coordination.sql import CoordinationTaskRepository, TaskConflict, TaskNotFound + +THREAD_PREFIX = "coord-worker-" +RUN_KEY_PREFIX = "coord-execution:" +INPUT_SOURCES = {"input": "task_api", "chat": "worker_chat", "run-input": "run_input"} + + +async def bind_admitted_worker(session, thread_id, owner_id, key, run_id): + """Called inside native SQL run admission, under the same tree lock as cancellation.""" + from deerflow.runtime.runs.manager import ConflictError + + worker = await session.scalar(select(Assignment).where(Assignment.thread_id == thread_id)) + if worker is None: + return # A pre-existing user-chosen thread ID may share the prefix. + task_id = worker.task_id + root_id = await session.scalar(select(CoordinationTaskRow.root_id).filter_by(id=task_id)) + await session.execute(select(CoordinationTaskRow.id).filter_by(id=root_id).with_for_update()) + task = await session.get(CoordinationTaskRow, task_id) + execution = await session.get(Execution, key[len(RUN_KEY_PREFIX) :]) if key and key.startswith(RUN_KEY_PREFIX) else None + if ( + worker.epoch != task.execution_epoch + or task.owner_id != owner_id + or task.status != TaskStatus.RUNNING + or execution is None + or execution.task_id != task_id + or execution.epoch != task.execution_epoch + or execution.status != "admitting" + ): + raise ConflictError("Managed worker admission no longer matches the task") + execution.run_id = run_id + execution.status = "running" + + +class CoordinationWorkerRepository(CoordinationTaskRepository): + async def task_for_thread(self, thread_id): + async with self._sessions() as session: + return await session.scalar(select(Assignment.task_id).filter_by(thread_id=thread_id)) + + @staticmethod + def _latest(task_id): + return select(Execution).filter_by(task_id=task_id).order_by(Execution.turn.desc()).limit(1) + + @staticmethod + async def _artifact_refs(session, task_id, *, execution_id=None, limit=None, offset=0): + query = ( + select(Artifact.id, Artifact.path, Artifact.sha256, Artifact.size, Execution.run_id, Execution.turn) + .join(Execution, Execution.id == Artifact.execution_id) + .where(Execution.task_id == task_id) + .order_by(Execution.turn, Artifact.path, Artifact.id) + ) + if execution_id is not None: + query = query.where(Execution.id == execution_id) + rows = await session.execute(query.limit(limit).offset(offset)) + return [ + { + "id": row.id, + "source_path": row.path, + "sha256": row.sha256, + "size": row.size, + "run_id": row.run_id, + "turn": row.turn, + "url": f"/api/coordination/tasks/{task_id}/artifacts/{row.id}", + } + for row in rows + ] + + async def view(self, owner, task_id): + async with self._sessions() as session: + task = await self._lookup(session, owner, task_id) + assignment = await session.scalar(select(Assignment).filter_by(task_id=task_id).order_by(Assignment.epoch.desc()).limit(1)) + execution = await session.scalar(self._latest(task_id)) + result = None + if execution is not None: + result = execution.to_dict(exclude={"frozen_input", "resume_key"}) + result["artifacts"] = await self._artifact_refs(session, task_id, execution_id=execution.id) + return { + "task": task.to_dict(), + "assignment": assignment.to_dict() if assignment else None, + "execution": result, + } + + async def admission(self, owner, task_id, execution_id): + async with self._sessions() as session: + task = await self._lookup(session, owner, task_id) + execution = await session.get(Execution, execution_id) + if execution is None or execution.task_id != task_id or execution.epoch != task.execution_epoch or task.status != TaskStatus.RUNNING: + raise TaskConflict("Task no longer accepts this execution") + worker = await session.get(Assignment, (task_id, execution.epoch)) + return worker.to_dict(), execution.frozen_input, task.project_id + + async def list_artifacts(self, owner, task_id, *, limit, offset): + async with self._sessions() as session: + await self._lookup(session, owner, task_id) + return await self._artifact_refs(session, task_id, limit=limit, offset=offset) + + async def read_artifact(self, owner, task_id, artifact_id): + async with self._sessions() as session: + query = select(Artifact).join(Execution, Execution.id == Artifact.execution_id).join(CoordinationTaskRow, CoordinationTaskRow.id == Execution.task_id) + artifact = await session.scalar(query.where(Artifact.id == artifact_id, Execution.task_id == task_id, CoordinationTaskRow.owner_id == owner)) + if artifact is None: + raise TaskNotFound(artifact_id) + return artifact.path, artifact.content + + async def prepare(self, owner, task_id, assistant_id=None): + """Freeze a bounded turn before admission. Duplicate drivers get the same turn.""" + async with self._write() as session: + task = await self._lock_tree(session, owner, task_id) + worker = await session.get(Assignment, (task_id, task.execution_epoch)) + if worker is not None and assistant_id is not None and worker.assistant_id != assistant_id: + raise TaskConflict("Task already has a different worker") + if task.status == TaskStatus.QUEUED: + await self._start_locked(session, task) + if task.status != TaskStatus.RUNNING: + return None + if worker is None: + if assistant_id is None: + raise TaskConflict("Task has no assigned worker") + worker = Assignment(task_id=task_id, epoch=task.execution_epoch, assistant_id=assistant_id, thread_id=THREAD_PREFIX + uuid4().hex) + session.add(worker) + await session.flush() + execution = await session.scalar(self._latest(task_id)) + if execution is None or (execution.status == "superseded" and execution.covered_seq < task.latest_input_seq): + previous = execution.covered_seq if execution else 0 + if execution is None: + messages = [{"role": "user", "content": task.objective}] + elif execution.run_id is None: + messages = list(execution.frozen_input.get("messages", [])) + else: + messages = [] + inputs = await session.scalars(select(CoordinationInputRow).filter_by(task_id=task_id).where(CoordinationInputRow.sequence > previous).order_by(CoordinationInputRow.sequence).limit(8)) + for item in inputs: + messages.append({"role": "user", "content": item.content}) + previous = item.sequence + execution = Execution( + id=uuid4().hex, + task_id=task_id, + turn=execution.turn + 1 if execution else 1, + epoch=task.execution_epoch, + covered_seq=previous, + frozen_input={"messages": messages}, + status="admitting", + ) + session.add(execution) + return worker.to_dict(), execution.to_dict(), task.project_id + + async def resume(self, owner, task_id, *, key, value): + async with self._write() as session: + task = await self._lock_tree(session, owner, task_id) + payload = {"command": {"resume": value}} + execution = await session.scalar(select(Execution).filter_by(task_id=task_id, resume_key=key)) + if execution is not None: + if execution.frozen_input != payload: + raise TaskConflict("Request ID was already used with a different response") + else: + if task.status != TaskStatus.RUNNING: + raise TaskConflict("Task is closed") + previous = await session.scalar(self._latest(task_id)) + if previous is None or previous.status != "waiting_for_input": + raise TaskConflict("Worker has no pending interrupt") + execution = Execution( + id=uuid4().hex, + task_id=task_id, + turn=previous.turn + 1, + epoch=task.execution_epoch, + covered_seq=previous.covered_seq, + frozen_input=payload, + status="admitting", + resume_key=key, + ) + session.add(execution) + worker = await session.get(Assignment, (task_id, execution.epoch)) + return worker.to_dict(), execution.to_dict(), task.project_id + + async def settle(self, owner, task_id, execution_id, run_id, status, artifacts): + async with self._write() as session: + task = await self._lock_tree(session, owner, task_id) + execution = await session.get(Execution, execution_id) + if execution is None or execution.task_id != task_id or execution.run_id != run_id or execution.status not in ("running", "admitting"): + return + if task.status != TaskStatus.RUNNING or task.execution_epoch != execution.epoch: + return + for path, digest, content in artifacts: + session.add(Artifact(id=uuid4().hex, execution_id=execution.id, path=path, sha256=digest, size=len(content), content=content)) + if status == "waiting_for_input": + execution.status = status + return + if execution.covered_seq < task.latest_input_seq: + execution.status = "superseded" + return + worker = await session.get(Assignment, (task_id, task.execution_epoch)) + result_ref = f"/api/threads/{worker.thread_id}/runs/{run_id}/messages" if run_id else None + outcome = TaskStatus.SUCCEEDED if status == "success" else TaskStatus.FAILED + await self._finish_locked(session, task, outcome, result_ref or f"/api/coordination/tasks/{task_id}") + execution.status = status + + @staticmethod + def _admitted_run(task_id, sequence): + return select(Execution.run_id).where(Execution.task_id == task_id, Execution.covered_seq >= sequence, Execution.run_id.is_not(None)).order_by(Execution.turn).limit(1) + + async def list_worker_inputs(self, owner, task_id, *, after, limit): + async with self._sessions() as session: + task = await self._lookup(session, owner, task_id) + admitted = self._admitted_run(task_id, CoordinationInputRow.sequence).scalar_subquery() + query = select(CoordinationInputRow, admitted).where(CoordinationInputRow.task_id == task_id, CoordinationInputRow.sequence > after).order_by(CoordinationInputRow.sequence).limit(limit) + return [ + { + "sequence": row.sequence, + "content": row.content, + "created_at": row.created_at, + "source": INPUT_SOURCES.get(row.request_id.partition(":")[0], "unknown"), + "status": "admitted" if run_id else "pending" if task.status in (TaskStatus.QUEUED, TaskStatus.RUNNING) else "not_admitted", + "run_id": run_id, + } + for row, run_id in (await session.execute(query)).all() + ] + + async def input_receipts(self, owner, task_id): + async with self._sessions() as session: + task = await self._lookup(session, owner, task_id) + consuming = self._admitted_run(task_id, CoordinationInputRow.sequence).scalar_subquery() + query = select(CoordinationInputRow, consuming).filter(CoordinationInputRow.task_id == task_id, CoordinationInputRow.request_id.startswith("run-input:")).order_by(CoordinationInputRow.sequence.desc()).limit(32) + results = [(row.to_dict(), run) for row, run in (await session.execute(query)).all() if row.request_id.count(":") >= 2] + execution = await session.scalar(self._latest(task_id)) + active = execution.run_id if task.status == TaskStatus.RUNNING and execution and execution.status == "running" else None + return list(reversed(results)), active, task.status in (TaskStatus.CANCELLING, TaskStatus.CANCELLED) + + async def run_for_input(self, owner, task_id, sequence): + async with self._sessions() as session: + task = await self._lookup(session, owner, task_id) + run_id = await session.scalar(self._admitted_run(task_id, sequence)) + if run_id is None and task.status not in (TaskStatus.RUNNING, TaskStatus.QUEUED): + raise TaskConflict("Task stopped before this input could run") + return run_id + + async def cancellations(self, owner, task_id): + async with self._write() as session: + task = await self._lock_tree(session, owner, task_id) + ids = [row.id for row in await self._subtree(session, task) if row.status == TaskStatus.CANCELLING] + rows = await session.execute( + select(Assignment.thread_id, Execution.run_id) + .join(Execution, (Assignment.task_id == Execution.task_id) & (Assignment.epoch == Execution.epoch)) + .where(Assignment.task_id.in_(ids), Execution.run_id.is_not(None), Execution.status == "running") + ) + return list(rows) + + async def active_batches(self, owner, task_id): + from deerflow.persistence.subagent_batches.model import SubagentBatchRow + + async with self._write() as session: + task = await self._lock_tree(session, owner, task_id) + ids = [row.id for row in await self._subtree(session, task) if row.status == TaskStatus.CANCELLING] + threads = select(Assignment.thread_id).where(Assignment.task_id.in_(ids)) + query = select(SubagentBatchRow.thread_id, SubagentBatchRow.id).where(SubagentBatchRow.user_id == owner, SubagentBatchRow.thread_id.in_(threads), SubagentBatchRow.status.in_(("queued", "running", "paused"))) + return (await session.execute(query)).all() diff --git a/backend/packages/harness/deerflow/persistence/migrations/versions/0057_coordination_workers.py b/backend/packages/harness/deerflow/persistence/migrations/versions/0057_coordination_workers.py new file mode 100644 index 0000000000..a3f14ba540 --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/migrations/versions/0057_coordination_workers.py @@ -0,0 +1,56 @@ +"""Bind coordination tasks to worker threads and native runs.""" + +import sqlalchemy as sa +from alembic import op + +revision = "0057_coordination_workers" +down_revision = "0056_coordination_task_state" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table("coord_assignments"): + op.create_table( + "coord_assignments", + sa.Column("task_id", sa.String(64), sa.ForeignKey("coord_tasks.id"), primary_key=True), + sa.Column("epoch", sa.BigInteger(), primary_key=True), + sa.Column("thread_id", sa.String(64), nullable=False, unique=True), + sa.Column("assistant_id", sa.String(128), nullable=False), + ) + if not inspector.has_table("coord_executions"): + op.create_table( + "coord_executions", + sa.Column("id", sa.String(64), primary_key=True), + sa.Column("task_id", sa.String(64), nullable=False), + sa.Column("epoch", sa.BigInteger(), nullable=False), + sa.Column("turn", sa.BigInteger(), nullable=False), + sa.Column("resume_key", sa.String(200)), + sa.Column("covered_seq", sa.BigInteger(), nullable=False), + sa.Column("frozen_input", sa.JSON(), nullable=False), + sa.Column("status", sa.String(32), nullable=False), + sa.Column("run_id", sa.String(64), unique=True), + sa.ForeignKeyConstraint(["task_id", "epoch"], ["coord_assignments.task_id", "coord_assignments.epoch"]), + sa.UniqueConstraint("task_id", "turn", name="uq_coord_execution_turn"), + sa.UniqueConstraint("task_id", "resume_key", name="uq_coord_execution_resume"), + ) + + if not inspector.has_table("coord_artifacts"): + op.create_table( + "coord_artifacts", + sa.Column("id", sa.String(64), primary_key=True), + sa.Column("execution_id", sa.String(64), sa.ForeignKey("coord_executions.id"), nullable=False), + sa.Column("path", sa.Text(), nullable=False), + sa.Column("sha256", sa.String(64), nullable=False), + sa.Column("size", sa.BigInteger(), nullable=False), + sa.Column("content", sa.LargeBinary(), nullable=False), + ) + if "ix_coord_artifacts_execution_id" not in {idx["name"] for idx in inspector.get_indexes("coord_artifacts")}: + op.create_index("ix_coord_artifacts_execution_id", "coord_artifacts", ["execution_id"]) + + +def downgrade() -> None: + for table in ("coord_artifacts", "coord_executions", "coord_assignments"): + if sa.inspect(op.get_bind()).has_table(table): + op.drop_table(table) diff --git a/backend/packages/harness/deerflow/persistence/models/__init__.py b/backend/packages/harness/deerflow/persistence/models/__init__.py index b8df77f963..2aa676edc0 100644 --- a/backend/packages/harness/deerflow/persistence/models/__init__.py +++ b/backend/packages/harness/deerflow/persistence/models/__init__.py @@ -24,7 +24,7 @@ from deerflow.persistence.channel_connections.model import ( from deerflow.persistence.channel_leases.model import ChannelLeaseRow from deerflow.persistence.channel_runtime_configs.model import ChannelRuntimeConfigRow from deerflow.persistence.channel_thread_bindings.model import ChannelThreadBindingRow -from deerflow.persistence.coordination.model import CoordinationInputRow, CoordinationTaskRow +from deerflow.persistence.coordination.model import CoordinationArtifactRow, CoordinationAssignmentRow, CoordinationExecutionRow, CoordinationInputRow, CoordinationTaskRow from deerflow.persistence.extension_leases.model import ExtensionExecutionLeaseRow from deerflow.persistence.feedback.model import FeedbackRow from deerflow.persistence.login_throttle.model import LoginThrottleRow @@ -62,6 +62,9 @@ __all__ = [ "ChannelOAuthStateRow", "ChannelRuntimeConfigRow", "ChannelThreadBindingRow", + "CoordinationArtifactRow", + "CoordinationAssignmentRow", + "CoordinationExecutionRow", "CoordinationInputRow", "CoordinationTaskRow", "FeedbackRow", diff --git a/backend/packages/harness/deerflow/persistence/run/sql.py b/backend/packages/harness/deerflow/persistence/run/sql.py index f4fde85b67..6dafee2dbe 100644 --- a/backend/packages/harness/deerflow/persistence/run/sql.py +++ b/backend/packages/harness/deerflow/persistence/run/sql.py @@ -1200,6 +1200,11 @@ class RunRepository(RunStore): if existing is not None: raise RunIdempotencyConflict(self._row_to_dict(existing)) + from deerflow.persistence.coordination.workers import THREAD_PREFIX, bind_admitted_worker + + if thread_id.startswith(THREAD_PREFIX) and values["operation_kind"] == "run": + await bind_admitted_worker(session, thread_id, resolved_user_id, idempotency_key, run_id) + if multitask_strategy in ("interrupt", "rollback"): stmt = ( select(RunRow) diff --git a/backend/tests/test_boxd_sandbox_provider.py b/backend/tests/test_boxd_sandbox_provider.py index 25957fc3b1..0fe14a53f2 100644 --- a/backend/tests/test_boxd_sandbox_provider.py +++ b/backend/tests/test_boxd_sandbox_provider.py @@ -773,14 +773,13 @@ def test_timeout_runs_inside_the_machine(installed) -> None: assert call["timeout"] > 3 -@posix_only def test_timed_out_command_keeps_partial_output_and_exit_124(installed) -> None: _, fake = installed - fake.local_shell = True + # Simulate a remote timeout after the command has already produced output. + fake.exec_override = lambda *_: _Result(124, "before") output = _box(fake).execute_command("echo before; sleep 30; echo after", timeout=1) - assert output.startswith("before") - assert "after" not in output - assert output.endswith("Exit Code: 124") + assert fake.execs[-1]["argv"] == ["timeout", "-k", "5", "1", "bash", "-lc", "echo before; sleep 30; echo after"] + assert output == "before\nExit Code: 124" @posix_only diff --git a/backend/tests/test_coordination_task_repository.py b/backend/tests/test_coordination_task_repository.py index afb29daf36..fe43b39c61 100644 --- a/backend/tests/test_coordination_task_repository.py +++ b/backend/tests/test_coordination_task_repository.py @@ -175,13 +175,13 @@ async def test_coordination_migration_matches_model_and_downgrades(database): engine, _, schema = database config = bootstrap._get_alembic_config(engine, postgres_schema=schema) await asyncio.to_thread(command.upgrade, config, "0055_thread_shares") - await asyncio.to_thread(command.upgrade, config, "0056_coordination_task_state") + await asyncio.to_thread(command.upgrade, config, "0057_coordination_workers") async with engine.connect() as conn: def diff(sync): context = MigrationContext.configure( sync, - opts={"compare_type": True, "include_object": lambda _, name, kind, *_args: kind != "table" or name in {"coord_tasks", "coord_inputs"}}, + opts={"compare_type": True, "include_object": lambda _, name, kind, *_args: kind != "table" or name in {"coord_tasks", "coord_inputs", "coord_assignments", "coord_executions", "coord_artifacts"}}, ) return compare_metadata(context, Base.metadata) @@ -193,6 +193,56 @@ async def test_coordination_migration_matches_model_and_downgrades(database): async with engine.connect() as conn: tables = await conn.run_sync(lambda sync: set(inspect(sync).get_table_names())) assert "threads_meta" in tables - assert not {"coord_tasks", "coord_inputs"} & tables - await asyncio.to_thread(command.upgrade, config, "0056_coordination_task_state") + assert not {"coord_tasks", "coord_inputs", "coord_assignments", "coord_executions", "coord_artifacts"} & tables + await asyncio.to_thread(command.upgrade, config, "0057_coordination_workers") assert (await repo.create("alice", request_id="root", objective="After re-upgrade"))["status"] == TaskStatus.QUEUED + + +@pytest.mark.asyncio +async def test_worker_admission_is_atomic_with_cancellation_and_replay(database): + from deerflow.persistence.coordination.workers import RUN_KEY_PREFIX, CoordinationWorkerRepository + from deerflow.persistence.run import RunRepository + from deerflow.runtime.runs.manager import ConflictError + + engine, dialect, schema = database + await bootstrap.bootstrap_schema(engine, backend=dialect, postgres_schema=schema) + sessions = async_sessionmaker(engine) + workers = CoordinationWorkerRepository(sessions) + runs = RunRepository(sessions) + task = await workers.create("alice", request_id="dispatch", objective="Review") + await workers.append_input("alice", task["id"], request_id="input:follow-up", content="Check the result") + first, retry = await asyncio.gather(*(workers.prepare("alice", task["id"], "lead_agent") for _ in range(2))) + assert first == retry + worker, execution, project = first + assert project is None + kwargs = dict(thread_id=worker["thread_id"], owner_worker_id="gateway", user_id="alice", lease_expires_at=None) + key = RUN_KEY_PREFIX + execution["id"] + with pytest.raises(ConflictError): + await runs.create_thread_operation_atomic("unmanaged", **kwargs, idempotency_key="client-key") + with pytest.raises(TaskNotFound): + await workers.view("bob", task["id"]) + with pytest.raises(TaskNotFound): + await workers.list_artifacts("bob", task["id"], limit=1, offset=0) + with pytest.raises(TaskNotFound): + await workers.list_worker_inputs("bob", task["id"], after=0, limit=1) + (pending,) = await workers.list_worker_inputs("alice", task["id"], after=0, limit=1) + assert (pending["source"], pending["status"], pending["run_id"]) == ("task_api", "pending", None) + assert await workers.list_artifacts("alice", task["id"], limit=1, offset=0) == [] + current = await workers.get("alice", task["id"]) + admitted, cancelled = await asyncio.gather( + runs.create_thread_operation_atomic("native", **kwargs, idempotency_key=key), + workers.request_cancel("alice", task["id"], expected_revision=current["revision"]), + return_exceptions=True, + ) + assert cancelled["status"] == TaskStatus.CANCELLING + stopped = await workers.cancellations("alice", task["id"]) + (receipt,) = await workers.list_worker_inputs("alice", task["id"], after=0, limit=1) + if isinstance(admitted, ConflictError): + assert stopped == [] and await runs.get("native", user_id="alice") is None + assert (receipt["status"], receipt["run_id"]) == ("not_admitted", None) + else: + assert not isinstance(admitted, BaseException) and stopped == [(worker["thread_id"], "native")] + assert (await workers.view("alice", task["id"]))["execution"]["run_id"] == "native" + assert (receipt["status"], receipt["run_id"]) == ("admitted", "native") + with pytest.raises(ConflictError): + await runs.create_thread_operation_atomic("late", **kwargs, idempotency_key=RUN_KEY_PREFIX + uuid4().hex) diff --git a/backend/tests/test_extension_agent_runs.py b/backend/tests/test_extension_agent_runs.py index 3975c5443c..69354f0b44 100644 --- a/backend/tests/test_extension_agent_runs.py +++ b/backend/tests/test_extension_agent_runs.py @@ -161,7 +161,9 @@ async def runtime(host, monkeypatch, tmp_path): prompt = state["messages"][-1].content if prompt == "wait": entered.set() - await asyncio.Event().wait() + await getattr(request.app.state, "release_worker", asyncio.Event()).wait() + if present := getattr(request.app.state, "present_worker_report", None): + await present(prompt, runtime) if prompt == "approve": prompt = interrupt({"question": "Continue?"}) return {"messages": [AIMessage(content=f"answer:{prompt}")]} @@ -776,3 +778,208 @@ async def test_restored_delegated_handle_is_plugin_scoped_with_extension_origin( assert runs._origin == {"kind": "extension", "namespace": "workflows"} with pytest.raises(AgentRunError, match="already bound"): runs.for_plugin("other") + + +@pytest_asyncio.fixture +async def worker_gateway(runtime, tmp_path): + import asyncio + + from httpx import ASGITransport, AsyncClient + from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + + from app.gateway.coordination_workers import CoordinationWorkers + from app.gateway.routers import coordination, thread_runs + from deerflow.persistence.bootstrap import bootstrap_schema + from deerflow.persistence.coordination.workers import CoordinationWorkerRepository + from deerflow.persistence.run import RunRepository + from deerflow.runtime import RunManager + from deerflow.runtime.user_context import reset_current_user, set_current_user + + runs, state, request, entered, _ = runtime + engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'workers.db'}") + await bootstrap_schema(engine, backend="sqlite") + factory = async_sessionmaker(engine) + state.run_store = RunRepository(factory) + await state.run_manager.shutdown() + state.run_manager = RunManager(store=state.run_store) + workers = state.coordination_workers = CoordinationWorkers(CoordinationWorkerRepository(factory)) + state.release_worker = asyncio.Event() + request.app.include_router(coordination.router) + request.app.include_router(thread_runs.router) + + async def authenticated(scope, receive, send): + token = set_current_user(request.state.user) + try: + await request.app({**scope, "state": dict(request.scope["state"])}, receive, send) + finally: + reset_current_user(token) + + try: + async with AsyncClient(transport=ASGITransport(app=authenticated), base_url="http://gateway") as client: + yield client, runs, state, entered, str(request.state.user.id) + finally: + await workers.close() + await state.run_manager.shutdown() + await engine.dispose() + + +async def _wait_for_worker(client, task_id, section, field, value): + import asyncio + + async with asyncio.timeout(12): + while True: + response = await client.get(f"/api/coordination/tasks/{task_id}") + response.raise_for_status() + view = response.json() + if view[section][field] == value: + return view + await asyncio.sleep(0.05) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("followup_report", [None, b"revised report"], ids=["text-followup", "same-name-file"]) +async def test_managed_worker_routes_followups_and_retains_the_producing_runs_file(worker_gateway, followup_report): + import asyncio + from uuid import uuid4 + + from langchain_core.messages import AIMessage, ToolMessage + from langgraph.types import Command + + from deerflow.config.paths import get_paths + from deerflow.persistence.coordination.sql import TaskNotFound + + client, runs, state, entered, owner = worker_gateway + + async def present_report(prompt, runtime): + assert runtime.context["subagent_enabled"] is True + report = b"original report" if prompt == "wait" else followup_report + if report is not None: + outputs = get_paths().sandbox_outputs_dir(runtime.context["thread_id"], user_id=owner) + await asyncio.to_thread(outputs.mkdir, parents=True, exist_ok=True) + await asyncio.to_thread((outputs / "report.txt").write_bytes, report) + journal = runtime.context["__run_journal"] + journal._remember_current_run_tool_calls(AIMessage(content="", tool_calls=[{"id": "present", "name": "present_files", "args": {}}]), caller="lead_agent") + journal.on_tool_end(Command(update={"artifacts": ["/mnt/user-data/outputs/report.txt"], "messages": [ToolMessage("Presented", tool_call_id="present")]}), run_id=uuid4()) + + state.present_worker_report = present_report + url = "/api/coordination/tasks" + body = {"request_id": "dispatch", "objective": "wait"} + first = (await client.post(url, json=body)).raise_for_status().json() + task_id, thread, run = first["task"]["id"], first["assignment"]["thread_id"], first["execution"]["run_id"] + assert first["task"]["project_id"] is None and (await client.post(url, json=body)).json()["execution"]["run_id"] == run + await asyncio.wait_for(entered.wait(), 10) + thread_url = f"/api/threads/{thread}" + inbox_url = f"{url}/{task_id}/inputs" + (await client.post(inbox_url, json={"request_id": "task-input", "content": "task follow-up"})).raise_for_status() + followup = {"request_id": "fix", "mode": "queue", "text": "new requirement"} + receipt = (await client.post(f"{thread_url}/runs/{run}/inputs", json=followup)).raise_for_status().json() + assert receipt["status"] == "pending" + assert (await client.post(f"{thread_url}/runs/{run}/inputs", json=followup)).json() == receipt + direct = asyncio.create_task(client.post(f"{thread_url}/runs", json={"input": {"messages": [{"type": "human", "content": [{"type": "text", "text": "final instruction"}]}]}})) + try: + await _wait_for_worker(client, task_id, "task", "latest_input_seq", 3) + pending = (await client.get(inbox_url)).raise_for_status().json() + assert [(item["sequence"], item["source"], item["content"], item["status"]) for item in pending] == [ + (1, "task_api", "task follow-up", "pending"), + (2, "run_input", "new requirement", "pending"), + (3, "worker_chat", "final instruction", "pending"), + ] + assert (await client.get(inbox_url, params={"after": 1, "limit": 1})).raise_for_status().json() == pending[1:2] + assert (await client.get(f"{url}/missing-task/inputs")).status_code == 404 + finally: + state.release_worker.set() + (await direct).raise_for_status() + result = await _wait_for_worker(client, task_id, "task", "status", "succeeded") + final_run = result["execution"]["run_id"] + assert final_run != run and result["task"]["result_ref"] == f"{thread_url}/runs/{final_run}/messages" + assert (await direct).raise_for_status().json()["run_id"] == final_run + assert (await runs.get_state(thread_id=thread))["values"]["messages"][-1]["content"] == "answer:final instruction" + assert (await client.get(f"{thread_url}/inputs")).json()["items"][0]["run_id"] == final_run + admitted = (await client.get(inbox_url)).raise_for_status().json() + assert [(item["run_id"], item["status"]) for item in admitted] == [(final_run, "admitted")] * 3 + artifact_list = f"{url}/{task_id}/artifacts" + artifacts = (await client.get(artifact_list)).raise_for_status().json() + expected_versions = [(run, 1, b"original report")] + if followup_report: + expected_versions.append((final_run, 2, followup_report)) + assert [(item["run_id"], item["turn"]) for item in artifacts] == [(producer, turn) for producer, turn, _ in expected_versions] + assert {item["source_path"] for item in artifacts} == {"/mnt/user-data/outputs/report.txt"} + assert len({item["url"] for item in artifacts}) == len(expected_versions) + assert result["execution"]["artifacts"] == (artifacts[1:] if followup_report else []) + assert (await client.get(artifact_list, params={"limit": 1, "offset": 1})).raise_for_status().json() == artifacts[1:] + assert (await client.get(f"{url}/missing-task/artifacts")).status_code == 404 + await asyncio.to_thread((get_paths().sandbox_outputs_dir(thread, user_id=owner) / "report.txt").unlink) + for artifact, (_, _, content) in zip(artifacts, expected_versions, strict=True): + assert (await client.get(artifact["url"])).raise_for_status().content == content + with pytest.raises(TaskNotFound): + await state.coordination_workers.repo.read_artifact("another-user", task_id, artifacts[0]["id"]) + + +@pytest.mark.asyncio +async def test_managed_worker_requires_explicit_resume_and_fences_cancellation(worker_gateway): + import asyncio + + client, runs, _, entered, _ = worker_gateway + url = "/api/coordination/tasks" + paused = (await client.post(url, json={"request_id": "pause", "objective": "approve"})).raise_for_status().json() + task_id = paused["task"]["id"] + await _wait_for_worker(client, task_id, "execution", "status", "waiting_for_input") + target = f"/api/threads/{paused['assignment']['thread_id']}/runs" + reply = {"json": {"command": {"resume": "confirmed"}}, "headers": {"Idempotency-Key": "reply"}} + answered = (await client.post(target, **reply)).raise_for_status().json() + result = await _wait_for_worker(client, task_id, "task", "status", "succeeded") + assert result["task"]["result_ref"].endswith(f"/{answered['run_id']}/messages") + assert (await client.post(target, **reply)).raise_for_status().json()["run_id"] == answered["run_id"] + + other = (await client.post(url, json={"request_id": "cancel", "objective": "wait"})).raise_for_status().json() + await asyncio.wait_for(entered.wait(), 10) + task_id, thread = other["task"]["id"], other["assignment"]["thread_id"] + cancelled = (await client.post(f"{url}/{task_id}/cancel", json={"expected_revision": other["task"]["revision"]})).raise_for_status().json() + assert cancelled["status"] == "cancelled" + assert (await runs.get(thread_id=thread, run_id=other["execution"]["run_id"])).status == "interrupted" + assert (await client.post(f"/api/threads/{thread}/runs", json={"input": {"messages": [{"role": "user", "content": "too late"}]}})).status_code == 409 + + +@pytest.mark.asyncio +async def test_managed_worker_cancel_returns_cancelling_and_finishes_in_background(worker_gateway, monkeypatch): + import asyncio + + from app.gateway import coordination_workers + + client, runs, _, entered, _ = worker_gateway + started, release = asyncio.Event(), asyncio.Event() + calls = [] + request_stop = type(runs).cancel + + async def paused_stop(self, *, thread_id, run_id): + calls.append(run_id) + started.set() + await release.wait() + if len(calls) == 1: + raise AgentRunError(503, "Run is temporarily unavailable") + await request_stop(self, thread_id=thread_id, run_id=run_id) + + monkeypatch.setattr(type(runs), "cancel", paused_stop) + monkeypatch.setattr(coordination_workers, "CANCEL_RESPONSE_TIMEOUT", 0.05, raising=False) + url = "/api/coordination/tasks" + first = (await client.post(url, json={"objective": "wait", "request_id": "background-cancel"})).raise_for_status().json() + task_id, run_id = first["task"]["id"], first["execution"]["run_id"] + await asyncio.wait_for(entered.wait(), 10) + cancel_url, body = f"{url}/{task_id}/cancel", {"expected_revision": first["task"]["revision"]} + pending = asyncio.create_task(client.post(cancel_url, json=body)) + try: + await asyncio.wait_for(started.wait(), 5) + done, _ = await asyncio.wait({pending}, timeout=3) + assert pending in done, "Cancellation response waited for the blocked run" + response = pending.result() + assert response.status_code == 202 and response.json()["status"] == "cancelling" + assert (await client.post(cancel_url, json=body)).json()["status"] == "cancelling" + assert calls == [run_id] + assert (await client.post(f"{url}/{task_id}/inputs", json={"request_id": "late", "content": "too late"})).status_code == 409 + finally: + release.set() + await pending + final = await _wait_for_worker(client, task_id, "task", "status", "cancelled") + assert calls == [run_id, run_id] + assert final["execution"]["status"] == "cancelled" + assert (await runs.get(thread_id=first["assignment"]["thread_id"], run_id=run_id)).status == "interrupted" diff --git a/backend/tests/test_migration_0047_mcp_oauth_tokens.py b/backend/tests/test_migration_0047_mcp_oauth_tokens.py index ff99f32d81..5ac1634564 100644 --- a/backend/tests/test_migration_0047_mcp_oauth_tokens.py +++ b/backend/tests/test_migration_0047_mcp_oauth_tokens.py @@ -44,9 +44,10 @@ pytestmark = pytest.mark.asyncio async def test_0047_remains_in_the_single_migration_chain(): - assert bootstrap._get_head_revision() == "0056_coordination_task_state" + assert bootstrap._get_head_revision() == "0057_coordination_workers" script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR)) - assert script.get_heads() == ["0056_coordination_task_state"] + assert script.get_heads() == ["0057_coordination_workers"] + assert script.get_revision("0057_coordination_workers").down_revision == "0056_coordination_task_state" assert script.get_revision("0056_coordination_task_state").down_revision == "0055_thread_shares" assert script.get_revision("0055_thread_shares").down_revision == "0054_project_memory" assert script.get_revision("0054_project_memory").down_revision == "0053_tool_approvals"