From 778dbe456de7d331eaefb776bfceda5310c8bca6 Mon Sep 17 00:00:00 2001 From: Wenchao An Date: Thu, 19 Nov 2026 07:08:56 +0800 Subject: [PATCH] feat(coordination): persist task state and ordered inputs (#8346) * feat(coordination): persist task state and ordered inputs * fix(persistence): follow numbered migration revision naming --- backend/docs/task-coordination-state.md | 31 +++ .../persistence/coordination/__init__.py | 1 + .../persistence/coordination/model.py | 54 ++++ .../deerflow/persistence/coordination/sql.py | 262 ++++++++++++++++++ .../versions/0056_coordination_task_state.py | 55 ++++ .../deerflow/persistence/models/__init__.py | 3 + .../test_coordination_task_repository.py | 198 +++++++++++++ .../test_migration_0047_mcp_oauth_tokens.py | 5 +- 8 files changed, 607 insertions(+), 2 deletions(-) create mode 100644 backend/docs/task-coordination-state.md create mode 100644 backend/packages/harness/deerflow/persistence/coordination/__init__.py create mode 100644 backend/packages/harness/deerflow/persistence/coordination/model.py create mode 100644 backend/packages/harness/deerflow/persistence/coordination/sql.py create mode 100644 backend/packages/harness/deerflow/persistence/migrations/versions/0056_coordination_task_state.py create mode 100644 backend/tests/test_coordination_task_repository.py diff --git a/backend/docs/task-coordination-state.md b/backend/docs/task-coordination-state.md new file mode 100644 index 0000000000..1835e22388 --- /dev/null +++ b/backend/docs/task-coordination-state.md @@ -0,0 +1,31 @@ +# Task coordination state + +The coordination repository persists a unit of work and its ordered follow-up +inputs. Task IDs are independent of Thread and Run IDs. The Gateway can use +the repository to retain work identity when a later execution moves to a +different agent or thread. + +Each call takes an owner resolved by the caller. Task creation deduplicates +requests per owner. Input submission deduplicates requests per task and +assigns consecutive sequence numbers. Reusing a request ID with changed +content returns a conflict. Input request IDs should include the source +namespace if more than one source can submit to a task. + +A child inherits its parent's project. Dependencies are immutable and +refer to already-created tasks under the same owner, project, and parent. +A task can start after its dependencies succeed and its parent has started. +Success requires every +owned child to succeed and the accepted result to cover the current input +sequence. Failure can be recorded after all owned children settle. + +Mutations within one task tree run under a shared root lock on PostgreSQL and +a short write transaction on SQLite. Callers pass the revision they used to +decide a start, finish, or cancellation. Completion also checks the execution +epoch and input sequence. + +Requesting cancellation marks the task and its open descendants as cancelling +and advances their execution epochs in one transaction. Independent +prerequisites keep their state. A running caller can record cancellation +once it has confirmed that the subtree's executions have stopped. The +repository handles work state. Host admission, run control, and artifact +authorization belong to the execution integration. diff --git a/backend/packages/harness/deerflow/persistence/coordination/__init__.py b/backend/packages/harness/deerflow/persistence/coordination/__init__.py new file mode 100644 index 0000000000..beb246e2d4 --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/coordination/__init__.py @@ -0,0 +1 @@ +"""Durable work records for task coordination.""" diff --git a/backend/packages/harness/deerflow/persistence/coordination/model.py b/backend/packages/harness/deerflow/persistence/coordination/model.py new file mode 100644 index 0000000000..3b508f2f2e --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/coordination/model.py @@ -0,0 +1,54 @@ +"""Task state and ordered inputs, independent of agent runs.""" + +from datetime import datetime +from enum import StrEnum + +from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Index, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from deerflow.persistence.base import Base + + +class TaskStatus(StrEnum): + QUEUED = "queued" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLING = "cancelling" + CANCELLED = "cancelled" + + +class CoordinationTaskRow(Base): + __tablename__ = "coord_tasks" + __table_args__ = ( + UniqueConstraint("owner_id", "request_id", name="uq_coord_task_request"), + Index("ix_coord_tasks_owner_root", "owner_id", "root_id"), + Index("ix_coord_tasks_owner_parent", "owner_id", "parent_id"), + ) + + id: Mapped[str] = mapped_column(String(64), primary_key=True) + owner_id: Mapped[str] = mapped_column(String(64)) + request_id: Mapped[str] = mapped_column(String(200)) + project_id: Mapped[str | None] = mapped_column(String(64)) + root_id: Mapped[str] = mapped_column(String(64)) + parent_id: Mapped[str | None] = mapped_column(ForeignKey("coord_tasks.id")) + depends_on: Mapped[list[str]] = mapped_column(JSON) + objective: Mapped[str] = mapped_column(Text) + status: Mapped[str] = mapped_column(String(16)) + revision: Mapped[int] = mapped_column(BigInteger) + execution_epoch: Mapped[int] = mapped_column(BigInteger) + latest_input_seq: Mapped[int] = mapped_column(BigInteger) + result_ref: Mapped[str | None] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class CoordinationInputRow(Base): + __tablename__ = "coord_inputs" + __table_args__ = (UniqueConstraint("task_id", "request_id", name="uq_coord_input_request"),) + + task_id: Mapped[str] = mapped_column(ForeignKey("coord_tasks.id"), primary_key=True) + sequence: Mapped[int] = mapped_column(BigInteger, primary_key=True) + request_id: Mapped[str] = mapped_column(String(200)) + content: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) diff --git a/backend/packages/harness/deerflow/persistence/coordination/sql.py b/backend/packages/harness/deerflow/persistence/coordination/sql.py new file mode 100644 index 0000000000..027e70d7de --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/coordination/sql.py @@ -0,0 +1,262 @@ +"""Transactional task state. Callers verify identity and execution evidence.""" + +from __future__ import annotations + +from collections import defaultdict, deque +from collections.abc import AsyncIterator, Sequence +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from uuid import uuid4 + +from sqlalchemy import select, text +from sqlalchemy.dialects.postgresql import insert as postgres_insert +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from deerflow.persistence.coordination.model import CoordinationInputRow, CoordinationTaskRow, TaskStatus +from deerflow.utils.time import coerce_iso + +_OPEN = {TaskStatus.QUEUED, TaskStatus.RUNNING} + + +class TaskNotFound(LookupError): + """The task is absent or belongs to a different owner.""" + + +class TaskConflict(ValueError): + """The requested update conflicts with the current task state.""" + + +def _request_id(value: str) -> None: + if not value.strip() or len(value) > 200: + raise ValueError("Request ID must contain 1 to 200 characters") + + +def _touch(row: CoordinationTaskRow) -> None: + row.revision += 1 + row.updated_at = datetime.now(UTC) + + +def _check_revision(row: CoordinationTaskRow, expected_revision: int) -> None: + if row.revision != expected_revision: + raise TaskConflict("Task revision changed") + + +def _snapshot(row: CoordinationTaskRow | CoordinationInputRow) -> dict: + result = row.to_dict() + result["created_at"] = coerce_iso(result["created_at"]) + if isinstance(row, CoordinationTaskRow): + result["updated_at"] = coerce_iso(result["updated_at"]) + return result + + +class CoordinationTaskRepository: + def __init__(self, session_factory: async_sessionmaker[AsyncSession]): + self._sessions = session_factory + + @asynccontextmanager + async def _write(self) -> AsyncIterator[AsyncSession]: + async with self._sessions.begin() as session: + if session.get_bind().dialect.name == "sqlite": + await session.execute(text("BEGIN IMMEDIATE")) + yield session + + async def _lookup(self, session: AsyncSession, owner_id: str, task_id: str) -> CoordinationTaskRow: + row = await session.scalar(select(CoordinationTaskRow).filter_by(id=task_id, owner_id=owner_id)) + if row is None: + raise TaskNotFound(task_id) + return row + + async def _lock_tree(self, session: AsyncSession, owner_id: str, task_id: str) -> CoordinationTaskRow: + # All writes to an existing tree lock the root first. Fetch the target + # after taking the lock so a concurrent cancellation cannot leave a + # stale status in the ORM identity map. + root_id = await session.scalar(select(CoordinationTaskRow.root_id).filter_by(id=task_id, owner_id=owner_id)) + if root_id is None: + raise TaskNotFound(task_id) + await session.execute(select(CoordinationTaskRow.id).filter_by(id=root_id, owner_id=owner_id).with_for_update()) + return await self._lookup(session, owner_id, task_id) + + async def get(self, owner_id: str, task_id: str) -> dict: + async with self._sessions() as session: + return _snapshot(await self._lookup(session, owner_id, task_id)) + + async def create( + self, + owner_id: str, + *, + request_id: str, + objective: str, + project_id: str | None = None, + parent_id: str | None = None, + depends_on: Sequence[str] = (), + ) -> dict: + """Create a task once. Dependencies refer to previously created siblings.""" + _request_id(request_id) + if not objective.strip(): + raise ValueError("Task objective is required") + dependencies = sorted(set(depends_on)) + async with self._write() as session: + parent = await self._lock_tree(session, owner_id, parent_id) if parent_id is not None else None + if parent is not None: + if project_id is not None and project_id != parent.project_id: + raise TaskConflict("A child task must use its parent's project") + project_id = parent.project_id + create_fields = dict(objective=objective, project_id=project_id, parent_id=parent_id, depends_on=dependencies) + existing = await session.scalar(select(CoordinationTaskRow).filter_by(owner_id=owner_id, request_id=request_id)) + if existing is None: + if parent is not None and parent.status not in _OPEN: + raise TaskConflict("Parent task is closed") + if dependencies: + query = select(CoordinationTaskRow.id).filter_by(owner_id=owner_id, project_id=project_id, parent_id=parent_id) + found = set((await session.scalars(query.where(CoordinationTaskRow.id.in_(dependencies)))).all()) + if found != set(dependencies): + raise TaskConflict("Dependencies must be existing tasks with the same owner, project and parent") + task_id = uuid4().hex + now = datetime.now(UTC) + insert = postgres_insert if session.get_bind().dialect.name == "postgresql" else sqlite_insert + await session.execute( + insert(CoordinationTaskRow) + .values( + **create_fields, + id=task_id, + owner_id=owner_id, + request_id=request_id, + root_id=parent.root_id if parent is not None else task_id, + status=TaskStatus.QUEUED, + revision=0, + execution_epoch=0, + latest_input_seq=0, + created_at=now, + updated_at=now, + ) + .on_conflict_do_nothing(index_elements=["owner_id", "request_id"]) + ) + existing = (await session.scalars(select(CoordinationTaskRow).filter_by(owner_id=owner_id, request_id=request_id))).one() + if existing.id == task_id and parent is not None: + _touch(parent) + if any(getattr(existing, key) != value for key, value in create_fields.items()): + raise TaskConflict("Request ID was already used with different task data") + return _snapshot(existing) + + async def append_input(self, owner_id: str, task_id: str, *, request_id: str, content: str) -> dict: + _request_id(request_id) + if not content.strip(): + raise ValueError("Task input is required") + async with self._write() as session: + task = await self._lock_tree(session, owner_id, task_id) + existing = await session.scalar(select(CoordinationInputRow).filter_by(task_id=task_id, request_id=request_id)) + if existing is not None: + if existing.content != content: + raise TaskConflict("Request ID was already used with different input") + return _snapshot(existing) + if task.status not in _OPEN: + raise TaskConflict("Task is closed to new input") + task.latest_input_seq += 1 + _touch(task) + row = CoordinationInputRow(task_id=task_id, sequence=task.latest_input_seq, request_id=request_id, content=content, created_at=task.updated_at) + session.add(row) + return _snapshot(row) + + async def list_inputs(self, owner_id: str, task_id: str, *, after: int = 0, limit: int = 100) -> list[dict]: + if after < 0 or not 1 <= limit <= 100: + raise ValueError("Invalid input page") + async with self._sessions() as session: + query = ( + select(CoordinationInputRow) + .join(CoordinationTaskRow, CoordinationTaskRow.id == CoordinationInputRow.task_id) + .where(CoordinationTaskRow.owner_id == owner_id, CoordinationInputRow.task_id == task_id, CoordinationInputRow.sequence > after) + .order_by(CoordinationInputRow.sequence) + .limit(limit) + ) + return [_snapshot(row) for row in (await session.scalars(query)).all()] + + async def start(self, owner_id: str, task_id: str, *, expected_revision: int) -> dict: + """Claim the task for work and return the revision and input watermark.""" + async with self._write() as session: + task = await self._lock_tree(session, owner_id, task_id) + _check_revision(task, expected_revision) + if task.status != TaskStatus.QUEUED: + raise TaskConflict("Only a queued task can start") + if task.parent_id is not None: + parent = await self._lookup(session, owner_id, task.parent_id) + if parent.status != TaskStatus.RUNNING: + raise TaskConflict("Parent task has not started") + if task.depends_on: + query = select(CoordinationTaskRow.id).filter_by(owner_id=owner_id, status=TaskStatus.SUCCEEDED) + ready = set((await session.scalars(query.where(CoordinationTaskRow.id.in_(task.depends_on)))).all()) + if ready != set(task.depends_on): + raise TaskConflict("Task dependencies have not succeeded") + task.status = TaskStatus.RUNNING + _touch(task) + return _snapshot(task) + + async def finish( + self, + owner_id: str, + task_id: str, + *, + expected_revision: int, + execution_epoch: int, + covered_input_seq: int, + outcome: TaskStatus, + result_ref: str, + ) -> dict: + if outcome not in (TaskStatus.SUCCEEDED, TaskStatus.FAILED) or not result_ref.strip(): + raise ValueError("A success or failure outcome and a result reference are required") + async with self._write() as session: + task = await self._lock_tree(session, owner_id, task_id) + if task.status != TaskStatus.RUNNING or task.execution_epoch != execution_epoch: + raise TaskConflict("Task no longer accepts results from this execution") + _check_revision(task, expected_revision) + if task.latest_input_seq != covered_input_seq: + raise TaskConflict("Result does not cover the current task inputs") + children = (await session.scalars(select(CoordinationTaskRow.status).filter_by(parent_id=task_id, owner_id=owner_id))).all() + accepted = {TaskStatus.SUCCEEDED} if outcome == TaskStatus.SUCCEEDED else {TaskStatus.SUCCEEDED, TaskStatus.FAILED, TaskStatus.CANCELLED} + if any(status not in accepted for status in children): + raise TaskConflict("Child tasks have not settled for this outcome") + task.status = outcome + task.result_ref = result_ref + _touch(task) + return _snapshot(task) + + async def _subtree(self, session: AsyncSession, task: CoordinationTaskRow) -> list[CoordinationTaskRow]: + rows = (await session.scalars(select(CoordinationTaskRow).filter_by(owner_id=task.owner_id, root_id=task.root_id))).all() + children = defaultdict(list) + for row in rows: + children[row.parent_id].append(row) + result = [] + pending = deque([task]) + while pending: + row = pending.popleft() + result.append(row) + pending.extend(children[row.id]) + return result + + async def request_cancel(self, owner_id: str, task_id: str, *, expected_revision: int) -> dict: + """Stop accepting work in the ownership subtree before stopping its runs.""" + async with self._write() as session: + task = await self._lock_tree(session, owner_id, task_id) + if task.status in (TaskStatus.CANCELLING, TaskStatus.CANCELLED): + return _snapshot(task) + _check_revision(task, expected_revision) + if task.status not in _OPEN: + raise TaskConflict("Task has already finished") + for row in await self._subtree(session, task): + if row.status in _OPEN: + row.status = TaskStatus.CANCELLING + row.execution_epoch += 1 + _touch(row) + return _snapshot(task) + + async def confirm_cancel(self, owner_id: str, task_id: str, *, execution_epoch: int) -> dict: + """Record cancellation after the caller confirms all subtree runs stopped.""" + async with self._write() as session: + task = await self._lock_tree(session, owner_id, task_id) + if task.status not in (TaskStatus.CANCELLING, TaskStatus.CANCELLED) or task.execution_epoch != execution_epoch: + raise TaskConflict("Cancellation no longer matches the task") + for row in await self._subtree(session, task): + if row.status == TaskStatus.CANCELLING: + row.status = TaskStatus.CANCELLED + _touch(row) + return _snapshot(task) diff --git a/backend/packages/harness/deerflow/persistence/migrations/versions/0056_coordination_task_state.py b/backend/packages/harness/deerflow/persistence/migrations/versions/0056_coordination_task_state.py new file mode 100644 index 0000000000..f32bfc71e2 --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/migrations/versions/0056_coordination_task_state.py @@ -0,0 +1,55 @@ +"""Add independent coordination tasks and durable ordered input.""" + +import sqlalchemy as sa +from alembic import op + +revision = "0056_coordination_task_state" +down_revision = "0055_thread_shares" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + if not sa.inspect(op.get_bind()).has_table("coord_tasks"): + op.create_table( + "coord_tasks", + sa.Column("id", sa.String(64), primary_key=True), + sa.Column("owner_id", sa.String(64), nullable=False), + sa.Column("request_id", sa.String(200), nullable=False), + sa.Column("project_id", sa.String(64)), + sa.Column("root_id", sa.String(64), nullable=False), + sa.Column("parent_id", sa.String(64), sa.ForeignKey("coord_tasks.id")), + sa.Column("depends_on", sa.JSON(), nullable=False), + sa.Column("objective", sa.Text(), nullable=False), + sa.Column("status", sa.String(16), nullable=False), + sa.Column("revision", sa.BigInteger(), nullable=False), + sa.Column("execution_epoch", sa.BigInteger(), nullable=False), + sa.Column("latest_input_seq", sa.BigInteger(), nullable=False), + sa.Column("result_ref", sa.Text()), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.UniqueConstraint("owner_id", "request_id", name="uq_coord_task_request"), + ) + # A non-transactional SQLite upgrade can be interrupted between DDL steps. + indexes = {index["name"] for index in sa.inspect(op.get_bind()).get_indexes("coord_tasks")} + if "ix_coord_tasks_owner_root" not in indexes: + op.create_index("ix_coord_tasks_owner_root", "coord_tasks", ["owner_id", "root_id"]) + if "ix_coord_tasks_owner_parent" not in indexes: + op.create_index("ix_coord_tasks_owner_parent", "coord_tasks", ["owner_id", "parent_id"]) + if not sa.inspect(op.get_bind()).has_table("coord_inputs"): + op.create_table( + "coord_inputs", + sa.Column("task_id", sa.String(64), sa.ForeignKey("coord_tasks.id"), primary_key=True), + sa.Column("sequence", sa.BigInteger(), primary_key=True), + sa.Column("request_id", sa.String(200), nullable=False), + sa.Column("content", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.UniqueConstraint("task_id", "request_id", name="uq_coord_input_request"), + ) + + +def downgrade() -> None: + if sa.inspect(op.get_bind()).has_table("coord_inputs"): + op.drop_table("coord_inputs") + if sa.inspect(op.get_bind()).has_table("coord_tasks"): + op.drop_table("coord_tasks") diff --git a/backend/packages/harness/deerflow/persistence/models/__init__.py b/backend/packages/harness/deerflow/persistence/models/__init__.py index cd0cb8a5e8..b8df77f963 100644 --- a/backend/packages/harness/deerflow/persistence/models/__init__.py +++ b/backend/packages/harness/deerflow/persistence/models/__init__.py @@ -24,6 +24,7 @@ from deerflow.persistence.channel_connections.model import ( from deerflow.persistence.channel_leases.model import ChannelLeaseRow from deerflow.persistence.channel_runtime_configs.model import ChannelRuntimeConfigRow from deerflow.persistence.channel_thread_bindings.model import ChannelThreadBindingRow +from deerflow.persistence.coordination.model import CoordinationInputRow, CoordinationTaskRow from deerflow.persistence.extension_leases.model import ExtensionExecutionLeaseRow from deerflow.persistence.feedback.model import FeedbackRow from deerflow.persistence.login_throttle.model import LoginThrottleRow @@ -61,6 +62,8 @@ __all__ = [ "ChannelOAuthStateRow", "ChannelRuntimeConfigRow", "ChannelThreadBindingRow", + "CoordinationInputRow", + "CoordinationTaskRow", "FeedbackRow", "LoginThrottleRow", "McpOAuthTokenRow", diff --git a/backend/tests/test_coordination_task_repository.py b/backend/tests/test_coordination_task_repository.py new file mode 100644 index 0000000000..afb29daf36 --- /dev/null +++ b/backend/tests/test_coordination_task_repository.py @@ -0,0 +1,198 @@ +"""Exercise real SQL task transitions on both supported databases.""" + +import asyncio +import os +from uuid import uuid4 + +import pytest +import pytest_asyncio +from alembic import command +from alembic.autogenerate import compare_metadata +from alembic.migration import MigrationContext +from sqlalchemy import inspect, text +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from support.postgres import asyncpg_test_url + +from deerflow.persistence import bootstrap +from deerflow.persistence.base import Base +from deerflow.persistence.coordination.model import TaskStatus +from deerflow.persistence.coordination.sql import CoordinationTaskRepository, TaskConflict, TaskNotFound +from deerflow.persistence.postgres_schema import build_asyncpg_connect_args + + +@pytest_asyncio.fixture(params=["sqlite", "postgres"]) +async def database(tmp_path, request): + backend = request.param + schema = "" + if backend == "sqlite": + engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'coord.db'}") + else: + uri = os.environ.get("TEST_POSTGRES_URI") + if not uri: + pytest.skip("requires TEST_POSTGRES_URI") + schema = f"coord_test_{uuid4().hex}" + engine = create_async_engine(asyncpg_test_url(uri), connect_args=build_asyncpg_connect_args(schema)) + async with engine.begin() as conn: + await conn.execute(text(f'CREATE SCHEMA "{schema}"')) + try: + yield engine, backend, schema + finally: + if schema: + async with engine.begin() as conn: + await conn.execute(text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE')) + await engine.dispose() + + +@pytest_asyncio.fixture +async def repo(database): + engine, backend, schema = database + await bootstrap.bootstrap_schema(engine, backend=backend, postgres_schema=schema) + return CoordinationTaskRepository(async_sessionmaker(engine)) + + +async def _finish(repo, task, *, covered_input_seq=None): + return await repo.finish( + "alice", + task["id"], + expected_revision=task["revision"], + execution_epoch=task["execution_epoch"], + covered_input_seq=task["latest_input_seq"] if covered_input_seq is None else covered_input_seq, + outcome=TaskStatus.SUCCEEDED, + result_ref="result-1", + ) + + +@pytest.mark.asyncio +async def test_concurrent_replay_and_owner_isolation(repo): + async with asyncio.TaskGroup() as group: + submissions = [group.create_task(repo.create("alice", request_id="message", objective="Review")) for _ in range(4)] + (task_id,) = {job.result()["id"] for job in submissions} + bob = await repo.create("bob", request_id="message", objective="Review") + assert bob["id"] != task_id + with pytest.raises(TaskConflict, match="different task data"): + await repo.create("alice", request_id="message", objective="Other work") + with pytest.raises(TaskNotFound): + await repo.get("bob", task_id) + with pytest.raises(TaskNotFound): + await repo.append_input("bob", task_id, request_id="msg", content="wrong owner") + + async with asyncio.TaskGroup() as group: + repeats = [group.create_task(repo.append_input("alice", task_id, request_id="one", content="Check storage")) for _ in range(3)] + other = group.create_task(repo.append_input("alice", task_id, request_id="two", content="Check logs")) + (sequence,) = {job.result()["sequence"] for job in repeats} + assert {sequence, other.result()["sequence"]} == {1, 2} + assert [item["sequence"] for item in await repo.list_inputs("alice", task_id, after=1)] == [2] + assert await repo.list_inputs("bob", task_id) == [] + with pytest.raises(TaskConflict, match="different input"): + await repo.append_input("alice", task_id, request_id="one", content="mutated replay") + + +@pytest.mark.asyncio +async def test_dependencies_release_in_order_and_completion_covers_current_input(repo): + root = await repo.create("alice", request_id="root", project_id="project", objective="Release") + first = await repo.create("alice", request_id="a", parent_id=root["id"], objective="Research") + second = await repo.create("alice", request_id="b", parent_id=root["id"], depends_on=[first["id"]], objective="Review") + assert first["project_id"] == second["project_id"] == "project" + with pytest.raises(TaskConflict, match="Dependencies"): + await repo.create("alice", request_id="bad", depends_on=[first["id"]], objective="Foreign scope") + with pytest.raises(TaskConflict, match="Parent task has not started"): + await repo.start("alice", first["id"], expected_revision=0) + root = await repo.get("alice", root["id"]) + root = await repo.start("alice", root["id"], expected_revision=root["revision"]) + with pytest.raises(TaskConflict, match="dependencies"): + await repo.start("alice", second["id"], expected_revision=0) + with pytest.raises(TaskConflict, match="Child tasks"): + await _finish(repo, root) + started = await repo.start("alice", first["id"], expected_revision=0) + await repo.append_input("alice", first["id"], request_id="new", content="Check another file") + with pytest.raises(TaskConflict, match="revision changed"): + await _finish(repo, started) + latest = await repo.get("alice", first["id"]) + with pytest.raises(TaskConflict, match="current task inputs"): + await _finish(repo, latest, covered_input_seq=0) + assert (await _finish(repo, latest))["status"] == TaskStatus.SUCCEEDED + assert (await repo.start("alice", second["id"], expected_revision=0))["status"] == TaskStatus.RUNNING + + +@pytest.mark.asyncio +async def test_cancel_tree_fences_running_work_and_leaves_external_dependency(repo): + prerequisite = await repo.create("alice", request_id="pre", objective="Shared work") + prerequisite = await repo.start("alice", prerequisite["id"], expected_revision=0) + await _finish(repo, prerequisite) + root = await repo.create("alice", request_id="root", objective="Check release", depends_on=[prerequisite["id"]]) + child = await repo.create("alice", request_id="child", parent_id=root["id"], objective="Review") + nested = await repo.create("alice", request_id="nested", parent_id=child["id"], objective="Test") + root = await repo.get("alice", root["id"]) + root = await repo.start("alice", root["id"], expected_revision=root["revision"]) + child = await repo.get("alice", child["id"]) + await repo.start("alice", child["id"], expected_revision=child["revision"]) + nested = await repo.start("alice", nested["id"], expected_revision=0) + cancelling = await repo.request_cancel("alice", root["id"], expected_revision=root["revision"]) + assert cancelling["execution_epoch"] == 1 + assert (await repo.request_cancel("alice", root["id"], expected_revision=root["revision"])) == cancelling + assert (await repo.get("alice", nested["id"]))["status"] == TaskStatus.CANCELLING + with pytest.raises(TaskConflict, match="no longer accepts"): + await _finish(repo, nested) + with pytest.raises(TaskConflict, match="closed to new input"): + await repo.append_input("alice", nested["id"], request_id="late", content="Do more") + with pytest.raises(TaskConflict, match="Parent task is closed"): + await repo.create("alice", request_id="late-child", parent_id=child["id"], objective="New work") + with pytest.raises(TaskConflict, match="Cancellation no longer"): + await repo.confirm_cancel("alice", root["id"], execution_epoch=0) + await repo.confirm_cancel("alice", root["id"], execution_epoch=1) + for task in (root, child, nested): + assert (await repo.get("alice", task["id"]))["status"] == TaskStatus.CANCELLED + assert (await repo.get("alice", prerequisite["id"]))["status"] == TaskStatus.SUCCEEDED + + +@pytest.mark.asyncio +async def test_cancel_and_followup_in_separate_transactions(repo): + task = await repo.create("alice", request_id="root", objective="Check versions") + task_id = task["id"] + + async def append(): + try: + return await repo.append_input("alice", task_id, request_id="one", content="Also check dependencies") + except TaskConflict: + return None + + async def cancel(): + try: + return await repo.request_cancel("alice", task_id, expected_revision=0) + except TaskConflict: + return None + + appended, cancelled = await asyncio.gather(append(), cancel()) + assert (appended is None) != (cancelled is None) + current = await repo.get("alice", task_id) + assert current["revision"] == 1 + assert current["latest_input_seq"] == int(appended is not None) + assert current["execution_epoch"] == int(cancelled is not None) + + +@pytest.mark.asyncio +async def test_coordination_migration_matches_model_and_downgrades(database): + engine, _, schema = database + config = bootstrap._get_alembic_config(engine, postgres_schema=schema) + await asyncio.to_thread(command.upgrade, config, "0055_thread_shares") + await asyncio.to_thread(command.upgrade, config, "0056_coordination_task_state") + async with engine.connect() as conn: + + def diff(sync): + context = MigrationContext.configure( + sync, + opts={"compare_type": True, "include_object": lambda _, name, kind, *_args: kind != "table" or name in {"coord_tasks", "coord_inputs"}}, + ) + return compare_metadata(context, Base.metadata) + + assert await conn.run_sync(diff) == [] + repo = CoordinationTaskRepository(async_sessionmaker(engine)) + task = await repo.create("alice", request_id="root", objective="Test the upgraded schema") + await repo.append_input("alice", task["id"], request_id="update", content="Record an input") + await asyncio.to_thread(command.downgrade, config, "0055_thread_shares") + async with engine.connect() as conn: + tables = await conn.run_sync(lambda sync: set(inspect(sync).get_table_names())) + assert "threads_meta" in tables + assert not {"coord_tasks", "coord_inputs"} & tables + await asyncio.to_thread(command.upgrade, config, "0056_coordination_task_state") + assert (await repo.create("alice", request_id="root", objective="After re-upgrade"))["status"] == TaskStatus.QUEUED diff --git a/backend/tests/test_migration_0047_mcp_oauth_tokens.py b/backend/tests/test_migration_0047_mcp_oauth_tokens.py index 551239a414..ff99f32d81 100644 --- a/backend/tests/test_migration_0047_mcp_oauth_tokens.py +++ b/backend/tests/test_migration_0047_mcp_oauth_tokens.py @@ -44,9 +44,10 @@ pytestmark = pytest.mark.asyncio async def test_0047_remains_in_the_single_migration_chain(): - assert bootstrap._get_head_revision() == "0055_thread_shares" + assert bootstrap._get_head_revision() == "0056_coordination_task_state" script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR)) - assert script.get_heads() == ["0055_thread_shares"] + assert script.get_heads() == ["0056_coordination_task_state"] + assert script.get_revision("0056_coordination_task_state").down_revision == "0055_thread_shares" assert script.get_revision("0055_thread_shares").down_revision == "0054_project_memory" assert script.get_revision("0054_project_memory").down_revision == "0053_tool_approvals" assert script.get_revision("0053_tool_approvals").down_revision == "0052_batch_continuation"