mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
feat(coordination): persist task state and ordered inputs (#8346)
* feat(coordination): persist task state and ordered inputs * fix(persistence): follow numbered migration revision naming
This commit is contained in:
parent
e0cccd6b37
commit
778dbe456d
31
backend/docs/task-coordination-state.md
Normal file
31
backend/docs/task-coordination-state.md
Normal file
@ -0,0 +1,31 @@
|
||||
# Task coordination state
|
||||
|
||||
The coordination repository persists a unit of work and its ordered follow-up
|
||||
inputs. Task IDs are independent of Thread and Run IDs. The Gateway can use
|
||||
the repository to retain work identity when a later execution moves to a
|
||||
different agent or thread.
|
||||
|
||||
Each call takes an owner resolved by the caller. Task creation deduplicates
|
||||
requests per owner. Input submission deduplicates requests per task and
|
||||
assigns consecutive sequence numbers. Reusing a request ID with changed
|
||||
content returns a conflict. Input request IDs should include the source
|
||||
namespace if more than one source can submit to a task.
|
||||
|
||||
A child inherits its parent's project. Dependencies are immutable and
|
||||
refer to already-created tasks under the same owner, project, and parent.
|
||||
A task can start after its dependencies succeed and its parent has started.
|
||||
Success requires every
|
||||
owned child to succeed and the accepted result to cover the current input
|
||||
sequence. Failure can be recorded after all owned children settle.
|
||||
|
||||
Mutations within one task tree run under a shared root lock on PostgreSQL and
|
||||
a short write transaction on SQLite. Callers pass the revision they used to
|
||||
decide a start, finish, or cancellation. Completion also checks the execution
|
||||
epoch and input sequence.
|
||||
|
||||
Requesting cancellation marks the task and its open descendants as cancelling
|
||||
and advances their execution epochs in one transaction. Independent
|
||||
prerequisites keep their state. A running caller can record cancellation
|
||||
once it has confirmed that the subtree's executions have stopped. The
|
||||
repository handles work state. Host admission, run control, and artifact
|
||||
authorization belong to the execution integration.
|
||||
@ -0,0 +1 @@
|
||||
"""Durable work records for task coordination."""
|
||||
@ -0,0 +1,54 @@
|
||||
"""Task state and ordered inputs, independent of agent runs."""
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
|
||||
from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Index, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class TaskStatus(StrEnum):
|
||||
QUEUED = "queued"
|
||||
RUNNING = "running"
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
CANCELLING = "cancelling"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
class CoordinationTaskRow(Base):
|
||||
__tablename__ = "coord_tasks"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("owner_id", "request_id", name="uq_coord_task_request"),
|
||||
Index("ix_coord_tasks_owner_root", "owner_id", "root_id"),
|
||||
Index("ix_coord_tasks_owner_parent", "owner_id", "parent_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
owner_id: Mapped[str] = mapped_column(String(64))
|
||||
request_id: Mapped[str] = mapped_column(String(200))
|
||||
project_id: Mapped[str | None] = mapped_column(String(64))
|
||||
root_id: Mapped[str] = mapped_column(String(64))
|
||||
parent_id: Mapped[str | None] = mapped_column(ForeignKey("coord_tasks.id"))
|
||||
depends_on: Mapped[list[str]] = mapped_column(JSON)
|
||||
objective: Mapped[str] = mapped_column(Text)
|
||||
status: Mapped[str] = mapped_column(String(16))
|
||||
revision: Mapped[int] = mapped_column(BigInteger)
|
||||
execution_epoch: Mapped[int] = mapped_column(BigInteger)
|
||||
latest_input_seq: Mapped[int] = mapped_column(BigInteger)
|
||||
result_ref: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class CoordinationInputRow(Base):
|
||||
__tablename__ = "coord_inputs"
|
||||
__table_args__ = (UniqueConstraint("task_id", "request_id", name="uq_coord_input_request"),)
|
||||
|
||||
task_id: Mapped[str] = mapped_column(ForeignKey("coord_tasks.id"), primary_key=True)
|
||||
sequence: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
request_id: Mapped[str] = mapped_column(String(200))
|
||||
content: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
@ -0,0 +1,262 @@
|
||||
"""Transactional task state. Callers verify identity and execution evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as postgres_insert
|
||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.coordination.model import CoordinationInputRow, CoordinationTaskRow, TaskStatus
|
||||
from deerflow.utils.time import coerce_iso
|
||||
|
||||
_OPEN = {TaskStatus.QUEUED, TaskStatus.RUNNING}
|
||||
|
||||
|
||||
class TaskNotFound(LookupError):
|
||||
"""The task is absent or belongs to a different owner."""
|
||||
|
||||
|
||||
class TaskConflict(ValueError):
|
||||
"""The requested update conflicts with the current task state."""
|
||||
|
||||
|
||||
def _request_id(value: str) -> None:
|
||||
if not value.strip() or len(value) > 200:
|
||||
raise ValueError("Request ID must contain 1 to 200 characters")
|
||||
|
||||
|
||||
def _touch(row: CoordinationTaskRow) -> None:
|
||||
row.revision += 1
|
||||
row.updated_at = datetime.now(UTC)
|
||||
|
||||
|
||||
def _check_revision(row: CoordinationTaskRow, expected_revision: int) -> None:
|
||||
if row.revision != expected_revision:
|
||||
raise TaskConflict("Task revision changed")
|
||||
|
||||
|
||||
def _snapshot(row: CoordinationTaskRow | CoordinationInputRow) -> dict:
|
||||
result = row.to_dict()
|
||||
result["created_at"] = coerce_iso(result["created_at"])
|
||||
if isinstance(row, CoordinationTaskRow):
|
||||
result["updated_at"] = coerce_iso(result["updated_at"])
|
||||
return result
|
||||
|
||||
|
||||
class CoordinationTaskRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]):
|
||||
self._sessions = session_factory
|
||||
|
||||
@asynccontextmanager
|
||||
async def _write(self) -> AsyncIterator[AsyncSession]:
|
||||
async with self._sessions.begin() as session:
|
||||
if session.get_bind().dialect.name == "sqlite":
|
||||
await session.execute(text("BEGIN IMMEDIATE"))
|
||||
yield session
|
||||
|
||||
async def _lookup(self, session: AsyncSession, owner_id: str, task_id: str) -> CoordinationTaskRow:
|
||||
row = await session.scalar(select(CoordinationTaskRow).filter_by(id=task_id, owner_id=owner_id))
|
||||
if row is None:
|
||||
raise TaskNotFound(task_id)
|
||||
return row
|
||||
|
||||
async def _lock_tree(self, session: AsyncSession, owner_id: str, task_id: str) -> CoordinationTaskRow:
|
||||
# All writes to an existing tree lock the root first. Fetch the target
|
||||
# after taking the lock so a concurrent cancellation cannot leave a
|
||||
# stale status in the ORM identity map.
|
||||
root_id = await session.scalar(select(CoordinationTaskRow.root_id).filter_by(id=task_id, owner_id=owner_id))
|
||||
if root_id is None:
|
||||
raise TaskNotFound(task_id)
|
||||
await session.execute(select(CoordinationTaskRow.id).filter_by(id=root_id, owner_id=owner_id).with_for_update())
|
||||
return await self._lookup(session, owner_id, task_id)
|
||||
|
||||
async def get(self, owner_id: str, task_id: str) -> dict:
|
||||
async with self._sessions() as session:
|
||||
return _snapshot(await self._lookup(session, owner_id, task_id))
|
||||
|
||||
async def create(
|
||||
self,
|
||||
owner_id: str,
|
||||
*,
|
||||
request_id: str,
|
||||
objective: str,
|
||||
project_id: str | None = None,
|
||||
parent_id: str | None = None,
|
||||
depends_on: Sequence[str] = (),
|
||||
) -> dict:
|
||||
"""Create a task once. Dependencies refer to previously created siblings."""
|
||||
_request_id(request_id)
|
||||
if not objective.strip():
|
||||
raise ValueError("Task objective is required")
|
||||
dependencies = sorted(set(depends_on))
|
||||
async with self._write() as session:
|
||||
parent = await self._lock_tree(session, owner_id, parent_id) if parent_id is not None else None
|
||||
if parent is not None:
|
||||
if project_id is not None and project_id != parent.project_id:
|
||||
raise TaskConflict("A child task must use its parent's project")
|
||||
project_id = parent.project_id
|
||||
create_fields = dict(objective=objective, project_id=project_id, parent_id=parent_id, depends_on=dependencies)
|
||||
existing = await session.scalar(select(CoordinationTaskRow).filter_by(owner_id=owner_id, request_id=request_id))
|
||||
if existing is None:
|
||||
if parent is not None and parent.status not in _OPEN:
|
||||
raise TaskConflict("Parent task is closed")
|
||||
if dependencies:
|
||||
query = select(CoordinationTaskRow.id).filter_by(owner_id=owner_id, project_id=project_id, parent_id=parent_id)
|
||||
found = set((await session.scalars(query.where(CoordinationTaskRow.id.in_(dependencies)))).all())
|
||||
if found != set(dependencies):
|
||||
raise TaskConflict("Dependencies must be existing tasks with the same owner, project and parent")
|
||||
task_id = uuid4().hex
|
||||
now = datetime.now(UTC)
|
||||
insert = postgres_insert if session.get_bind().dialect.name == "postgresql" else sqlite_insert
|
||||
await session.execute(
|
||||
insert(CoordinationTaskRow)
|
||||
.values(
|
||||
**create_fields,
|
||||
id=task_id,
|
||||
owner_id=owner_id,
|
||||
request_id=request_id,
|
||||
root_id=parent.root_id if parent is not None else task_id,
|
||||
status=TaskStatus.QUEUED,
|
||||
revision=0,
|
||||
execution_epoch=0,
|
||||
latest_input_seq=0,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
.on_conflict_do_nothing(index_elements=["owner_id", "request_id"])
|
||||
)
|
||||
existing = (await session.scalars(select(CoordinationTaskRow).filter_by(owner_id=owner_id, request_id=request_id))).one()
|
||||
if existing.id == task_id and parent is not None:
|
||||
_touch(parent)
|
||||
if any(getattr(existing, key) != value for key, value in create_fields.items()):
|
||||
raise TaskConflict("Request ID was already used with different task data")
|
||||
return _snapshot(existing)
|
||||
|
||||
async def append_input(self, owner_id: str, task_id: str, *, request_id: str, content: str) -> dict:
|
||||
_request_id(request_id)
|
||||
if not content.strip():
|
||||
raise ValueError("Task input is required")
|
||||
async with self._write() as session:
|
||||
task = await self._lock_tree(session, owner_id, task_id)
|
||||
existing = await session.scalar(select(CoordinationInputRow).filter_by(task_id=task_id, request_id=request_id))
|
||||
if existing is not None:
|
||||
if existing.content != content:
|
||||
raise TaskConflict("Request ID was already used with different input")
|
||||
return _snapshot(existing)
|
||||
if task.status not in _OPEN:
|
||||
raise TaskConflict("Task is closed to new input")
|
||||
task.latest_input_seq += 1
|
||||
_touch(task)
|
||||
row = CoordinationInputRow(task_id=task_id, sequence=task.latest_input_seq, request_id=request_id, content=content, created_at=task.updated_at)
|
||||
session.add(row)
|
||||
return _snapshot(row)
|
||||
|
||||
async def list_inputs(self, owner_id: str, task_id: str, *, after: int = 0, limit: int = 100) -> list[dict]:
|
||||
if after < 0 or not 1 <= limit <= 100:
|
||||
raise ValueError("Invalid input page")
|
||||
async with self._sessions() as session:
|
||||
query = (
|
||||
select(CoordinationInputRow)
|
||||
.join(CoordinationTaskRow, CoordinationTaskRow.id == CoordinationInputRow.task_id)
|
||||
.where(CoordinationTaskRow.owner_id == owner_id, CoordinationInputRow.task_id == task_id, CoordinationInputRow.sequence > after)
|
||||
.order_by(CoordinationInputRow.sequence)
|
||||
.limit(limit)
|
||||
)
|
||||
return [_snapshot(row) for row in (await session.scalars(query)).all()]
|
||||
|
||||
async def start(self, owner_id: str, task_id: str, *, expected_revision: int) -> dict:
|
||||
"""Claim the task for work and return the revision and input watermark."""
|
||||
async with self._write() as session:
|
||||
task = await self._lock_tree(session, owner_id, task_id)
|
||||
_check_revision(task, expected_revision)
|
||||
if task.status != TaskStatus.QUEUED:
|
||||
raise TaskConflict("Only a queued task can start")
|
||||
if task.parent_id is not None:
|
||||
parent = await self._lookup(session, owner_id, task.parent_id)
|
||||
if parent.status != TaskStatus.RUNNING:
|
||||
raise TaskConflict("Parent task has not started")
|
||||
if task.depends_on:
|
||||
query = select(CoordinationTaskRow.id).filter_by(owner_id=owner_id, status=TaskStatus.SUCCEEDED)
|
||||
ready = set((await session.scalars(query.where(CoordinationTaskRow.id.in_(task.depends_on)))).all())
|
||||
if ready != set(task.depends_on):
|
||||
raise TaskConflict("Task dependencies have not succeeded")
|
||||
task.status = TaskStatus.RUNNING
|
||||
_touch(task)
|
||||
return _snapshot(task)
|
||||
|
||||
async def finish(
|
||||
self,
|
||||
owner_id: str,
|
||||
task_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
execution_epoch: int,
|
||||
covered_input_seq: int,
|
||||
outcome: TaskStatus,
|
||||
result_ref: str,
|
||||
) -> dict:
|
||||
if outcome not in (TaskStatus.SUCCEEDED, TaskStatus.FAILED) or not result_ref.strip():
|
||||
raise ValueError("A success or failure outcome and a result reference are required")
|
||||
async with self._write() as session:
|
||||
task = await self._lock_tree(session, owner_id, task_id)
|
||||
if task.status != TaskStatus.RUNNING or task.execution_epoch != execution_epoch:
|
||||
raise TaskConflict("Task no longer accepts results from this execution")
|
||||
_check_revision(task, expected_revision)
|
||||
if task.latest_input_seq != covered_input_seq:
|
||||
raise TaskConflict("Result does not cover the current task inputs")
|
||||
children = (await session.scalars(select(CoordinationTaskRow.status).filter_by(parent_id=task_id, owner_id=owner_id))).all()
|
||||
accepted = {TaskStatus.SUCCEEDED} if outcome == TaskStatus.SUCCEEDED else {TaskStatus.SUCCEEDED, TaskStatus.FAILED, TaskStatus.CANCELLED}
|
||||
if any(status not in accepted for status in children):
|
||||
raise TaskConflict("Child tasks have not settled for this outcome")
|
||||
task.status = outcome
|
||||
task.result_ref = result_ref
|
||||
_touch(task)
|
||||
return _snapshot(task)
|
||||
|
||||
async def _subtree(self, session: AsyncSession, task: CoordinationTaskRow) -> list[CoordinationTaskRow]:
|
||||
rows = (await session.scalars(select(CoordinationTaskRow).filter_by(owner_id=task.owner_id, root_id=task.root_id))).all()
|
||||
children = defaultdict(list)
|
||||
for row in rows:
|
||||
children[row.parent_id].append(row)
|
||||
result = []
|
||||
pending = deque([task])
|
||||
while pending:
|
||||
row = pending.popleft()
|
||||
result.append(row)
|
||||
pending.extend(children[row.id])
|
||||
return result
|
||||
|
||||
async def request_cancel(self, owner_id: str, task_id: str, *, expected_revision: int) -> dict:
|
||||
"""Stop accepting work in the ownership subtree before stopping its runs."""
|
||||
async with self._write() as session:
|
||||
task = await self._lock_tree(session, owner_id, task_id)
|
||||
if task.status in (TaskStatus.CANCELLING, TaskStatus.CANCELLED):
|
||||
return _snapshot(task)
|
||||
_check_revision(task, expected_revision)
|
||||
if task.status not in _OPEN:
|
||||
raise TaskConflict("Task has already finished")
|
||||
for row in await self._subtree(session, task):
|
||||
if row.status in _OPEN:
|
||||
row.status = TaskStatus.CANCELLING
|
||||
row.execution_epoch += 1
|
||||
_touch(row)
|
||||
return _snapshot(task)
|
||||
|
||||
async def confirm_cancel(self, owner_id: str, task_id: str, *, execution_epoch: int) -> dict:
|
||||
"""Record cancellation after the caller confirms all subtree runs stopped."""
|
||||
async with self._write() as session:
|
||||
task = await self._lock_tree(session, owner_id, task_id)
|
||||
if task.status not in (TaskStatus.CANCELLING, TaskStatus.CANCELLED) or task.execution_epoch != execution_epoch:
|
||||
raise TaskConflict("Cancellation no longer matches the task")
|
||||
for row in await self._subtree(session, task):
|
||||
if row.status == TaskStatus.CANCELLING:
|
||||
row.status = TaskStatus.CANCELLED
|
||||
_touch(row)
|
||||
return _snapshot(task)
|
||||
@ -0,0 +1,55 @@
|
||||
"""Add independent coordination tasks and durable ordered input."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0056_coordination_task_state"
|
||||
down_revision = "0055_thread_shares"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not sa.inspect(op.get_bind()).has_table("coord_tasks"):
|
||||
op.create_table(
|
||||
"coord_tasks",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("owner_id", sa.String(64), nullable=False),
|
||||
sa.Column("request_id", sa.String(200), nullable=False),
|
||||
sa.Column("project_id", sa.String(64)),
|
||||
sa.Column("root_id", sa.String(64), nullable=False),
|
||||
sa.Column("parent_id", sa.String(64), sa.ForeignKey("coord_tasks.id")),
|
||||
sa.Column("depends_on", sa.JSON(), nullable=False),
|
||||
sa.Column("objective", sa.Text(), nullable=False),
|
||||
sa.Column("status", sa.String(16), nullable=False),
|
||||
sa.Column("revision", sa.BigInteger(), nullable=False),
|
||||
sa.Column("execution_epoch", sa.BigInteger(), nullable=False),
|
||||
sa.Column("latest_input_seq", sa.BigInteger(), nullable=False),
|
||||
sa.Column("result_ref", sa.Text()),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("owner_id", "request_id", name="uq_coord_task_request"),
|
||||
)
|
||||
# A non-transactional SQLite upgrade can be interrupted between DDL steps.
|
||||
indexes = {index["name"] for index in sa.inspect(op.get_bind()).get_indexes("coord_tasks")}
|
||||
if "ix_coord_tasks_owner_root" not in indexes:
|
||||
op.create_index("ix_coord_tasks_owner_root", "coord_tasks", ["owner_id", "root_id"])
|
||||
if "ix_coord_tasks_owner_parent" not in indexes:
|
||||
op.create_index("ix_coord_tasks_owner_parent", "coord_tasks", ["owner_id", "parent_id"])
|
||||
if not sa.inspect(op.get_bind()).has_table("coord_inputs"):
|
||||
op.create_table(
|
||||
"coord_inputs",
|
||||
sa.Column("task_id", sa.String(64), sa.ForeignKey("coord_tasks.id"), primary_key=True),
|
||||
sa.Column("sequence", sa.BigInteger(), primary_key=True),
|
||||
sa.Column("request_id", sa.String(200), nullable=False),
|
||||
sa.Column("content", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("task_id", "request_id", name="uq_coord_input_request"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table("coord_inputs"):
|
||||
op.drop_table("coord_inputs")
|
||||
if sa.inspect(op.get_bind()).has_table("coord_tasks"):
|
||||
op.drop_table("coord_tasks")
|
||||
@ -24,6 +24,7 @@ from deerflow.persistence.channel_connections.model import (
|
||||
from deerflow.persistence.channel_leases.model import ChannelLeaseRow
|
||||
from deerflow.persistence.channel_runtime_configs.model import ChannelRuntimeConfigRow
|
||||
from deerflow.persistence.channel_thread_bindings.model import ChannelThreadBindingRow
|
||||
from deerflow.persistence.coordination.model import CoordinationInputRow, CoordinationTaskRow
|
||||
from deerflow.persistence.extension_leases.model import ExtensionExecutionLeaseRow
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.login_throttle.model import LoginThrottleRow
|
||||
@ -61,6 +62,8 @@ __all__ = [
|
||||
"ChannelOAuthStateRow",
|
||||
"ChannelRuntimeConfigRow",
|
||||
"ChannelThreadBindingRow",
|
||||
"CoordinationInputRow",
|
||||
"CoordinationTaskRow",
|
||||
"FeedbackRow",
|
||||
"LoginThrottleRow",
|
||||
"McpOAuthTokenRow",
|
||||
|
||||
198
backend/tests/test_coordination_task_repository.py
Normal file
198
backend/tests/test_coordination_task_repository.py
Normal file
@ -0,0 +1,198 @@
|
||||
"""Exercise real SQL task transitions on both supported databases."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from alembic import command
|
||||
from alembic.autogenerate import compare_metadata
|
||||
from alembic.migration import MigrationContext
|
||||
from sqlalchemy import inspect, text
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
from support.postgres import asyncpg_test_url
|
||||
|
||||
from deerflow.persistence import bootstrap
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.coordination.model import TaskStatus
|
||||
from deerflow.persistence.coordination.sql import CoordinationTaskRepository, TaskConflict, TaskNotFound
|
||||
from deerflow.persistence.postgres_schema import build_asyncpg_connect_args
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(params=["sqlite", "postgres"])
|
||||
async def database(tmp_path, request):
|
||||
backend = request.param
|
||||
schema = ""
|
||||
if backend == "sqlite":
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'coord.db'}")
|
||||
else:
|
||||
uri = os.environ.get("TEST_POSTGRES_URI")
|
||||
if not uri:
|
||||
pytest.skip("requires TEST_POSTGRES_URI")
|
||||
schema = f"coord_test_{uuid4().hex}"
|
||||
engine = create_async_engine(asyncpg_test_url(uri), connect_args=build_asyncpg_connect_args(schema))
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text(f'CREATE SCHEMA "{schema}"'))
|
||||
try:
|
||||
yield engine, backend, schema
|
||||
finally:
|
||||
if schema:
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE'))
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def repo(database):
|
||||
engine, backend, schema = database
|
||||
await bootstrap.bootstrap_schema(engine, backend=backend, postgres_schema=schema)
|
||||
return CoordinationTaskRepository(async_sessionmaker(engine))
|
||||
|
||||
|
||||
async def _finish(repo, task, *, covered_input_seq=None):
|
||||
return await repo.finish(
|
||||
"alice",
|
||||
task["id"],
|
||||
expected_revision=task["revision"],
|
||||
execution_epoch=task["execution_epoch"],
|
||||
covered_input_seq=task["latest_input_seq"] if covered_input_seq is None else covered_input_seq,
|
||||
outcome=TaskStatus.SUCCEEDED,
|
||||
result_ref="result-1",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_replay_and_owner_isolation(repo):
|
||||
async with asyncio.TaskGroup() as group:
|
||||
submissions = [group.create_task(repo.create("alice", request_id="message", objective="Review")) for _ in range(4)]
|
||||
(task_id,) = {job.result()["id"] for job in submissions}
|
||||
bob = await repo.create("bob", request_id="message", objective="Review")
|
||||
assert bob["id"] != task_id
|
||||
with pytest.raises(TaskConflict, match="different task data"):
|
||||
await repo.create("alice", request_id="message", objective="Other work")
|
||||
with pytest.raises(TaskNotFound):
|
||||
await repo.get("bob", task_id)
|
||||
with pytest.raises(TaskNotFound):
|
||||
await repo.append_input("bob", task_id, request_id="msg", content="wrong owner")
|
||||
|
||||
async with asyncio.TaskGroup() as group:
|
||||
repeats = [group.create_task(repo.append_input("alice", task_id, request_id="one", content="Check storage")) for _ in range(3)]
|
||||
other = group.create_task(repo.append_input("alice", task_id, request_id="two", content="Check logs"))
|
||||
(sequence,) = {job.result()["sequence"] for job in repeats}
|
||||
assert {sequence, other.result()["sequence"]} == {1, 2}
|
||||
assert [item["sequence"] for item in await repo.list_inputs("alice", task_id, after=1)] == [2]
|
||||
assert await repo.list_inputs("bob", task_id) == []
|
||||
with pytest.raises(TaskConflict, match="different input"):
|
||||
await repo.append_input("alice", task_id, request_id="one", content="mutated replay")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dependencies_release_in_order_and_completion_covers_current_input(repo):
|
||||
root = await repo.create("alice", request_id="root", project_id="project", objective="Release")
|
||||
first = await repo.create("alice", request_id="a", parent_id=root["id"], objective="Research")
|
||||
second = await repo.create("alice", request_id="b", parent_id=root["id"], depends_on=[first["id"]], objective="Review")
|
||||
assert first["project_id"] == second["project_id"] == "project"
|
||||
with pytest.raises(TaskConflict, match="Dependencies"):
|
||||
await repo.create("alice", request_id="bad", depends_on=[first["id"]], objective="Foreign scope")
|
||||
with pytest.raises(TaskConflict, match="Parent task has not started"):
|
||||
await repo.start("alice", first["id"], expected_revision=0)
|
||||
root = await repo.get("alice", root["id"])
|
||||
root = await repo.start("alice", root["id"], expected_revision=root["revision"])
|
||||
with pytest.raises(TaskConflict, match="dependencies"):
|
||||
await repo.start("alice", second["id"], expected_revision=0)
|
||||
with pytest.raises(TaskConflict, match="Child tasks"):
|
||||
await _finish(repo, root)
|
||||
started = await repo.start("alice", first["id"], expected_revision=0)
|
||||
await repo.append_input("alice", first["id"], request_id="new", content="Check another file")
|
||||
with pytest.raises(TaskConflict, match="revision changed"):
|
||||
await _finish(repo, started)
|
||||
latest = await repo.get("alice", first["id"])
|
||||
with pytest.raises(TaskConflict, match="current task inputs"):
|
||||
await _finish(repo, latest, covered_input_seq=0)
|
||||
assert (await _finish(repo, latest))["status"] == TaskStatus.SUCCEEDED
|
||||
assert (await repo.start("alice", second["id"], expected_revision=0))["status"] == TaskStatus.RUNNING
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_tree_fences_running_work_and_leaves_external_dependency(repo):
|
||||
prerequisite = await repo.create("alice", request_id="pre", objective="Shared work")
|
||||
prerequisite = await repo.start("alice", prerequisite["id"], expected_revision=0)
|
||||
await _finish(repo, prerequisite)
|
||||
root = await repo.create("alice", request_id="root", objective="Check release", depends_on=[prerequisite["id"]])
|
||||
child = await repo.create("alice", request_id="child", parent_id=root["id"], objective="Review")
|
||||
nested = await repo.create("alice", request_id="nested", parent_id=child["id"], objective="Test")
|
||||
root = await repo.get("alice", root["id"])
|
||||
root = await repo.start("alice", root["id"], expected_revision=root["revision"])
|
||||
child = await repo.get("alice", child["id"])
|
||||
await repo.start("alice", child["id"], expected_revision=child["revision"])
|
||||
nested = await repo.start("alice", nested["id"], expected_revision=0)
|
||||
cancelling = await repo.request_cancel("alice", root["id"], expected_revision=root["revision"])
|
||||
assert cancelling["execution_epoch"] == 1
|
||||
assert (await repo.request_cancel("alice", root["id"], expected_revision=root["revision"])) == cancelling
|
||||
assert (await repo.get("alice", nested["id"]))["status"] == TaskStatus.CANCELLING
|
||||
with pytest.raises(TaskConflict, match="no longer accepts"):
|
||||
await _finish(repo, nested)
|
||||
with pytest.raises(TaskConflict, match="closed to new input"):
|
||||
await repo.append_input("alice", nested["id"], request_id="late", content="Do more")
|
||||
with pytest.raises(TaskConflict, match="Parent task is closed"):
|
||||
await repo.create("alice", request_id="late-child", parent_id=child["id"], objective="New work")
|
||||
with pytest.raises(TaskConflict, match="Cancellation no longer"):
|
||||
await repo.confirm_cancel("alice", root["id"], execution_epoch=0)
|
||||
await repo.confirm_cancel("alice", root["id"], execution_epoch=1)
|
||||
for task in (root, child, nested):
|
||||
assert (await repo.get("alice", task["id"]))["status"] == TaskStatus.CANCELLED
|
||||
assert (await repo.get("alice", prerequisite["id"]))["status"] == TaskStatus.SUCCEEDED
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_and_followup_in_separate_transactions(repo):
|
||||
task = await repo.create("alice", request_id="root", objective="Check versions")
|
||||
task_id = task["id"]
|
||||
|
||||
async def append():
|
||||
try:
|
||||
return await repo.append_input("alice", task_id, request_id="one", content="Also check dependencies")
|
||||
except TaskConflict:
|
||||
return None
|
||||
|
||||
async def cancel():
|
||||
try:
|
||||
return await repo.request_cancel("alice", task_id, expected_revision=0)
|
||||
except TaskConflict:
|
||||
return None
|
||||
|
||||
appended, cancelled = await asyncio.gather(append(), cancel())
|
||||
assert (appended is None) != (cancelled is None)
|
||||
current = await repo.get("alice", task_id)
|
||||
assert current["revision"] == 1
|
||||
assert current["latest_input_seq"] == int(appended is not None)
|
||||
assert current["execution_epoch"] == int(cancelled is not None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_coordination_migration_matches_model_and_downgrades(database):
|
||||
engine, _, schema = database
|
||||
config = bootstrap._get_alembic_config(engine, postgres_schema=schema)
|
||||
await asyncio.to_thread(command.upgrade, config, "0055_thread_shares")
|
||||
await asyncio.to_thread(command.upgrade, config, "0056_coordination_task_state")
|
||||
async with engine.connect() as conn:
|
||||
|
||||
def diff(sync):
|
||||
context = MigrationContext.configure(
|
||||
sync,
|
||||
opts={"compare_type": True, "include_object": lambda _, name, kind, *_args: kind != "table" or name in {"coord_tasks", "coord_inputs"}},
|
||||
)
|
||||
return compare_metadata(context, Base.metadata)
|
||||
|
||||
assert await conn.run_sync(diff) == []
|
||||
repo = CoordinationTaskRepository(async_sessionmaker(engine))
|
||||
task = await repo.create("alice", request_id="root", objective="Test the upgraded schema")
|
||||
await repo.append_input("alice", task["id"], request_id="update", content="Record an input")
|
||||
await asyncio.to_thread(command.downgrade, config, "0055_thread_shares")
|
||||
async with engine.connect() as conn:
|
||||
tables = await conn.run_sync(lambda sync: set(inspect(sync).get_table_names()))
|
||||
assert "threads_meta" in tables
|
||||
assert not {"coord_tasks", "coord_inputs"} & tables
|
||||
await asyncio.to_thread(command.upgrade, config, "0056_coordination_task_state")
|
||||
assert (await repo.create("alice", request_id="root", objective="After re-upgrade"))["status"] == TaskStatus.QUEUED
|
||||
@ -44,9 +44,10 @@ pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
async def test_0047_remains_in_the_single_migration_chain():
|
||||
assert bootstrap._get_head_revision() == "0055_thread_shares"
|
||||
assert bootstrap._get_head_revision() == "0056_coordination_task_state"
|
||||
script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR))
|
||||
assert script.get_heads() == ["0055_thread_shares"]
|
||||
assert script.get_heads() == ["0056_coordination_task_state"]
|
||||
assert script.get_revision("0056_coordination_task_state").down_revision == "0055_thread_shares"
|
||||
assert script.get_revision("0055_thread_shares").down_revision == "0054_project_memory"
|
||||
assert script.get_revision("0054_project_memory").down_revision == "0053_tool_approvals"
|
||||
assert script.get_revision("0053_tool_approvals").down_revision == "0052_batch_continuation"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user