deer-flow/backend/tests/test_scheduler_agent_repository.py
ordinary being 5690221272
fix(scheduler): preserve terminal occurrence when a newer run is active (#8269)
* fix(scheduler): preserve terminal occurrence when a newer run is active

reconcile_launched_run revived a terminal occurrence whose launch returned late, assuming no other occurrence could own the active slot. A queue timeout or pause can terminalize the old occurrence while a newer occurrence is admitted and active; reviving the old row then violates uq_scheduled_task_run_active with an IntegrityError and aborts the launch path, leaving the durable run unassociated.

Under the parent and occurrence locks, detect another active occurrence first. If one exists, keep the terminal status, error and finished_at, attach only the durable run_id/started_at, and log a warning. Revive to running only when the slot is still free. Tests: repo regression reproduces the SQLite UNIQUE violation on main and passes here; full scheduler suite 1115 passed / 132 skipped; ruff and guidance clean.

* test(scheduler): pin terminal payload preservation in reconciliation

Seed the terminal occurrence with its real error and finished_at, then assert reconciliation preserves status, error and finished_at while attaching the late durable run. A future edit that clears either field now fails the regression.

* fix(scheduler): avoid backfilling start after a terminal finish

The preserve path attached the late durable run's start to an occurrence that had already finished, producing started_at > finished_at. Leave started_at untouched: the occurrence never started, and the durable run row carries the real start. Parametrize the regression over newer occurrences in running and queued status so both active-index shapes stay covered.

---------

Co-authored-by: Tong-bit-art <Tong-bit-art@users.noreply.github.com>
2026-11-16 10:22:10 +08:00

817 lines
46 KiB
Python

import asyncio
import os
import uuid
from contextlib import asynccontextmanager
from datetime import UTC, datetime, timedelta
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
import pytest
from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from deerflow.persistence.base import Base
from deerflow.persistence.postgres_schema import build_asyncpg_connect_args
from deerflow.persistence.run import RunRepository
from deerflow.persistence.run.model import RunRow
from deerflow.persistence.scheduled_task_events import ScheduledTaskEventRow
from deerflow.persistence.scheduled_task_runs import ScheduledTaskAdmissionRejected, ScheduledTaskRunRepository
from deerflow.persistence.scheduled_task_runs.model import ScheduledTaskRunRow
from deerflow.persistence.scheduled_tasks import ScheduledTaskLimitsExhausted, ScheduledTaskRepository
from deerflow.persistence.scheduled_tasks.model import ScheduledTaskRow
from deerflow.persistence.scheduled_tasks.sql import ActiveScheduledTaskMutationConflict
from deerflow.persistence.thread_meta.model import ThreadMetaRow
NOW = datetime(2026, 10, 3, 8, tzinfo=UTC)
VERDICT = {"satisfied": False, "blocker": "goal_not_met_yet", "reason": "missing evidence", "evidence_summary": "none", "relied_on_assumption": False, "stand_down_reason": "max_continuations"}
@pytest.fixture(params=["sqlite", "postgres"])
def database_backend(request):
return request.param
@asynccontextmanager
async def database(tmp_path, *, backend="sqlite"):
schema = None
if backend == "postgres":
uri = os.environ.get("TEST_POSTGRES_URI")
if not uri:
pytest.skip("requires TEST_POSTGRES_URI (real Postgres scheduler contracts)")
parts = urlsplit(uri)
scheme = "postgresql+asyncpg" if parts.scheme in {"postgres", "postgresql"} else parts.scheme
query = urlencode([(key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) if key not in {"sslmode", "channel_binding"}])
uri = urlunsplit(parts._replace(scheme=scheme, query=query))
schema = f"scheduler_contract_{uuid.uuid4().hex}"
engine = create_async_engine(uri, connect_args=build_asyncpg_connect_args(schema))
else:
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'agent-schedules.db'}")
try:
async with engine.begin() as conn:
if schema:
await conn.execute(text(f'CREATE SCHEMA "{schema}"'))
await conn.run_sync(lambda sync: Base.metadata.create_all(sync, tables=[ScheduledTaskRow.__table__, ScheduledTaskRunRow.__table__, RunRow.__table__, ThreadMetaRow.__table__, ScheduledTaskEventRow.__table__]))
sf = async_sessionmaker(engine, expire_on_commit=False)
yield sf, ScheduledTaskRepository(sf), ScheduledTaskRunRepository(sf)
finally:
if schema:
async with engine.begin() as conn:
await conn.execute(text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE'))
await engine.dispose()
async def task(repo, task_id="task", **extra):
return await repo.create(
task_id=task_id,
user_id="owner",
thread_id=None,
context_mode="fresh_thread_per_run",
assistant_id=None,
title="report",
prompt="write report",
schedule_type="interval",
schedule_spec={"every_seconds": 3600},
timezone="UTC",
next_run_at=NOW,
origin_thread_id="origin",
**extra,
)
async def occurrence(sf, repo, task_id="task", suffix="1", *, trigger="scheduled", verdict=None, durable_status="success"):
occurrence_id, run_id = f"occ-{suffix}", f"run-{suffix}"
await repo.create(run_record_id=occurrence_id, task_id=task_id, thread_id=f"thread-{suffix}", scheduled_for=NOW, trigger=trigger, status="running")
await repo.update_status(occurrence_id, status="running", run_id=run_id, started_at=NOW)
async with sf() as session:
session.add(RunRow(run_id=run_id, thread_id=f"thread-{suffix}", user_id="owner", status=durable_status, goal_verdict=verdict, metadata_json={"scheduled_task_id": task_id, "scheduled_task_run_id": occurrence_id}, created_at=NOW))
await session.commit()
return occurrence_id, run_id
async def complete(tasks, occurrence_id, run_id, *, status="success", verdict=None, error=None):
return await tasks.complete_run("task", user_id="owner", task_run_id=occurrence_id, run_id=run_id, status=status, error=error, finished_at=NOW, goal_verdict=verdict)
@pytest.mark.asyncio
async def test_origin_scope_notes_and_serialized_deadline(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
created = await task(tasks, end_at=NOW + timedelta(days=1))
assert created["end_at"] == (NOW + timedelta(days=1)).isoformat()
assert len(await tasks.list_by_user_and_thread("owner", "origin")) == 1
assert await tasks.append_standing_note("task", user_id="owner", origin_thread_id="wrong", note="develop branch") is None
updated = await tasks.append_standing_note("task", user_id="owner", origin_thread_id="origin", note="develop branch")
assert updated["standing_notes"] == ["develop branch"]
await occurrence(sf, runs)
with pytest.raises(ActiveScheduledTaskMutationConflict, match="active"):
await tasks.append_standing_note("task", user_id="owner", origin_thread_id="origin", note="another note")
@pytest.mark.asyncio
async def test_owner_quota_serializes_concurrent_creates_and_counts_paused(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (_sf, tasks, _runs):
for index in range(19):
await task(tasks, f"task-{index}")
await tasks.update("task-0", user_id="owner", updates={"status": "paused"})
results = await asyncio.gather(task(tasks, "a"), task(tasks, "b"), return_exceptions=True)
assert sum(isinstance(result, dict) for result in results) == 1
assert "20" in str(next(result for result in results if isinstance(result, Exception)))
@pytest.mark.asyncio
async def test_stop_request_is_first_terminal_only_after_resume(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
await task(tasks)
occurrence_id, run_id = await occurrence(sf, runs, durable_status="running")
assert await runs.request_stop(occurrence_id, task_id="task", run_id=run_id, user_id="intruder") is False
assert await runs.request_stop(occurrence_id, task_id="task", run_id=run_id, user_id="owner") is True
assert await complete(tasks, occurrence_id, run_id) is True
stopped = await tasks.get("task", user_id="owner")
assert stopped["status"] == "paused"
assert stopped["last_error"] == f"stopped by the agent in run {run_id}"
await tasks.update("task", user_id="owner", updates={"status": "enabled", "last_error": None})
assert await complete(tasks, occurrence_id, run_id, status="failed", error="late") is False
assert (await tasks.get("task", user_id="owner"))["status"] == "enabled"
assert (await runs.list_by_task("task"))[0]["status"] == "success"
@pytest.mark.asyncio
@pytest.mark.parametrize("method", ["recover_expired_launch_claims", "mark_stale_active_runs", "reconcile_active_runs"])
async def test_restart_restores_stop_and_goal_outcome_in_same_transaction(tmp_path, database_backend, method):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
await task(tasks, goal_objective="report is complete")
occurrence_id, run_id = await occurrence(sf, runs, verdict=VERDICT, durable_status="running")
assert await runs.request_stop(occurrence_id, task_id="task", run_id=run_id, user_id="owner")
async with sf() as session:
durable = await session.get(RunRow, run_id)
durable.status = "success"
row = await session.get(ScheduledTaskRunRow, occurrence_id)
row.status = "launching" if method == "recover_expired_launch_claims" else "running"
await session.commit()
kwargs = {"error": "restart"}
if method != "mark_stale_active_runs":
kwargs["now"] = NOW
assert await getattr(runs, method)(**kwargs) == 1
assert (await tasks.get("task", user_id="owner"))["status"] == "paused"
recorded = (await runs.list_by_task("task"))[0]
assert recorded["status"] == "unmet"
assert recorded["goal_objective"] == "report is complete"
assert recorded["goal_verdict"] == VERDICT
assert recorded["error"] == "max_continuations"
@pytest.mark.asyncio
async def test_manual_trial_does_not_consume_limit_and_deadline_prevents_admission(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
await task(tasks, max_runs=2)
for suffix, trigger in [("trial", "manual"), ("first", "scheduled"), ("second", "scheduled")]:
occurrence_id, run_id = await occurrence(sf, runs, suffix=suffix, trigger=trigger)
await complete(tasks, occurrence_id, run_id)
current = await tasks.get("task", user_id="owner")
assert current["status"] == ("completed" if suffix == "second" else "enabled")
assert current["run_count"] == 3
# Reactivating a task whose cap is used up is refused at the repository.
with pytest.raises(ScheduledTaskLimitsExhausted):
await tasks.update("task", user_id="owner", updates={"status": "enabled"})
assert (await tasks.get("task", user_id="owner"))["status"] == "completed"
# A row enabled by an older writer is still completed before admission.
async with sf() as session:
(await session.get(ScheduledTaskRow, "task")).status = "enabled"
await session.commit()
assert await tasks.complete_if_ended("task", user_id="owner", now=NOW) is True
await task(tasks, "expired", end_at=NOW - timedelta(seconds=1))
assert await tasks.complete_if_ended("expired", now=NOW) is True
assert (await tasks.get("expired", user_id="owner"))["status"] == "completed"
@pytest.mark.asyncio
async def test_streak_skips_manual_external_wait_and_failure_and_notifies_once(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
await task(tasks, goal_objective="deliver report")
observed = []
async def observer(session, parent, row, *, events):
observed.append((row.id, events, parent.status))
tasks.set_finalization_observer(observer)
for suffix, status, trigger, error in [
("1", "unmet", "scheduled", "goal_not_met_yet"),
("trial", "unmet", "manual", "goal_not_met_yet"),
("wait", "unmet", "scheduled", "external_wait"),
("failure", "failed", "scheduled", "broken"),
("2", "unmet", "scheduled", "goal_not_met_yet"),
("3", "unmet", "scheduled", "goal_not_met_yet"),
]:
occurrence_id, run_id = await occurrence(sf, runs, suffix=suffix, trigger=trigger)
await complete(tasks, occurrence_id, run_id, status=status, error=error)
assert (await tasks.get("task", user_id="owner"))["status"] == "paused"
assert observed[-1][1] == ("run_unmet", "task_paused")
assert await complete(tasks, "occ-3", "run-3", status="unmet", error="goal_not_met_yet") is False
assert len(observed) == 6
@pytest.mark.asyncio
async def test_end_condition_wins_and_observer_failure_rolls_back(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
await task(tasks, max_runs=1)
occurrence_id, run_id = await occurrence(sf, runs, durable_status="running")
assert await runs.request_stop(occurrence_id, task_id="task", run_id=run_id, user_id="owner")
async def failing_observer(*args, **kwargs):
raise RuntimeError("outbox unavailable")
tasks.set_finalization_observer(failing_observer)
with pytest.raises(RuntimeError, match="outbox unavailable"):
await complete(tasks, occurrence_id, run_id)
assert (await runs.list_by_task("task"))[0]["status"] == "running"
assert (await tasks.get("task", user_id="owner"))["run_count"] == 0
tasks.set_finalization_observer(None)
await complete(tasks, occurrence_id, run_id)
assert (await tasks.get("task", user_id="owner"))["status"] == "completed"
@pytest.mark.asyncio
async def test_previous_occurrence_uses_admission_sequence_and_only_launched_history(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
await task(tasks)
first, run_id = await occurrence(sf, runs, suffix="first", trigger="manual")
await complete(tasks, first, run_id)
await runs.create(run_record_id="unlaunched", task_id="task", thread_id="unlaunched-thread", scheduled_for=NOW + timedelta(days=1), trigger="scheduled", status="failed")
current, _run_id = await occurrence(sf, runs, suffix="current")
previous = await runs.previous_occurrence("task", before_task_run_id=current)
assert previous["id"] == first
assert previous["thread_id"] == "thread-first"
@pytest.mark.asyncio
async def test_no_verdict_once_is_failed_and_receipt_backfill_preserves_unmet(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
await task(tasks, goal_objective="deliver report")
await tasks.update("task", user_id="owner", updates={"schedule_type": "once", "schedule_spec": {"run_at": NOW.isoformat()}})
occurrence_id, run_id = await occurrence(sf, runs)
await complete(tasks, occurrence_id, run_id)
assert (await tasks.get("task", user_id="owner"))["status"] == "failed"
assert await runs.update_status(occurrence_id, status="running", run_id=run_id, started_at=NOW + timedelta(seconds=1), protect_terminal=True)
assert await runs.reconcile_launched_run(occurrence_id, task_id="task", run_id=run_id, started_at=NOW)
row = (await runs.list_by_task("task"))[0]
assert row["status"] == "unmet"
assert row["error"] == "no_verdict"
assert (await tasks.get("task", user_id="owner"))["run_count"] == 1
@pytest.mark.parametrize("newer_status", ["running", "queued"])
@pytest.mark.asyncio
async def test_reconcile_keeps_terminal_run_when_newer_occurrence_is_active(tmp_path, database_backend, newer_status):
async with database(tmp_path, backend=database_backend) as (_sf, tasks, runs):
await task(tasks)
await runs.create(
run_record_id="occ-old",
task_id="task",
thread_id="thread-old",
scheduled_for=NOW,
trigger="scheduled",
status="failed",
)
# Seed the real terminal payload: runs.create cannot set error or
# finished_at, and the guard must preserve both, not just the status.
await runs.update_status("occ-old", status="failed", error="queue timeout", finished_at=NOW)
await runs.create(
run_record_id="occ-new",
task_id="task",
thread_id="thread-new",
scheduled_for=NOW + timedelta(minutes=1),
trigger="manual",
status=newer_status,
)
assert await runs.reconcile_launched_run("occ-old", task_id="task", run_id="run-late", started_at=NOW + timedelta(seconds=1)) is True
rows = {row["id"]: row for row in await runs.list_by_task("task")}
assert rows["occ-old"]["status"] == "failed"
assert rows["occ-old"]["error"] == "queue timeout"
assert datetime.fromisoformat(rows["occ-old"]["finished_at"]) == NOW
assert rows["occ-old"]["started_at"] is None
assert rows["occ-old"]["run_id"] == "run-late"
assert rows["occ-new"]["status"] == newer_status
@pytest.mark.asyncio
async def test_success_resets_unmet_streak_and_old_callback_cannot_pause_new_parent(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
await task(tasks, goal_objective="deliver report")
for suffix, satisfied in [("1", False), ("2", False), ("3", True), ("4", False), ("5", False)]:
occurrence_id, run_id = await occurrence(sf, runs, suffix=suffix)
verdict = dict(VERDICT, satisfied=satisfied, relied_on_assumption=satisfied)
await complete(tasks, occurrence_id, run_id, verdict=verdict)
assert (await tasks.get("task", user_id="owner"))["status"] == "enabled"
# The old result cannot be changed after it first became terminal.
assert await complete(tasks, "occ-3", "run-3", verdict=VERDICT) is False
assert (await tasks.get("task", user_id="owner"))["status"] == "enabled"
@pytest.mark.asyncio
async def test_queued_dispatch_checks_deadline_under_parent_lock(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (_sf, tasks, runs):
await task(tasks, end_at=NOW + timedelta(seconds=1))
await runs.create(run_record_id="waiting", task_id="task", thread_id="thread", scheduled_for=NOW, trigger="scheduled", status="queued")
assert await runs.claim_queued_run("waiting", lease_owner="worker", now=NOW + timedelta(seconds=2), lease_seconds=30, global_max_concurrent_runs=1) is None
assert (await tasks.get("task", user_id="owner"))["status"] == "completed"
assert (await runs.list_by_task("task"))[0]["status"] == "skipped"
@pytest.mark.asyncio
async def test_notes_bounds_and_tool_quota_excludes_terminal_and_legacy_tasks(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (_sf, tasks, _runs):
for index in range(20):
await task(tasks, f"task-{index}")
await tasks.update("task-0", user_id="owner", updates={"status": "completed"})
await task(tasks, "replacement", standing_notes=["note"] * 10)
with pytest.raises(ValueError, match="10"):
await tasks.append_standing_note("replacement", user_id="owner", origin_thread_id="origin", note="extra")
with pytest.raises(ValueError, match="500"):
await tasks.append_standing_note("replacement", user_id="owner", origin_thread_id="origin", note="x" * 501)
# Existing REST callers retain their unbounded quota behavior.
await tasks.create(
task_id="legacy",
user_id="owner",
thread_id=None,
context_mode="fresh_thread_per_run",
assistant_id=None,
title="legacy",
prompt="test",
schedule_type="interval",
schedule_spec={"every_seconds": 3600},
timezone="UTC",
next_run_at=NOW,
)
@pytest.mark.asyncio
async def test_concurrent_terminal_callbacks_accept_only_one_outcome(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
await task(tasks)
occurrence_id, run_id = await occurrence(sf, runs)
results = await asyncio.gather(complete(tasks, occurrence_id, run_id), complete(tasks, occurrence_id, run_id, status="failed", error="competing callback"))
assert sorted(results) == [False, True]
row = (await runs.list_by_task("task"))[0]
assert row["status"] == ("success" if results[0] else "failed")
assert (await tasks.get("task", user_id="owner"))["run_count"] == 1
@pytest.mark.asyncio
async def test_postgres_owner_quota_serializes_separate_sessions():
uri = os.environ.get("TEST_POSTGRES_URI")
if not uri:
pytest.skip("requires TEST_POSTGRES_URI (real Postgres quota concurrency)")
parts = urlsplit(uri)
scheme = "postgresql+asyncpg" if parts.scheme in {"postgres", "postgresql"} else parts.scheme
query = urlencode([(key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) if key not in {"sslmode", "channel_binding"}])
uri = urlunsplit(parts._replace(scheme=scheme, query=query))
schema = f"scheduler_quota_{uuid.uuid4().hex}"
engine = create_async_engine(uri, connect_args=build_asyncpg_connect_args(schema))
try:
async with engine.begin() as conn:
await conn.execute(text(f'CREATE SCHEMA "{schema}"'))
await conn.run_sync(lambda sync: Base.metadata.create_all(sync, tables=[ScheduledTaskRow.__table__, ScheduledTaskRunRow.__table__]))
sf = async_sessionmaker(engine, expire_on_commit=False)
tasks_a, tasks_b = ScheduledTaskRepository(sf), ScheduledTaskRepository(sf)
for index in range(19):
await task(tasks_a, f"task-{index}")
results = await asyncio.gather(task(tasks_a, "a"), task(tasks_b, "b"), return_exceptions=True)
assert sum(isinstance(result, dict) for result in results) == 1
assert len(await tasks_a.list_by_origin_thread("owner", "origin")) == 20
winner = next(result["id"] for result in results if isinstance(result, dict))
await tasks_a.update(winner, user_id="owner", updates={"status": "completed"})
results = await asyncio.gather(tasks_a.update(winner, user_id="owner", updates={"status": "enabled"}, require_mutable=True), task(tasks_b, "competing-new"), return_exceptions=True)
assert sum(isinstance(result, dict) for result in results) == 1
assert type(next(result for result in results if isinstance(result, Exception))).__name__ == "ScheduledTaskQuotaExceeded"
assert sum(row["status"] in {"enabled", "running", "paused"} for row in await tasks_a.list_by_origin_thread("owner", "origin")) == 20
finally:
async with engine.begin() as conn:
await conn.execute(text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE'))
await engine.dispose()
@pytest.mark.asyncio
async def test_reactivation_and_creation_share_atomic_owner_quota(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (_sf, tasks, _runs):
await task(tasks, "old-terminal")
await tasks.update("old-terminal", user_id="owner", updates={"status": "completed"})
for index in range(19):
await task(tasks, f"active-{index}")
results = await asyncio.gather(tasks.update("old-terminal", user_id="owner", updates={"status": "enabled"}, require_mutable=True), task(tasks, "new"), return_exceptions=True)
assert sum(isinstance(result, dict) for result in results) == 1
assert type(next(result for result in results if isinstance(result, Exception))).__name__ == "ScheduledTaskQuotaExceeded"
assert sum(row["status"] in {"enabled", "running", "paused"} for row in await tasks.list_by_origin_thread("owner", "origin")) == 20
@pytest.mark.asyncio
async def test_full_quota_blocks_terminal_reactivation_but_preserves_paused_and_legacy(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (_sf, tasks, _runs):
await task(tasks, "old-terminal")
await tasks.update("old-terminal", user_id="owner", updates={"status": "completed"})
for index in range(20):
await task(tasks, f"active-{index}")
with pytest.raises(ValueError, match="20"):
await tasks.update("old-terminal", user_id="owner", updates={"status": "enabled"}, require_mutable=True)
assert (await tasks.get("old-terminal", user_id="owner"))["status"] == "completed"
await tasks.update("active-0", user_id="owner", updates={"status": "paused"})
assert (await tasks.update("active-0", user_id="owner", updates={"status": "enabled"}, require_mutable=True))["status"] == "enabled"
await tasks.create(
task_id="legacy",
user_id="owner",
thread_id=None,
context_mode="fresh_thread_per_run",
assistant_id=None,
title="legacy",
prompt="test",
schedule_type="interval",
schedule_spec={"every_seconds": 3600},
timezone="UTC",
next_run_at=NOW,
)
await tasks.update("legacy", user_id="owner", updates={"status": "completed"})
assert (await tasks.update("legacy", user_id="owner", updates={"status": "enabled"}, require_mutable=True))["status"] == "enabled"
@pytest.mark.asyncio
async def test_terminal_durable_result_wins_a_failed_takeover_race(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
await task(tasks, goal_objective="deliver report")
occurrence_id, run_id = await occurrence(sf, runs, durable_status="running")
class CompletionWinsTakeover(RunRepository):
async def claim_for_takeover(self, selected_run_id, **kwargs):
# Another worker completes between the recovery snapshot and
# takeover CAS. The real repository read observes that commit.
async with sf() as session:
row = await session.get(RunRow, selected_run_id)
row.status = "success"
row.goal_verdict = {"satisfied": True, "relied_on_assumption": False}
await session.commit()
return False
recovering = ScheduledTaskRunRepository(sf, run_repository=CompletionWinsTakeover(sf))
assert await recovering.reconcile_active_runs(error="restart", now=NOW) == 1
recorded = (await runs.list_by_task("task"))[0]
assert recorded["status"] == "success"
assert recorded["goal_verdict"]["satisfied"] is True
assert (await tasks.get("task", user_id="owner"))["run_count"] == 1
@pytest.mark.asyncio
async def test_update_preserves_goal_fresh_thread_invariant_under_lock(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (_sf, tasks, _runs):
await task(tasks, goal_objective="deliver report")
before = await tasks.get("task", user_id="owner")
with pytest.raises(ValueError, match="fresh_thread_per_run"):
await tasks.update("task", user_id="owner", updates={"context_mode": "reuse_thread", "thread_id": "reused"}, require_mutable=True)
assert await tasks.get("task", user_id="owner") == before
await task(tasks, "no-goal")
updated = await tasks.update("no-goal", user_id="owner", updates={"context_mode": "reuse_thread", "thread_id": "reused"}, require_mutable=True)
assert updated["context_mode"] == "reuse_thread"
@pytest.mark.asyncio
@pytest.mark.parametrize("objective", [pytest.param("", id="empty"), pytest.param(" \t\n ", id="blank"), pytest.param("x" * 4001, id="too-long")])
async def test_create_rejects_invalid_goal_without_inserting_task(tmp_path, database_backend, objective):
async with database(tmp_path, backend=database_backend) as (_sf, tasks, _runs):
with pytest.raises(ValueError, match="Goal objective"):
await task(tasks, goal_objective=objective)
assert await tasks.list_by_origin_thread("owner", "origin") == []
@pytest.mark.asyncio
@pytest.mark.parametrize("objective", [pytest.param("", id="empty"), pytest.param(" \t\n ", id="blank"), pytest.param("x" * 4001, id="too-long")])
async def test_update_rejects_invalid_goal_without_changing_task(tmp_path, database_backend, objective):
async with database(tmp_path, backend=database_backend) as (_sf, tasks, _runs):
before = await task(tasks, goal_objective="deliver report")
with pytest.raises(ValueError, match="Goal objective"):
await tasks.update("task", user_id="owner", updates={"goal_objective": objective, "title": "changed"}, require_mutable=True)
assert await tasks.get("task", user_id="owner") == before
@pytest.mark.asyncio
@pytest.mark.parametrize("objective", [pytest.param(None, id="none"), pytest.param(" deliver\t report\n ", id="original-text"), pytest.param(" \n" + "x" * 4000 + "\t ", id="normalized-limit")])
async def test_create_and_update_preserve_valid_goal_text(tmp_path, database_backend, objective):
async with database(tmp_path, backend=database_backend) as (_sf, tasks, _runs):
created = await task(tasks, goal_objective=objective)
assert created["goal_objective"] == objective
await task(tasks, "updated", goal_objective="previous goal")
updated = await tasks.update("updated", user_id="owner", updates={"goal_objective": objective}, require_mutable=True)
assert updated["goal_objective"] == objective
assert (await tasks.get("updated", user_id="owner"))["goal_objective"] == objective
@pytest.mark.asyncio
@pytest.mark.parametrize("live_count", [19, 20])
async def test_terminal_pause_is_finished_and_takes_no_quota_slot(tmp_path, database_backend, live_count):
async with database(tmp_path, backend=database_backend) as (_sf, tasks, _runs):
await task(tasks, "terminal")
await tasks.update("terminal", user_id="owner", updates={"status": "completed"})
for index in range(live_count):
await task(tasks, f"live-{index}")
before = await tasks.get("terminal", user_id="owner")
assert await tasks.pause_with_queue_cancellation("terminal", user_id="owner", error="pause", now=NOW) == "finished"
assert await tasks.get("terminal", user_id="owner") == before
assert sum(row["status"] in {"enabled", "running", "paused"} for row in await tasks.list_by_origin_thread("owner", "origin")) == live_count
await tasks.create(
task_id="legacy",
user_id="owner",
thread_id=None,
context_mode="fresh_thread_per_run",
assistant_id=None,
title="legacy",
prompt="test",
schedule_type="interval",
schedule_spec={"every_seconds": 3600},
timezone="UTC",
next_run_at=NOW,
)
await tasks.update("legacy", user_id="owner", updates={"status": "completed"})
assert await tasks.pause_with_queue_cancellation("legacy", user_id="owner", error="legacy pause", now=NOW) == "finished"
@pytest.mark.asyncio
async def test_terminal_pause_cannot_race_a_create_out_of_its_quota_slot(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (_sf, tasks, _runs):
await task(tasks, "terminal")
await tasks.update("terminal", user_id="owner", updates={"status": "completed"})
for index in range(19):
await task(tasks, f"live-{index}")
results = await asyncio.gather(tasks.pause_with_queue_cancellation("terminal", user_id="owner", error="pause", now=NOW), task(tasks, "new"), return_exceptions=True)
assert results[0] == "finished"
assert isinstance(results[1], dict)
assert sum(row["status"] in {"enabled", "running", "paused"} for row in await tasks.list_by_origin_thread("owner", "origin")) == 20
# --- Lifecycle events (task_stopped / task_paused / task_finished) ----------------------
async def lifecycle_task(sf, tasks, runs, *, once=False, **extra):
"""The task (``once`` makes it a once task) and its originating chat, with the
scheduler's real finalization observer installed; returns each recorded call."""
from app.scheduler.service import ScheduledTaskService
await task(tasks, **extra)
if once:
await tasks.update("task", user_id="owner", updates={"schedule_type": "once", "schedule_spec": {"run_at": NOW.isoformat()}})
async with sf() as session:
session.add(ThreadMetaRow(thread_id="origin", user_id="owner"))
await session.commit()
service = ScheduledTaskService(task_repo=tasks, task_run_repo=runs, launch_run=None, poll_interval_seconds=60, lease_seconds=30, max_concurrent_runs=3)
observed = []
async def observer(session, parent, row, *, events):
observed.append((row.id if row is not None else None, events))
await service._on_finalization(session, parent, row, events=events)
for repository in (tasks, runs):
repository.set_finalization_observer(observer)
return observed
def lifecycle_calls(observed):
return [(occurrence_id, tuple(event for event in events if event.startswith("task_"))) for occurrence_id, events in observed if any(event.startswith("task_") for event in events)]
async def event_rows(sf):
async with sf() as session:
return list((await session.execute(select(ScheduledTaskEventRow).order_by(ScheduledTaskEventRow.created_at, ScheduledTaskEventRow.id))).scalars())
@pytest.mark.asyncio
async def test_agent_stop_emits_task_stopped_once(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs)
occurrence_id, run_id = await occurrence(sf, runs, durable_status="running")
assert await runs.request_stop(occurrence_id, task_id="task", run_id=run_id, user_id="owner")
assert await complete(tasks, occurrence_id, run_id) is True
assert observed == [(occurrence_id, ("run_completed", "task_stopped"))]
assert await complete(tasks, occurrence_id, run_id) is False
assert len(observed) == 1
rows = await event_rows(sf)
assert [(row.event, row.reason_code, row.anchor, row.occurrence_id, row.thread_id) for row in rows] == [("task_stopped", "agent_stop", occurrence_id, occurrence_id, "origin")]
@pytest.mark.asyncio
async def test_max_runs_completion_emits_task_finished(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs, max_runs=2)
for suffix in ("1", "2"):
occurrence_id, run_id = await occurrence(sf, runs, suffix=suffix)
await complete(tasks, occurrence_id, run_id)
assert observed == [("occ-1", ("run_completed",)), ("occ-2", ("run_completed", "task_finished"))]
assert (await tasks.get("task", user_id="owner"))["status"] == "completed"
rows = await event_rows(sf)
assert [(row.event, row.reason_code, row.anchor) for row in rows] == [("task_finished", "max_runs", "occ-2")]
assert rows[0].payload_json["max_runs"] == 2
@pytest.mark.asyncio
async def test_end_condition_wins_over_agent_stop_events(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs, max_runs=1)
occurrence_id, run_id = await occurrence(sf, runs, durable_status="running")
assert await runs.request_stop(occurrence_id, task_id="task", run_id=run_id, user_id="owner")
await complete(tasks, occurrence_id, run_id)
assert observed == [(occurrence_id, ("run_completed", "task_finished"))]
assert [row.event for row in await event_rows(sf)] == ["task_finished"]
@pytest.mark.asyncio
async def test_idle_end_at_finish_emits_with_end_anchor_once(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs, end_at=NOW)
assert await tasks.complete_if_ended("task", user_id="owner", now=NOW + timedelta(seconds=1)) is True
assert await tasks.complete_if_ended("task", user_id="owner", now=NOW + timedelta(seconds=2)) is True
assert observed == [(None, ("task_finished",))]
rows = await event_rows(sf)
assert [(row.event, row.reason_code, row.anchor, row.occurrence_id) for row in rows] == [("task_finished", "end_at", f"end:{NOW.isoformat()}", None)]
assert (await tasks.get("task", user_id="owner"))["status"] == "completed"
@pytest.mark.asyncio
async def test_idle_max_runs_finish_then_end_time_finish_writes_two_events(tmp_path, database_backend):
"""An idle max_runs finish before the end time must not take the end-time
anchor: after a reactivation, the real end-time finish still gets its line."""
end = NOW + timedelta(hours=1)
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs, end_at=end, max_runs=1)
await runs.create(run_record_id="used", task_id="task", thread_id="thread", scheduled_for=NOW, trigger="scheduled", status="queued")
async with sf() as session:
row = await session.get(ScheduledTaskRunRow, "used")
row.status, row.launch_accounted, row.finished_at = "success", True, NOW
await session.commit()
assert await tasks.complete_if_ended("task", user_id="owner", now=NOW + timedelta(seconds=1)) is True
async with sf() as session:
parent = await session.get(ScheduledTaskRow, "task")
parent.status, parent.max_runs = "enabled", 5
await session.commit()
assert await tasks.complete_if_ended("task", user_id="owner", now=end + timedelta(seconds=1)) is True
assert observed == [(None, ("task_finished",)), (None, ("task_finished",))]
rows = await event_rows(sf)
assert [(row.event, row.reason_code, row.anchor) for row in rows] == [
("task_finished", "max_runs", "seq:1"),
("task_finished", "end_at", f"end:{end.isoformat()}"),
]
@pytest.mark.asyncio
async def test_claim_time_end_skip_emits_task_finished_once(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs, end_at=NOW + timedelta(seconds=1))
await runs.create(run_record_id="waiting", task_id="task", thread_id="thread", scheduled_for=NOW, trigger="scheduled", status="queued")
assert await runs.claim_queued_run("waiting", lease_owner="worker", now=NOW + timedelta(seconds=2), lease_seconds=30, global_max_concurrent_runs=1) is None
assert lifecycle_calls(observed) == [("waiting", ("task_finished",))]
assert len(observed) == 1
rows = await event_rows(sf)
assert [(row.event, row.reason_code, row.anchor) for row in rows] == [("task_finished", "end_at", "waiting")]
assert (await tasks.get("task", user_id="owner"))["status"] == "completed"
@pytest.mark.asyncio
async def test_complete_if_ended_with_queued_row_emits_once(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs, end_at=NOW + timedelta(seconds=1))
await runs.create(run_record_id="waiting", task_id="task", thread_id="thread", scheduled_for=NOW, trigger="scheduled", status="queued")
assert await tasks.complete_if_ended("task", user_id="owner", now=NOW + timedelta(seconds=2)) is True
assert observed == [("waiting", ("task_finished",))]
assert [(row.event, row.anchor) for row in await event_rows(sf)] == [("task_finished", "waiting")]
assert (await runs.list_by_task("task"))[0]["status"] == "skipped"
@pytest.mark.asyncio
async def test_complete_if_ended_with_queued_manual_row_emits_idle_finish(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs, end_at=NOW + timedelta(seconds=1))
await runs.create(run_record_id="trial", task_id="task", thread_id="thread", scheduled_for=NOW, trigger="manual", status="queued")
assert await tasks.complete_if_ended("task", user_id="owner", now=NOW + timedelta(seconds=2)) is True
assert observed == [(None, ("task_finished",))]
rows = await event_rows(sf)
assert [(row.event, row.anchor, row.occurrence_id) for row in rows] == [("task_finished", f"end:{(NOW + timedelta(seconds=1)).isoformat()}", None)]
# The waiting trial is left alone; only scheduled rows are skipped.
assert (await runs.list_by_task("task"))[0]["status"] == "queued"
@pytest.mark.asyncio
async def test_once_task_end_skip_stays_silent(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs, once=True, end_at=NOW + timedelta(seconds=1))
await runs.create(run_record_id="waiting", task_id="task", thread_id="thread", scheduled_for=NOW, trigger="scheduled", status="queued")
assert await runs.claim_queued_run("waiting", lease_owner="worker", now=NOW + timedelta(seconds=2), lease_seconds=30, global_max_concurrent_runs=1) is None
# As in PR1: skipped -> cancelled, then the end condition marks it completed.
assert (await tasks.get("task", user_id="owner"))["status"] == "completed"
assert lifecycle_calls(observed) == []
assert await event_rows(sf) == []
@pytest.mark.asyncio
@pytest.mark.parametrize(
("status", "error", "reason"),
[("success", None, "once_done"), ("failed", "boom", "once_failed"), ("unmet", "goal_not_met_yet", "once_failed"), ("interrupted", "cancelled", None), ("skipped", None, None)],
)
async def test_once_task_finish_reasons(tmp_path, database_backend, status, error, reason):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs, once=True)
occurrence_id, run_id = await occurrence(sf, runs)
assert await complete(tasks, occurrence_id, run_id, status=status, error=error) is True
rows = await event_rows(sf)
if reason is None:
assert lifecycle_calls(observed) == []
assert rows == []
else:
assert lifecycle_calls(observed) == [(occurrence_id, ("task_finished",))]
assert [(row.event, row.reason_code, row.anchor) for row in rows] == [("task_finished", reason, occurrence_id)]
@pytest.mark.asyncio
async def test_admission_time_end_emits_task_finished(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs, end_at=NOW - timedelta(seconds=1))
with pytest.raises(ScheduledTaskAdmissionRejected) as rejected:
await runs.create(run_record_id="admitted", task_id="task", thread_id="thread", scheduled_for=NOW, trigger="scheduled", status="queued", coordinate_with_task=True, expected_task_user_id="owner")
assert rejected.value.reason == "ended"
assert (await tasks.get("task", user_id="owner"))["status"] == "completed"
assert observed == [(None, ("task_finished",))]
rows = await event_rows(sf)
assert [(row.event, row.reason_code, row.occurrence_id) for row in rows] == [("task_finished", "end_at", None)]
assert await runs.list_by_task("task") == []
@pytest.mark.asyncio
async def test_manual_trial_that_reaches_end_at_emits_task_finished(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs, end_at=NOW)
occurrence_id, run_id = await occurrence(sf, runs, suffix="trial", trigger="manual")
await complete(tasks, occurrence_id, run_id)
# Finalization reports the run event too; the observer drops only that for a trial.
assert observed == [(occurrence_id, ("run_completed", "task_finished"))]
assert [(row.event, row.reason_code, row.anchor) for row in await event_rows(sf)] == [("task_finished", "end_at", occurrence_id)]
assert (await tasks.get("task", user_id="owner"))["status"] == "completed"
@pytest.mark.asyncio
async def test_manual_trial_agent_stop_emits_task_stopped(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs)
occurrence_id, run_id = await occurrence(sf, runs, suffix="trial", trigger="manual", durable_status="running")
assert await runs.request_stop(occurrence_id, task_id="task", run_id=run_id, user_id="owner")
await complete(tasks, occurrence_id, run_id)
assert observed == [(occurrence_id, ("run_completed", "task_stopped"))]
assert [row.event for row in await event_rows(sf)] == ["task_stopped"]
assert (await tasks.get("task", user_id="owner"))["status"] == "paused"
@pytest.mark.asyncio
async def test_agent_stop_after_failed_run_pauses_and_says_run_failed(tmp_path, database_backend):
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
observed = await lifecycle_task(sf, tasks, runs)
occurrence_id, run_id = await occurrence(sf, runs, durable_status="running")
assert await runs.request_stop(occurrence_id, task_id="task", run_id=run_id, user_id="owner")
await complete(tasks, occurrence_id, run_id, status="failed", error="boom")
# The transition is PR1's: a stop request pauses whatever the run outcome.
stopped = await tasks.get("task", user_id="owner")
assert stopped["status"] == "paused"
assert stopped["last_error"] == f"stopped by the agent in run {run_id}"
assert observed == [(occurrence_id, ("run_failed", "task_stopped"))]
(row,) = await event_rows(sf)
assert (row.event, row.payload_json["run_status"]) == ("task_stopped", "failed")
def test_no_inline_end_condition_completion():
"""Only finalize_occurrence and finish_task_at_end_condition finish a task.
The once-task repair in ``_associate_task_with_run`` mirrors a once run's
own outcome (not an end condition) and is the single allowed assignment.
"""
import ast
from pathlib import Path
from deerflow.persistence.scheduled_task_runs import sql as run_sql
from deerflow.persistence.scheduled_tasks import sql as task_sql
class Finder(ast.NodeVisitor):
def __init__(self):
self.stack = []
self.found = []
def _visit_function(self, node):
self.stack.append(node.name)
self.generic_visit(node)
self.stack.pop()
visit_FunctionDef = _visit_function
visit_AsyncFunctionDef = _visit_function
def visit_Assign(self, node):
targets_status = any(isinstance(target, ast.Attribute) and target.attr == "status" for target in node.targets)
if targets_status and isinstance(node.value, ast.Constant) and node.value.value == "completed":
self.found.append(self.stack[-1] if self.stack else "<module>")
self.generic_visit(node)
found = []
for module in (run_sql, task_sql):
finder = Finder()
finder.visit(ast.parse(Path(module.__file__).read_text(encoding="utf-8")))
found += [(module.__name__.rsplit(".", 2)[-2], name) for name in finder.found]
assert found == [("scheduled_task_runs", "_associate_task_with_run")]