diff --git a/backend/packages/harness/deerflow/persistence/AGENTS.md b/backend/packages/harness/deerflow/persistence/AGENTS.md index ea36586720..adae208cda 100644 --- a/backend/packages/harness/deerflow/persistence/AGENTS.md +++ b/backend/packages/harness/deerflow/persistence/AGENTS.md @@ -93,6 +93,9 @@ with best-effort cleanup, preserving the original publication outcome. - **Recovery never takes over a live run.** It finalizes an occurrence whose durable run row is already terminal, and it returns expired launch claims to the queue. +- **Late launch reconciliation keeps a terminal occurrence terminal** when a + newer occurrence owns the active slot; attach only `run_id`/`started_at`, + since reviving would violate the one-active index. - **The scheduler is single-instance by default.** Multi-instance mode needs shared PostgreSQL and the settings that startup validates. - **Run-history filters are occurrence states**, not task states. Apply the diff --git a/backend/packages/harness/deerflow/persistence/scheduled_task_runs/sql.py b/backend/packages/harness/deerflow/persistence/scheduled_task_runs/sql.py index d55800ac26..7ca055ceb6 100644 --- a/backend/packages/harness/deerflow/persistence/scheduled_task_runs/sql.py +++ b/backend/packages/harness/deerflow/persistence/scheduled_task_runs/sql.py @@ -880,6 +880,33 @@ class ScheduledTaskRunRepository: return False if row.status in TERMINAL_RUN_STATUSES: if row.run_id is None: + # A newer occurrence may already own the active slot when + # this late launch returns. Reviving this terminal row in + # that case would violate uq_scheduled_task_run_active and + # abort the launch path; preserve the terminal outcome but + # attach the durable run for completion and later fencing. + newer_active = await session.scalar( + select(ScheduledTaskRunRow.id) + .where( + ScheduledTaskRunRow.task_id == task_id, + ScheduledTaskRunRow.id != run_record_id, + ScheduledTaskRunRow.status.in_(ACTIVE_RUN_STATUSES), + ) + .limit(1) + ) + if newer_active is not None: + # The terminal occurrence never started; its durable + # run row carries the real start. Backfilling + # started_at here would place it after finished_at. + row.run_id = run_id + await session.commit() + logger.warning( + "Scheduled task-run %s launched durable run %s while occurrence %s holds the active slot; keeping the terminal status", + run_record_id, + run_id, + newer_active, + ) + return True # Timeout/pause can terminalize a claim that recovery # briefly returned to ``queued`` while Gateway admission # was still completing. A returned durable run is the diff --git a/backend/tests/test_scheduler_agent_repository.py b/backend/tests/test_scheduler_agent_repository.py index f5dd4a6e1b..af4f966ac0 100644 --- a/backend/tests/test_scheduler_agent_repository.py +++ b/backend/tests/test_scheduler_agent_repository.py @@ -257,6 +257,42 @@ async def test_no_verdict_once_is_failed_and_receipt_backfill_preserves_unmet(tm assert (await tasks.get("task", user_id="owner"))["run_count"] == 1 +@pytest.mark.parametrize("newer_status", ["running", "queued"]) +@pytest.mark.asyncio +async def test_reconcile_keeps_terminal_run_when_newer_occurrence_is_active(tmp_path, database_backend, newer_status): + async with database(tmp_path, backend=database_backend) as (_sf, tasks, runs): + await task(tasks) + await runs.create( + run_record_id="occ-old", + task_id="task", + thread_id="thread-old", + scheduled_for=NOW, + trigger="scheduled", + status="failed", + ) + # Seed the real terminal payload: runs.create cannot set error or + # finished_at, and the guard must preserve both, not just the status. + await runs.update_status("occ-old", status="failed", error="queue timeout", finished_at=NOW) + await runs.create( + run_record_id="occ-new", + task_id="task", + thread_id="thread-new", + scheduled_for=NOW + timedelta(minutes=1), + trigger="manual", + status=newer_status, + ) + + assert await runs.reconcile_launched_run("occ-old", task_id="task", run_id="run-late", started_at=NOW + timedelta(seconds=1)) is True + + rows = {row["id"]: row for row in await runs.list_by_task("task")} + assert rows["occ-old"]["status"] == "failed" + assert rows["occ-old"]["error"] == "queue timeout" + assert datetime.fromisoformat(rows["occ-old"]["finished_at"]) == NOW + assert rows["occ-old"]["started_at"] is None + assert rows["occ-old"]["run_id"] == "run-late" + assert rows["occ-new"]["status"] == newer_status + + @pytest.mark.asyncio async def test_success_resets_unmet_streak_and_old_callback_cannot_pause_new_parent(tmp_path, database_backend): async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):