mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-16 08:20:34 +00:00
fix(scheduler): preserve terminal occurrence when a newer run is active (#8269)
* fix(scheduler): preserve terminal occurrence when a newer run is active reconcile_launched_run revived a terminal occurrence whose launch returned late, assuming no other occurrence could own the active slot. A queue timeout or pause can terminalize the old occurrence while a newer occurrence is admitted and active; reviving the old row then violates uq_scheduled_task_run_active with an IntegrityError and aborts the launch path, leaving the durable run unassociated. Under the parent and occurrence locks, detect another active occurrence first. If one exists, keep the terminal status, error and finished_at, attach only the durable run_id/started_at, and log a warning. Revive to running only when the slot is still free. Tests: repo regression reproduces the SQLite UNIQUE violation on main and passes here; full scheduler suite 1115 passed / 132 skipped; ruff and guidance clean. * test(scheduler): pin terminal payload preservation in reconciliation Seed the terminal occurrence with its real error and finished_at, then assert reconciliation preserves status, error and finished_at while attaching the late durable run. A future edit that clears either field now fails the regression. * fix(scheduler): avoid backfilling start after a terminal finish The preserve path attached the late durable run's start to an occurrence that had already finished, producing started_at > finished_at. Leave started_at untouched: the occurrence never started, and the durable run row carries the real start. Parametrize the regression over newer occurrences in running and queued status so both active-index shapes stay covered. --------- Co-authored-by: Tong-bit-art <Tong-bit-art@users.noreply.github.com>
This commit is contained in:
parent
75443ecc23
commit
5690221272
@ -93,6 +93,9 @@ with best-effort cleanup, preserving the original publication outcome.
|
||||
- **Recovery never takes over a live run.** It finalizes an occurrence whose
|
||||
durable run row is already terminal, and it returns expired launch claims to
|
||||
the queue.
|
||||
- **Late launch reconciliation keeps a terminal occurrence terminal** when a
|
||||
newer occurrence owns the active slot; attach only `run_id`/`started_at`,
|
||||
since reviving would violate the one-active index.
|
||||
- **The scheduler is single-instance by default.** Multi-instance mode needs
|
||||
shared PostgreSQL and the settings that startup validates.
|
||||
- **Run-history filters are occurrence states**, not task states. Apply the
|
||||
|
||||
@ -880,6 +880,33 @@ class ScheduledTaskRunRepository:
|
||||
return False
|
||||
if row.status in TERMINAL_RUN_STATUSES:
|
||||
if row.run_id is None:
|
||||
# A newer occurrence may already own the active slot when
|
||||
# this late launch returns. Reviving this terminal row in
|
||||
# that case would violate uq_scheduled_task_run_active and
|
||||
# abort the launch path; preserve the terminal outcome but
|
||||
# attach the durable run for completion and later fencing.
|
||||
newer_active = await session.scalar(
|
||||
select(ScheduledTaskRunRow.id)
|
||||
.where(
|
||||
ScheduledTaskRunRow.task_id == task_id,
|
||||
ScheduledTaskRunRow.id != run_record_id,
|
||||
ScheduledTaskRunRow.status.in_(ACTIVE_RUN_STATUSES),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
if newer_active is not None:
|
||||
# The terminal occurrence never started; its durable
|
||||
# run row carries the real start. Backfilling
|
||||
# started_at here would place it after finished_at.
|
||||
row.run_id = run_id
|
||||
await session.commit()
|
||||
logger.warning(
|
||||
"Scheduled task-run %s launched durable run %s while occurrence %s holds the active slot; keeping the terminal status",
|
||||
run_record_id,
|
||||
run_id,
|
||||
newer_active,
|
||||
)
|
||||
return True
|
||||
# Timeout/pause can terminalize a claim that recovery
|
||||
# briefly returned to ``queued`` while Gateway admission
|
||||
# was still completing. A returned durable run is the
|
||||
|
||||
@ -257,6 +257,42 @@ async def test_no_verdict_once_is_failed_and_receipt_backfill_preserves_unmet(tm
|
||||
assert (await tasks.get("task", user_id="owner"))["run_count"] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("newer_status", ["running", "queued"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconcile_keeps_terminal_run_when_newer_occurrence_is_active(tmp_path, database_backend, newer_status):
|
||||
async with database(tmp_path, backend=database_backend) as (_sf, tasks, runs):
|
||||
await task(tasks)
|
||||
await runs.create(
|
||||
run_record_id="occ-old",
|
||||
task_id="task",
|
||||
thread_id="thread-old",
|
||||
scheduled_for=NOW,
|
||||
trigger="scheduled",
|
||||
status="failed",
|
||||
)
|
||||
# Seed the real terminal payload: runs.create cannot set error or
|
||||
# finished_at, and the guard must preserve both, not just the status.
|
||||
await runs.update_status("occ-old", status="failed", error="queue timeout", finished_at=NOW)
|
||||
await runs.create(
|
||||
run_record_id="occ-new",
|
||||
task_id="task",
|
||||
thread_id="thread-new",
|
||||
scheduled_for=NOW + timedelta(minutes=1),
|
||||
trigger="manual",
|
||||
status=newer_status,
|
||||
)
|
||||
|
||||
assert await runs.reconcile_launched_run("occ-old", task_id="task", run_id="run-late", started_at=NOW + timedelta(seconds=1)) is True
|
||||
|
||||
rows = {row["id"]: row for row in await runs.list_by_task("task")}
|
||||
assert rows["occ-old"]["status"] == "failed"
|
||||
assert rows["occ-old"]["error"] == "queue timeout"
|
||||
assert datetime.fromisoformat(rows["occ-old"]["finished_at"]) == NOW
|
||||
assert rows["occ-old"]["started_at"] is None
|
||||
assert rows["occ-old"]["run_id"] == "run-late"
|
||||
assert rows["occ-new"]["status"] == newer_status
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_success_resets_unmet_streak_and_old_callback_cannot_pause_new_parent(tmp_path, database_backend):
|
||||
async with database(tmp_path, backend=database_backend) as (sf, tasks, runs):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user