deer-flow/backend/tests/test_stale_active_renewal.py

"""Lease renewals that predate rollback admission need a bounded second check."""

import asyncio
from contextlib import suppress
from datetime import UTC, datetime, timedelta

import pytest
from test_terminal_store_contract import store as store

from deerflow.config.run_ownership_config import RunOwnershipConfig
from deerflow.runtime import RunManager, RunStatus


def manager_for(run_store):
    return RunManager(
        store=run_store,
        worker_id="owner",
        run_ownership_config=RunOwnershipConfig(heartbeat_enabled=True, lease_seconds=30, grace_seconds=0),
    )


async def running_record(manager):
    record = await manager.create_or_reject("thread")
    await manager.set_status(record.run_id, RunStatus.running)
    record.task = asyncio.create_task(asyncio.Event().wait())
    return record


async def finish(manager, record, *tasks):
    for task in tasks:
        if not task.done():
            task.cancel()
    for task in tasks:
        with suppress(asyncio.CancelledError):
            await task
    if not record.task.done():
        record.task.cancel()
    with suppress(asyncio.CancelledError):
        await record.task
    await manager.stop_heartbeat(timeout=0)


@pytest.mark.anyio
async def test_pre_rollback_renewal_retries_after_interrupted_ack(store):
    manager = manager_for(store)
    record = await running_record(manager)
    entered, release = asyncio.Event(), asyncio.Event()
    calls = []
    original_renew = store.renew_lease

    async def hold_first_active_renewal(run_id, **kwargs):
        calls.append(kwargs.get("allow_interrupted", False))
        if len(calls) == 1:
            entered.set()
            await release.wait()
        return await original_renew(run_id, **kwargs)

    store.renew_lease = hold_first_active_renewal
    renewal = asyncio.create_task(manager._renew_leases())
    try:
        await asyncio.wait_for(entered.wait(), 1)
        replacement = await manager.create_or_reject("thread", multitask_strategy="rollback")
        assert replacement.status == RunStatus.pending
        assert record.terminal_commit_proof is not None
        assert record.terminal_commit_proof.status == RunStatus.interrupted
        assert record.rollback_finalization_pending
        old_expiry = record.lease_expires_at

        release.set()
        await asyncio.wait_for(renewal, 2)

        assert calls == [False, True]
        assert not record.ownership_lost
        assert record.status == RunStatus.interrupted
        assert record.rollback_finalization_pending
        assert datetime.fromisoformat(record.lease_expires_at) > datetime.fromisoformat(old_expiry)
        row = await store.get(record.run_id)
        assert row["status"] == "interrupted"
        assert row["owner_worker_id"] == "owner"
        assert datetime.fromisoformat(row["lease_expires_at"]) > datetime.now(UTC)
    finally:
        release.set()
        await finish(manager, record, renewal)


@pytest.mark.anyio
async def test_pre_rollback_renewal_waits_for_inflight_interrupted_ack(store):
    manager = manager_for(store)
    record = await running_record(manager)
    renewal_entered, release_renewal = asyncio.Event(), asyncio.Event()
    admission_committed, release_admission_ack = asyncio.Event(), asyncio.Event()
    calls = []
    original_renew = store.renew_lease
    original_admission = store.create_thread_operation_atomic

    async def hold_first_active_renewal(run_id, **kwargs):
        calls.append(kwargs.get("allow_interrupted", False))
        if len(calls) == 1:
            renewal_entered.set()
            await release_renewal.wait()
        return await original_renew(run_id, **kwargs)

    async def hold_committed_admission_ack(*args, **kwargs):
        result = await original_admission(*args, **kwargs)
        admission_committed.set()
        await release_admission_ack.wait()
        return result

    store.renew_lease = hold_first_active_renewal
    store.create_thread_operation_atomic = hold_committed_admission_ack
    renewal = asyncio.create_task(manager._renew_leases())
    admission = None
    try:
        await asyncio.wait_for(renewal_entered.wait(), 1)
        admission = asyncio.create_task(manager.create_or_reject("thread", multitask_strategy="rollback"))
        await asyncio.wait_for(admission_committed.wait(), 1)
        assert record.terminal_commit_proof is None

        release_renewal.set()
        await asyncio.sleep(0)
        assert not renewal.done()
        assert not record.ownership_lost

        release_admission_ack.set()
        replacement = await asyncio.wait_for(admission, 2)
        await asyncio.wait_for(renewal, 2)

        assert replacement.status == RunStatus.pending
        assert calls == [False, True]
        assert record.terminal_commit_proof is not None
        assert record.terminal_commit_proof.status == RunStatus.interrupted
        assert not record.ownership_lost
        assert (await store.get(record.run_id))["status"] == "interrupted"
    finally:
        release_renewal.set()
        release_admission_ack.set()
        await finish(manager, record, renewal, *([admission] if admission is not None else []))


@pytest.mark.anyio
@pytest.mark.parametrize("taken_over", [False, True], ids=["expired", "peer"])
async def test_pre_rollback_retry_still_fences_when_live_owner_was_lost(store, taken_over):
    manager = manager_for(store)
    record = await running_record(manager)
    entered, release = asyncio.Event(), asyncio.Event()
    calls = []
    original_renew = store.renew_lease

    async def lose_owner_before_retry(run_id, **kwargs):
        calls.append(kwargs.get("allow_interrupted", False))
        if len(calls) == 1:
            entered.set()
            await release.wait()
        return await original_renew(run_id, **kwargs)

    store.renew_lease = lose_owner_before_retry
    renewal = asyncio.create_task(manager._renew_leases())
    try:
        await asyncio.wait_for(entered.wait(), 1)
        await manager.create_or_reject("thread", multitask_strategy="rollback")
        await store.put(
            record.run_id,
            thread_id=record.thread_id,
            status="interrupted",
            owner_worker_id="peer" if taken_over else "owner",
            lease_expires_at=(datetime.now(UTC) + (timedelta(seconds=30) if taken_over else -timedelta(seconds=1))).isoformat(),
            error="Cancelled by newer run",
        )

        release.set()
        await asyncio.wait_for(renewal, 2)

        assert calls == [False, True]
        assert record.ownership_lost
    finally:
        release.set()
        await finish(manager, record, renewal)