deer-flow/backend/tests/test_thread_run_idempotency.py

"""HTTP contract tests for idempotent thread-run creation (issue #5257)."""

from __future__ import annotations

import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
from uuid import uuid4

import pytest
from _router_auth_helpers import call_unwrapped, make_authed_test_app
from fastapi import HTTPException
from fastapi.testclient import TestClient

from app.gateway import services
from app.gateway.auth.models import User
from app.gateway.routers import thread_runs
from app.gateway.run_models import RunCreateRequest
from deerflow.config.app_config import AppConfig, reset_app_config, set_app_config
from deerflow.runtime import DisconnectMode, RunManager, RunRecord, RunStatus
from deerflow.runtime.events.store.memory import MemoryRunEventStore
from deerflow.runtime.runs.store.memory import MemoryRunStore


def _user(email: str) -> User:
    return User(email=email, password_hash="x", system_role="user", id=uuid4())


def _run(run_id: str, thread_id: str) -> RunRecord:
    return RunRecord(
        run_id=run_id,
        thread_id=thread_id,
        assistant_id=None,
        status=RunStatus.success,
        on_disconnect=DisconnectMode.continue_,
        error=run_id,
    )


def _make_client(monkeypatch, user: User, admissions: dict[str, RunRecord]) -> TestClient:
    async def fake_start_run(body, thread_id, request, *, idempotency_key=None, require_existing_thread=False):
        del body, request, require_existing_thread
        if idempotency_key is not None and idempotency_key in admissions:
            record = admissions[idempotency_key]
            record.idempotency_reused = True
            return record
        record = _run(f"run-{len(admissions) + 1}", thread_id)
        admissions[idempotency_key or f"unkeyed-{record.run_id}"] = record
        return record

    monkeypatch.setattr(thread_runs, "start_run", fake_start_run)
    app = make_authed_test_app(user_factory=lambda: user)
    app.include_router(thread_runs.router)
    app.state.stream_bridge = MagicMock(stream_exists=AsyncMock(return_value=False))
    app.state.run_manager = MagicMock()
    return TestClient(app)


def test_same_idempotency_key_reuses_thread_run(monkeypatch):
    admissions: dict[str, RunRecord] = {}
    client = _make_client(monkeypatch, _user("alice@example.com"), admissions)
    url = "/api/threads/thread-1/runs"
    headers = {"Idempotency-Key": "send-message-1"}

    first = client.post(url, json={"input": {"messages": []}}, headers=headers)
    retry = client.post(url, json={"input": {"messages": []}}, headers=headers)

    assert first.status_code == 200, first.text
    assert retry.status_code == 200, retry.text
    assert retry.json()["run_id"] == first.json()["run_id"]


def test_same_idempotency_key_reuses_stream_run(monkeypatch):
    admissions: dict[str, RunRecord] = {}
    client = _make_client(monkeypatch, _user("alice@example.com"), admissions)
    url = "/api/threads/thread-1/runs/stream"
    headers = {"Idempotency-Key": "send-message-1"}

    first = client.post(url, json={"input": {"messages": []}}, headers=headers)
    retry = client.post(url, json={"input": {"messages": []}}, headers=headers)

    assert first.status_code == 200, first.text
    assert retry.status_code == 200, retry.text
    assert retry.headers["Content-Location"] == first.headers["Content-Location"]
    assert "event: end" in first.text
    assert "event: gap" not in first.text
    assert "event: gap" in retry.text
    assert "stream_replay_gap" in retry.text
    assert "reload_durable_state" in retry.text
    assert "event: end" not in retry.text


def test_same_idempotency_key_reuses_wait_run(monkeypatch):
    admissions: dict[str, RunRecord] = {}
    client = _make_client(monkeypatch, _user("alice@example.com"), admissions)
    url = "/api/threads/thread-1/runs/wait"
    headers = {"Idempotency-Key": "send-message-1"}

    first = client.post(url, json={"input": {"messages": []}}, headers=headers)
    retry = client.post(url, json={"input": {"messages": []}}, headers=headers)

    assert first.status_code == 200, first.text
    assert retry.status_code == 200, retry.text
    assert retry.json()["error"] == first.json()["error"]


def test_idempotency_key_is_scoped_to_thread(monkeypatch):
    admissions: dict[str, RunRecord] = {}
    client = _make_client(monkeypatch, _user("alice@example.com"), admissions)
    headers = {"Idempotency-Key": "send-message-1"}

    first = client.post("/api/threads/thread-1/runs", json={}, headers=headers)
    second = client.post("/api/threads/thread-2/runs", json={}, headers=headers)

    assert first.status_code == 200, first.text
    assert second.status_code == 200, second.text
    assert second.json()["run_id"] != first.json()["run_id"]


def test_idempotency_key_is_scoped_to_authenticated_user(monkeypatch):
    admissions: dict[str, RunRecord] = {}
    alice = _make_client(monkeypatch, _user("alice@example.com"), admissions)
    bob = _make_client(monkeypatch, _user("bob@example.com"), admissions)
    url = "/api/threads/thread-1/runs"
    headers = {"Idempotency-Key": "send-message-1"}

    first = alice.post(url, json={}, headers=headers)
    second = bob.post(url, json={}, headers=headers)

    assert first.status_code == 200, first.text
    assert second.status_code == 200, second.text
    assert second.json()["run_id"] != first.json()["run_id"]


def test_missing_idempotency_key_keeps_creating_runs(monkeypatch):
    admissions: dict[str, RunRecord] = {}
    client = _make_client(monkeypatch, _user("alice@example.com"), admissions)
    url = "/api/threads/thread-1/runs"

    first = client.post(url, json={})
    second = client.post(url, json={})

    assert first.status_code == 200, first.text
    assert second.status_code == 200, second.text
    assert second.json()["run_id"] != first.json()["run_id"]


def test_different_idempotency_keys_create_different_runs(monkeypatch):
    admissions: dict[str, RunRecord] = {}
    client = _make_client(monkeypatch, _user("alice@example.com"), admissions)
    url = "/api/threads/thread-1/runs"

    first = client.post(url, json={}, headers={"Idempotency-Key": "send-message-1"})
    second = client.post(url, json={}, headers={"Idempotency-Key": "send-message-2"})

    assert first.status_code == 200, first.text
    assert second.status_code == 200, second.text
    assert second.json()["run_id"] != first.json()["run_id"]


def test_blank_idempotency_key_is_rejected(monkeypatch):
    client = _make_client(monkeypatch, _user("alice@example.com"), {})

    response = client.post(
        "/api/threads/thread-1/runs",
        json={},
        headers={"Idempotency-Key": "   "},
    )

    assert response.status_code == 422


def test_oversized_idempotency_key_is_rejected(monkeypatch):
    client = _make_client(monkeypatch, _user("alice@example.com"), {})

    response = client.post(
        "/api/threads/thread-1/runs",
        json={},
        headers={"Idempotency-Key": "x" * 256},
    )

    assert response.status_code == 422


class _LocalBridge:
    supports_cross_process = False

    async def stream_exists(self, run_id):
        del run_id
        return False


class _StaleSnapshot:
    config = {"configurable": {"checkpoint_id": "cp-previous"}}
    values = {"messages": [{"type": "ai", "content": "PREVIOUS_TURN"}]}


def test_wait_reused_store_only_run_does_not_return_stale_checkpoint(monkeypatch):
    """A reused running record has no local task; /wait must not serialize the current checkpoint."""

    async def fake_start_run(body, thread_id, request, *, idempotency_key=None, require_existing_thread=False):
        del body, request, idempotency_key, require_existing_thread
        return RunRecord(
            run_id="run-live",
            thread_id=thread_id,
            assistant_id=None,
            status=RunStatus.running,
            on_disconnect=DisconnectMode.continue_,
            store_only=True,
            idempotency_reused=True,
        )

    async def fake_aget(config):
        del config
        return _StaleSnapshot()

    monkeypatch.setattr(thread_runs, "start_run", fake_start_run)
    monkeypatch.setattr(
        services,
        "build_checkpoint_state_accessor",
        lambda *args, **kwargs: (SimpleNamespace(aget=fake_aget), {}),
    )
    monkeypatch.setattr(thread_runs, "serialize_channel_values_for_api", lambda values: values)

    app = make_authed_test_app(user_factory=lambda: _user("alice@example.com"))
    app.include_router(thread_runs.router)
    app.state.stream_bridge = _LocalBridge()
    app.state.run_manager = MagicMock()

    with TestClient(app) as client:
        response = client.post(
            "/api/threads/thread-1/runs/wait",
            json={"input": {"messages": []}},
            headers={"Idempotency-Key": "send-message-1"},
        )

    assert response.status_code == 200, response.text
    assert response.json() == {"status": "running", "error": None}
    assert "PREVIOUS_TURN" not in response.text


def test_wait_reused_completed_run_does_not_return_later_checkpoint(monkeypatch):
    """A locally cached completed reuse must not serialize a later thread head."""

    async def fake_start_run(body, thread_id, request, *, idempotency_key=None, require_existing_thread=False):
        del body, request, idempotency_key, require_existing_thread
        return RunRecord(
            run_id="run-a",
            thread_id=thread_id,
            assistant_id=None,
            status=RunStatus.success,
            on_disconnect=DisconnectMode.continue_,
            store_only=False,
            idempotency_reused=True,
        )

    async def fake_aget(config):
        del config
        return SimpleNamespace(
            config={"configurable": {"checkpoint_id": "cp-later"}},
            values={"messages": [{"type": "ai", "content": "LATER_RUN_RESULT"}]},
        )

    monkeypatch.setattr(thread_runs, "start_run", fake_start_run)
    monkeypatch.setattr(
        services,
        "build_checkpoint_state_accessor",
        lambda *args, **kwargs: (SimpleNamespace(aget=fake_aget), {}),
    )
    monkeypatch.setattr(thread_runs, "serialize_channel_values_for_api", lambda values: values)

    app = make_authed_test_app(user_factory=lambda: _user("alice@example.com"))
    app.include_router(thread_runs.router)
    app.state.stream_bridge = _LocalBridge()
    app.state.run_manager = MagicMock()

    with TestClient(app) as client:
        response = client.post(
            "/api/threads/thread-1/runs/wait",
            json={"input": {"messages": []}},
            headers={"Idempotency-Key": "send-message-1"},
        )

    assert response.status_code == 200, response.text
    assert response.json() == {"status": "success", "error": None}
    assert "LATER_RUN_RESULT" not in response.text


@pytest.mark.anyio
async def test_wait_original_request_keeps_checkpoint_when_retry_overlaps():
    """An overlapping retry must not suppress the original creating /wait result."""
    from deerflow.runtime.stream_bridge.memory import MemoryStreamBridge

    bridge = MemoryStreamBridge()
    record = RunRecord(
        run_id="run-a",
        thread_id="thread-1",
        assistant_id=None,
        status=RunStatus.running,
        on_disconnect=DisconnectMode.continue_,
        store_only=False,
        idempotency_reused=False,
    )
    record.task = asyncio.create_task(asyncio.Event().wait())
    snapshot = SimpleNamespace(
        config={"configurable": {"checkpoint_id": "cp-a"}},
        values={"messages": [{"type": "ai", "content": "FIRST_RUN_RESULT"}]},
    )
    request = SimpleNamespace(headers={}, is_disconnected=AsyncMock(return_value=False))

    async def fake_start_run(body, thread_id, request, *, idempotency_key=None, require_existing_thread=False):
        del body, thread_id, request, idempotency_key, require_existing_thread
        return record

    async def fake_aget(config):
        del config
        return snapshot

    with (
        patch.object(thread_runs, "start_run", fake_start_run),
        patch.object(thread_runs, "get_stream_bridge", return_value=bridge),
        patch.object(thread_runs, "get_run_manager", return_value=MagicMock()),
        patch.object(
            services,
            "build_checkpoint_state_accessor",
            lambda *args, **kwargs: (SimpleNamespace(aget=fake_aget), {}),
        ),
        patch.object(thread_runs, "serialize_channel_values_for_api", lambda values: values),
    ):
        wait_task = asyncio.create_task(
            call_unwrapped(
                thread_runs.wait_run,
                "thread-1",
                RunCreateRequest(input={"messages": []}),
                request,
            )
        )
        await asyncio.sleep(0.05)
        record.idempotency_reused = True
        record.status = RunStatus.success
        await bridge.publish_end(record.run_id)
        result = await asyncio.wait_for(wait_task, timeout=2)

    record.task.cancel()
    with pytest.raises(asyncio.CancelledError):
        await record.task

    assert result["messages"][0]["content"] == "FIRST_RUN_RESULT"


@pytest.mark.anyio
async def test_wait_peer_refreshes_status_after_owner_completes():
    """A cross-worker reuse must not keep admission-time running after END."""
    from deerflow.runtime.stream_bridge.memory import MemoryStreamBridge

    store = MemoryRunStore()
    owner = RunManager(store=store, worker_id="worker-a")
    peer = RunManager(store=store, worker_id="worker-b")
    bridge = MemoryStreamBridge()
    bridge.supports_cross_process = True
    input_payload = {"messages": [{"role": "user", "content": "hello"}]}
    first = await owner.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:same",
        kwargs={"input": input_payload, "config": None},
    )
    await owner.set_status(first.run_id, RunStatus.running)
    reused = await peer.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:same",
        kwargs={"input": input_payload, "config": None},
    )
    assert reused.run_id == first.run_id
    assert reused.store_only is True
    assert reused.idempotency_reused is True
    assert reused.status == RunStatus.running
    request = SimpleNamespace(headers={}, is_disconnected=AsyncMock(return_value=False))

    async def fake_start_run(body, thread_id, request, *, idempotency_key=None, require_existing_thread=False):
        del body, thread_id, request, idempotency_key, require_existing_thread
        return reused

    with (
        patch.object(thread_runs, "start_run", fake_start_run),
        patch.object(thread_runs, "get_stream_bridge", return_value=bridge),
        patch.object(thread_runs, "get_run_manager", return_value=peer),
        patch.object(
            services,
            "build_checkpoint_state_accessor",
            side_effect=AssertionError("reused wait must not read latest checkpoint"),
        ),
    ):
        wait_task = asyncio.create_task(
            call_unwrapped(
                thread_runs.wait_run,
                "thread-1",
                RunCreateRequest(input=input_payload),
                request,
            )
        )
        await asyncio.sleep(0.05)
        await owner.set_status(first.run_id, RunStatus.success)
        await bridge.publish_end(first.run_id)
        result = await asyncio.wait_for(wait_task, timeout=2)

    assert result == {"status": "success", "error": None}
    assert reused.status == RunStatus.success


def test_scope_http_run_idempotency_key_ignores_header_default():
    """Direct handler calls pass FastAPI's Header() object, not None."""
    from fastapi.params import Header as HeaderParam

    request = SimpleNamespace(state=SimpleNamespace(user=None))
    assert thread_runs._scope_http_run_idempotency_key(request, "thread-1", HeaderParam(default=None)) is None
    assert thread_runs._scope_http_run_idempotency_key(request, "thread-1", None) is None


def test_internal_caller_key_is_scoped_by_owner_not_thread():
    """The IM channel manager may route a redelivered message to a different
    thread after ``/new`` rebound the conversation; its key must still meet the
    run admitted for the first delivery."""
    from app.gateway.internal_auth import INTERNAL_OWNER_USER_ID_HEADER_NAME, INTERNAL_SYSTEM_ROLE

    internal = SimpleNamespace(state=SimpleNamespace(user=SimpleNamespace(id="default", system_role=INTERNAL_SYSTEM_ROLE)), headers={INTERNAL_OWNER_USER_ID_HEADER_NAME: "alice"})
    other_owner = SimpleNamespace(state=internal.state, headers={INTERNAL_OWNER_USER_ID_HEADER_NAME: "bob"})
    browser = SimpleNamespace(state=SimpleNamespace(user=SimpleNamespace(id="alice", system_role="user")), headers={})

    scope = thread_runs._scope_http_run_idempotency_key
    assert scope(internal, "thread-1", "message-1") == scope(internal, "thread-2", "message-1")
    assert scope(internal, "thread-1", "message-1") != scope(other_owner, "thread-1", "message-1")
    assert scope(browser, "thread-1", "message-1") != scope(browser, "thread-2", "message-1")
    assert scope(internal, "thread-1", "message-1") != scope(browser, "thread-1", "message-1")


def test_stream_reused_store_only_running_run_returns_409(monkeypatch):
    """A reused running record on a process-local bridge must not hang on an empty stream."""

    async def fake_start_run(body, thread_id, request, *, idempotency_key=None, require_existing_thread=False):
        del body, request, idempotency_key, require_existing_thread
        return RunRecord(
            run_id="run-live",
            thread_id=thread_id,
            assistant_id=None,
            status=RunStatus.running,
            on_disconnect=DisconnectMode.continue_,
            store_only=True,
            idempotency_reused=True,
        )

    monkeypatch.setattr(thread_runs, "start_run", fake_start_run)

    app = make_authed_test_app(user_factory=lambda: _user("alice@example.com"))
    app.include_router(thread_runs.router)
    app.state.stream_bridge = _LocalBridge()
    app.state.run_manager = MagicMock()

    with TestClient(app) as client:
        response = client.post(
            "/api/threads/thread-1/runs/stream",
            json={"input": {"messages": []}},
            headers={"Idempotency-Key": "send-message-1"},
        )

    assert response.status_code == 409, response.text
    assert "not active on this worker" in response.json()["detail"]


@pytest.mark.anyio
async def test_sse_consumer_reused_terminal_missing_stream_yields_gap():
    from app.gateway.services import sse_consumer

    record = RunRecord(
        run_id="run-done",
        thread_id="thread-1",
        assistant_id=None,
        status=RunStatus.success,
        on_disconnect=DisconnectMode.continue_,
        store_only=True,
        idempotency_reused=True,
    )
    request = SimpleNamespace(headers={}, is_disconnected=AsyncMock(return_value=False))

    frames = [
        frame
        async for frame in sse_consumer(
            _LocalBridge(),
            record,
            request,
            MagicMock(),
            emit_gap_on_missing_stream=True,
        )
    ]

    assert len(frames) == 1
    assert frames[0].startswith("event: gap\n")
    assert "stream_replay_gap" in frames[0]
    assert "reload_durable_state" in frames[0]
    assert "event: end" not in frames[0]


@pytest.mark.anyio
async def test_sse_consumer_observer_join_keeps_end_after_sticky_reuse_flag():
    """Observer joins must not inherit create_or_reject's sticky reuse flag."""
    from app.gateway.services import sse_consumer

    record = RunRecord(
        run_id="run-done",
        thread_id="thread-1",
        assistant_id=None,
        status=RunStatus.success,
        on_disconnect=DisconnectMode.continue_,
        idempotency_reused=True,
    )
    request = SimpleNamespace(headers={}, is_disconnected=AsyncMock(return_value=False))

    frames = [frame async for frame in sse_consumer(_LocalBridge(), record, request, MagicMock(), apply_on_disconnect=False)]

    assert len(frames) == 1
    assert frames[0].startswith("event: end\n")
    assert "event: gap" not in frames[0]


@pytest.mark.anyio
async def test_sse_consumer_default_path_keeps_end_after_sticky_reuse_flag():
    """Default sse_consumer, including stateless /api/runs/stream, must not emit gap
    just because create_or_reject left idempotency_reused set, or because
    apply_on_disconnect still defaults to True.
    """
    from app.gateway.services import sse_consumer

    record = RunRecord(
        run_id="run-done",
        thread_id="thread-1",
        assistant_id=None,
        status=RunStatus.success,
        on_disconnect=DisconnectMode.continue_,
        store_only=True,
        idempotency_reused=True,
    )
    request = SimpleNamespace(headers={}, is_disconnected=AsyncMock(return_value=False))

    frames = [frame async for frame in sse_consumer(_LocalBridge(), record, request, MagicMock())]

    assert len(frames) == 1
    assert frames[0].startswith("event: end\n")
    assert "event: gap" not in frames[0]


@pytest.mark.anyio
async def test_sse_consumer_missing_stream_gap_requires_explicit_flag():
    """apply_on_disconnect must not select gap vs end by itself."""
    from app.gateway.services import sse_consumer

    record = RunRecord(
        run_id="run-done",
        thread_id="thread-1",
        assistant_id=None,
        status=RunStatus.success,
        on_disconnect=DisconnectMode.continue_,
        store_only=True,
    )
    request = SimpleNamespace(headers={}, is_disconnected=AsyncMock(return_value=False))

    default_frames = [frame async for frame in sse_consumer(_LocalBridge(), record, request, MagicMock(), apply_on_disconnect=True)]
    gap_frames = [
        frame
        async for frame in sse_consumer(
            _LocalBridge(),
            record,
            request,
            MagicMock(),
            apply_on_disconnect=False,
            emit_gap_on_missing_stream=True,
        )
    ]

    assert default_frames[0].startswith("event: end\n")
    assert gap_frames[0].startswith("event: gap\n")


@pytest.mark.anyio
async def test_observer_join_stays_end_after_real_manager_reuse():
    """Join of a terminal missing stream stays `end` after a later key reuse.

    ``create_or_reject`` sets ``idempotency_reused`` on the cached record that
    ``RunManager.get()`` returns. Observer joins read that same object; the
    missing-stream branch must still follow ``emit_gap_on_missing_stream``,
    not the sticky flag or ``apply_on_disconnect``.
    """
    from app.gateway.services import sse_consumer

    store = MemoryRunStore()
    manager = RunManager(store=store, worker_id="worker-a")
    first = await manager.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:same",
    )
    await manager.set_status(first.run_id, RunStatus.success)
    request = SimpleNamespace(headers={}, is_disconnected=AsyncMock(return_value=False))

    async def _frames(*, apply_on_disconnect: bool = True, emit_gap_on_missing_stream: bool = False):
        record = await manager.get(first.run_id)
        assert record is not None
        return [
            frame
            async for frame in sse_consumer(
                _LocalBridge(),
                record,
                request,
                manager,
                apply_on_disconnect=apply_on_disconnect,
                emit_gap_on_missing_stream=emit_gap_on_missing_stream,
            )
        ]

    before = await _frames(apply_on_disconnect=False)
    assert before[0].startswith("event: end\n")

    reused = await manager.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:same",
    )
    assert reused.run_id == first.run_id
    assert reused.idempotency_reused is True

    after = await _frames(apply_on_disconnect=False)
    assert after[0].startswith("event: end\n")
    assert "event: gap" not in after[0]

    after_default = await _frames()
    assert after_default[0].startswith("event: end\n")
    assert "event: gap" not in after_default[0]

    creating = await _frames(emit_gap_on_missing_stream=True)
    assert creating[0].startswith("event: gap\n")
    assert "event: end" not in creating[0]


def _make_start_run_request(run_manager):
    from langgraph.checkpoint.memory import InMemorySaver
    from langgraph.store.memory import InMemoryStore

    from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore

    store = InMemoryStore()
    return SimpleNamespace(
        headers={},
        state=SimpleNamespace(auth_source=None, user=None),
        app=SimpleNamespace(
            state=SimpleNamespace(
                stream_bridge=SimpleNamespace(),
                run_manager=run_manager,
                checkpointer=InMemorySaver(),
                store=store,
                run_event_store=MemoryRunEventStore(),
                run_events_config=None,
                thread_store=MemoryThreadMetaStore(store),
            )
        ),
    )


@pytest.fixture
def _stub_app_config():
    set_app_config(AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}}))
    yield
    reset_app_config()


@pytest.mark.anyio
async def test_start_run_reuses_store_backed_running_row_without_attaching_worker(_stub_app_config):
    from app.gateway.services import start_run

    input_payload = {"messages": [{"role": "user", "content": "hello"}]}
    store = MemoryRunStore()
    owner = RunManager(store=store, worker_id="worker-a")
    peer = RunManager(store=store, worker_id="worker-b")
    first = await owner.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:same",
        kwargs={"input": input_payload, "config": None},
    )

    attached = False

    async def fake_run_agent(*args, **kwargs):
        del args, kwargs
        nonlocal attached
        attached = True

    with (
        patch("app.gateway.services.resolve_agent_factory", return_value=object()),
        patch("app.gateway.services.run_agent", side_effect=fake_run_agent),
    ):
        record = await start_run(
            RunCreateRequest(input=input_payload),
            "thread-1",
            _make_start_run_request(peer),
            idempotency_key="http-run:same",
        )

    assert record.run_id == first.run_id
    assert record.idempotency_reused is True
    assert record.store_only is True
    assert record.task is None
    assert attached is False


@pytest.mark.anyio
async def test_start_run_rejects_reused_key_with_different_input(_stub_app_config):
    from app.gateway.services import start_run

    store = MemoryRunStore()
    owner = RunManager(store=store, worker_id="worker-a")
    peer = RunManager(store=store, worker_id="worker-b")
    await owner.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:same",
        kwargs={"input": {"messages": [{"role": "user", "content": "summarize"}]}, "config": None},
    )

    with (
        patch("app.gateway.services.resolve_agent_factory", return_value=object()),
        patch("app.gateway.services.run_agent", side_effect=AssertionError("worker must not attach")),
        pytest.raises(HTTPException) as excinfo,
    ):
        await start_run(
            RunCreateRequest(input={"messages": [{"role": "user", "content": "translate"}]}),
            "thread-1",
            _make_start_run_request(peer),
            idempotency_key="http-run:same",
        )

    assert excinfo.value.status_code == 409
    assert "different request" in str(excinfo.value.detail)


@pytest.mark.anyio
@pytest.mark.parametrize(("stable_message_id", "reused"), [(True, True), (False, False)], ids=["keyed-message-id", "random-message-id"])
async def test_channel_redelivery_on_another_instance_reuses_the_admitted_run(_stub_app_config, stable_message_id, reused):
    """The IM manager's keyed request, as a second Gateway instance sees it.

    The human message id is part of the compared input, so only a message id
    derived from the same key lets a redelivery reuse the admitted run; a
    fresh uuid per delivery is a different request (409).
    """
    from app.channels.manager import ChannelManager, _human_input_message
    from app.channels.message_bus import InboundMessage
    from app.gateway.services import start_run

    msg = InboundMessage(channel_name="slack", chat_id="C1", user_id="U1", text="hi", metadata={"event_id": "evt-1", "team_id": "T1"})
    key = f"http-run:{ChannelManager._run_idempotency_key(msg)}"

    def body() -> RunCreateRequest:
        message_id = ChannelManager._human_message_id(msg) if stable_message_id else None
        return RunCreateRequest(input={"messages": [_human_input_message("hi", message_id=message_id)]}, on_disconnect="continue")

    async def fake_run_agent(*args, **kwargs):
        del args, kwargs
        await asyncio.Event().wait()

    store = MemoryRunStore()
    first_instance, second_instance = RunManager(store=store, worker_id="worker-a"), RunManager(store=store, worker_id="worker-b")
    with (
        patch("app.gateway.services.resolve_agent_factory", return_value=object()),
        patch("app.gateway.services.run_agent", side_effect=fake_run_agent),
    ):
        first = await start_run(body(), "thread-1", _make_start_run_request(first_instance), idempotency_key=key)
        try:
            if reused:
                second = await start_run(body(), "thread-1", _make_start_run_request(second_instance), idempotency_key=key)
                assert (second.run_id, second.idempotency_reused) == (first.run_id, True)
            else:
                with pytest.raises(HTTPException) as excinfo:
                    await start_run(body(), "thread-1", _make_start_run_request(second_instance), idempotency_key=key)
                assert excinfo.value.status_code == 409
                assert "different request" in str(excinfo.value.detail)
        finally:
            first.task.cancel()
            await asyncio.gather(first.task, return_exceptions=True)


@pytest.mark.anyio
async def test_key_already_admitted_on_another_thread_is_a_409_not_a_second_run(_stub_app_config):
    """A redelivery routed to the conversation's new thread after ``/new``."""
    from app.gateway.services import start_run

    input_payload = {"messages": [{"role": "user", "content": "hello"}]}
    store = MemoryRunStore()
    first_instance, second_instance = RunManager(store=store, worker_id="worker-a"), RunManager(store=store, worker_id="worker-b")
    await first_instance.create_or_reject("thread-old", user_id=None, idempotency_key="internal-run:same", kwargs={"input": input_payload, "config": None})

    for run_manager in (first_instance, second_instance):  # the local record, then the durable row
        with (
            patch("app.gateway.services.resolve_agent_factory", return_value=object()),
            patch("app.gateway.services.run_agent", side_effect=AssertionError("worker must not attach")),
            pytest.raises(HTTPException) as excinfo,
        ):
            await start_run(RunCreateRequest(input=input_payload), "thread-new", _make_start_run_request(run_manager), idempotency_key="internal-run:same")

        assert excinfo.value.status_code == 409
        assert "Idempotency-Key already used" in str(excinfo.value.detail)
    assert await store.list_by_thread("thread-new") == []


def test_resume_idempotency_fingerprint_is_typed_and_order_independent():
    from app.gateway.services import _run_idempotency_request

    first = _run_idempotency_request(RunCreateRequest(command={"resume": {"enabled": True, "count": 1, "ratio": 1.0}}))
    reordered = _run_idempotency_request(RunCreateRequest(command={"resume": {"ratio": 1.0, "count": 1, "enabled": True}}))

    assert first == reordered
    assert first != _run_idempotency_request(RunCreateRequest(command={"resume": {"enabled": 1, "count": 1, "ratio": 1.0}}))
    assert first != _run_idempotency_request(RunCreateRequest(command={"resume": {"enabled": True, "count": 1.0, "ratio": 1.0}}))
    assert first != _run_idempotency_request(RunCreateRequest(input={"command": {"resume": {"enabled": True, "count": 1, "ratio": 1.0}}}))


@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")])
def test_resume_idempotency_fingerprint_rejects_non_finite_numbers(value):
    from app.gateway.services import _run_idempotency_request

    with pytest.raises(HTTPException) as excinfo:
        _run_idempotency_request(RunCreateRequest(command={"resume": value}))

    assert excinfo.value.status_code == 422
    assert "strict JSON" in str(excinfo.value.detail)


@pytest.mark.anyio
@pytest.mark.parametrize("idempotency_key", [None, "http-run:same"])
async def test_start_run_rejects_invalid_resume_before_agent_assembly(_stub_app_config, idempotency_key):
    from app.gateway.services import start_run

    factory = MagicMock(side_effect=AssertionError("agent assembly must not run"))
    with (
        patch("app.gateway.services.resolve_agent_factory", factory),
        pytest.raises(HTTPException) as excinfo,
    ):
        await start_run(
            RunCreateRequest(command={"resume": float("nan")}),
            "thread-1",
            _make_start_run_request(RunManager(store=MemoryRunStore())),
            idempotency_key=idempotency_key,
        )

    assert excinfo.value.status_code == 422
    factory.assert_not_called()


def test_run_response_never_contains_internal_idempotency_fingerprint():
    record = _run("run-1", "thread-1")
    record.kwargs = {"input": None}
    record.idempotency_request = {
        "version": 1,
        "kind": "resume",
        "sha256": "a" * 64,
    }

    response = thread_runs._record_to_response(record)

    assert response.kwargs == {"input": None}


def test_run_response_sanitizes_legacy_resume_idempotency_fence():
    record = _run("run-1", "thread-1")
    record.kwargs = {
        "input": ["deerflow-resume-idempotency", 1, "a" * 64],
        "idempotency_request": {
            "version": 1,
            "kind": "resume",
            "sha256": "a" * 64,
        },
    }

    response = thread_runs._record_to_response(record)

    assert response.kwargs == {"input": None}


@pytest.mark.anyio
async def test_start_run_rejects_legacy_reused_key_for_resume(_stub_app_config):
    from app.gateway.services import start_run

    store = MemoryRunStore()
    owner = RunManager(store=store, worker_id="worker-a")
    peer = RunManager(store=store, worker_id="worker-b")
    await owner.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:same",
        kwargs={"input": None, "config": None},
    )

    with (
        patch("app.gateway.services.resolve_agent_factory", return_value=object()),
        patch("app.gateway.services.run_agent", side_effect=AssertionError("worker must not attach")),
        pytest.raises(HTTPException) as excinfo,
    ):
        await start_run(
            RunCreateRequest(command={"resume": {"answer": "approve"}}),
            "thread-1",
            _make_start_run_request(peer),
            idempotency_key="http-run:same",
        )

    assert excinfo.value.status_code == 409
    assert "different request" in str(excinfo.value.detail)


@pytest.mark.anyio
async def test_start_run_reuses_matching_legacy_writer_resume_fingerprint(_stub_app_config):
    from app.gateway.services import _run_idempotency_request, start_run

    approved = RunCreateRequest(command={"resume": {"answer": "approve"}})
    denied = RunCreateRequest(command={"resume": {"answer": "deny"}})
    store = MemoryRunStore()
    owner = RunManager(store=store, worker_id="worker-a")
    peer = RunManager(store=store, worker_id="worker-b")
    first = await owner.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:same",
        kwargs={
            "input": ["deerflow-resume-idempotency", 1, "a" * 64],
            "idempotency_request": _run_idempotency_request(approved),
            "config": None,
        },
    )

    with (
        patch("app.gateway.services.resolve_agent_factory", return_value=object()),
        patch("app.gateway.services.run_agent", side_effect=AssertionError("worker must not attach")),
    ):
        reused = await start_run(
            approved,
            "thread-1",
            _make_start_run_request(peer),
            idempotency_key="http-run:same",
        )
        with pytest.raises(HTTPException) as excinfo:
            await start_run(
                denied,
                "thread-1",
                _make_start_run_request(peer),
                idempotency_key="http-run:same",
            )

    assert reused.run_id == first.run_id
    assert reused.idempotency_reused is True
    assert excinfo.value.status_code == 409
    assert "different request" in str(excinfo.value.detail)


@pytest.mark.anyio
async def test_start_run_reuses_only_matching_resume_fingerprint(_stub_app_config):
    from app.gateway.services import _run_idempotency_request, start_run

    approved = RunCreateRequest(command={"resume": {"answer": "approve"}})
    denied = RunCreateRequest(command={"resume": {"answer": "deny"}})
    store = MemoryRunStore()
    owner = RunManager(store=store, worker_id="worker-a")
    peer = RunManager(store=store, worker_id="worker-b")
    first = await owner.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:same",
        idempotency_request=_run_idempotency_request(approved),
        kwargs={"input": None, "config": None},
    )

    with (
        patch("app.gateway.services.resolve_agent_factory", return_value=object()),
        patch("app.gateway.services.run_agent", side_effect=AssertionError("worker must not attach")),
    ):
        reused = await start_run(
            approved,
            "thread-1",
            _make_start_run_request(peer),
            idempotency_key="http-run:same",
        )
        with pytest.raises(HTTPException) as excinfo:
            await start_run(
                denied,
                "thread-1",
                _make_start_run_request(peer),
                idempotency_key="http-run:same",
            )

    assert reused.run_id == first.run_id
    assert reused.idempotency_reused is True
    assert reused.store_only is True
    assert reused.task is None
    assert excinfo.value.status_code == 409
    assert "different request" in str(excinfo.value.detail)


@pytest.mark.anyio
async def test_start_run_persists_resume_fingerprint_without_payload(_stub_app_config):
    from app.gateway.services import _run_idempotency_request, start_run

    request_body = RunCreateRequest(command={"resume": {"answer": "approve"}})
    manager = RunManager(store=MemoryRunStore(), worker_id="worker-a")

    with (
        patch("app.gateway.services.resolve_agent_factory", return_value=object()),
        patch("app.gateway.services.run_agent", new_callable=AsyncMock),
    ):
        record = await start_run(
            request_body,
            "thread-1",
            _make_start_run_request(manager),
            idempotency_key="http-run:same",
        )
        assert record.task is not None
        await record.task

    assert record.idempotency_request == _run_idempotency_request(request_body)
    assert record.idempotency_request["kind"] == "resume"
    assert set(record.idempotency_request) == {"version", "kind", "sha256"}
    assert record.kwargs["input"] is None
    assert "idempotency_request" not in record.kwargs


async def _pre_6499_resume_retry(session_factory, body, idempotency_key):
    """Freeze the null-input resume reuse policy from 02ce9ab2, without mapping the new column.

    This intentionally models only the historical resume case exercised below,
    not an alternative implementation of current Gateway admission.
    """
    import sqlalchemy as sa

    assert body.input is None and body.command.get("resume") is not None
    old_runs = sa.Table(
        "runs",
        sa.MetaData(),
        sa.Column("run_id", sa.String(64), primary_key=True),
        sa.Column("idempotency_key", sa.String(255)),
        sa.Column("assistant_id", sa.String(128)),
        sa.Column("kwargs_json", sa.JSON()),
    )
    async with session_factory() as session:
        row = (await session.execute(sa.select(old_runs).where(old_runs.c.idempotency_key == idempotency_key))).mappings().one()
    stored = row["kwargs_json"]
    # Old start_run compares input, assistant and references, ignoring command.resume.
    if stored.get("input") != body.input or row["assistant_id"] != body.assistant_id or stored.get("conversation_references", []) != list(body.conversation_references or []):
        raise HTTPException(status_code=409, detail="Idempotency-Key already used with a different request")
    return row["run_id"]


@pytest.mark.anyio
@pytest.mark.parametrize("writer_version", ["pre-6499", "current"])
async def test_mixed_version_resume_admission_requires_upgraded_routing(tmp_path, _stub_app_config, writer_version):
    """Read compatibility does not make old workers safe keyed-resume retry targets."""
    from deerflow.persistence.engine import close_engine, get_session_factory, init_engine
    from deerflow.persistence.run import RunRepository

    await init_engine("sqlite", url=f"sqlite+aiosqlite:///{tmp_path / 'mixed-resume.db'}", sqlite_dir=str(tmp_path))
    try:
        session_factory = get_session_factory()
        repo = RunRepository(session_factory)
        owner = RunManager(store=repo, worker_id="writer")
        peer = RunManager(store=repo, worker_id="upgraded-reader")
        approved = RunCreateRequest(command={"resume": {"answer": "approve"}})
        denied = RunCreateRequest(command={"resume": {"answer": "deny"}})
        key = "http-run:mixed-resume"
        with (
            patch("app.gateway.services.resolve_agent_factory", return_value=object()),
            patch("app.gateway.services.run_agent", new_callable=AsyncMock) as run_agent,
        ):
            if writer_version == "current":
                first = await services.start_run(approved, "thread-1", _make_start_run_request(owner), idempotency_key=key)
                assert first.task is not None
                await first.task
                assert run_agent.await_args.kwargs["graph_input"].resume == {"answer": "approve"}
            else:
                # Pre-6499 writers admit null input and persist no resume identity.
                first = await owner.create_or_reject("thread-1", approved.assistant_id, user_id=None, idempotency_key=key, kwargs={"input": None, "config": None})

            persisted = await repo.get(first.run_id, user_id=None)
            assert persisted is not None
            assert bool(persisted["idempotency_request"]) is (writer_version == "current")
            assert persisted["kwargs"]["input"] is None
            assert "idempotency_request" not in persisted["kwargs"]

            # The genuine old-column SQL projection cannot see the private identity:
            # both decisions reuse the same row under the historical input-only policy.
            assert await _pre_6499_resume_retry(session_factory, approved, key) == first.run_id
            assert await _pre_6499_resume_retry(session_factory, denied, key) == first.run_id

            if writer_version == "current":
                retry = await services.start_run(approved, "thread-1", _make_start_run_request(peer), idempotency_key=key)
                assert retry.run_id == first.run_id and retry.idempotency_reused
            else:
                # Even an identical retry is unverifiable for an identity-less legacy row.
                with pytest.raises(HTTPException) as error:
                    await services.start_run(approved, "thread-1", _make_start_run_request(peer), idempotency_key=key)
                assert error.value.status_code == 409

            with pytest.raises(HTTPException) as error:
                await services.start_run(denied, "thread-1", _make_start_run_request(peer), idempotency_key=key)
            assert error.value.status_code == 409
            assert run_agent.await_count == (1 if writer_version == "current" else 0)
            assert len(await repo.list_by_thread("thread-1", user_id=None)) == 1
            assert (await repo.get(first.run_id, user_id=None))["idempotency_request"] == persisted["idempotency_request"]
    finally:
        await close_engine()


@pytest.mark.anyio
@pytest.mark.parametrize("stored_kind", ["input", "resume"])
async def test_start_run_rejects_input_resume_kind_collision(_stub_app_config, stored_kind):
    from app.gateway.services import _run_idempotency_request, start_run

    collision = {"command": {"resume": {"answer": "approve"}}}
    input_request = RunCreateRequest(input=collision)
    resume_request = RunCreateRequest(input=collision, command={"resume": {"answer": "approve"}})
    stored_request, retry_request = (input_request, resume_request) if stored_kind == "input" else (resume_request, input_request)
    store = MemoryRunStore()
    owner = RunManager(store=store, worker_id="worker-a")
    peer = RunManager(store=store, worker_id="worker-b")
    await owner.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:same",
        idempotency_request=_run_idempotency_request(stored_request),
        kwargs={"input": collision, "config": None},
    )

    with (
        patch("app.gateway.services.resolve_agent_factory", return_value=object()),
        patch("app.gateway.services.run_agent", side_effect=AssertionError("worker must not attach")),
        pytest.raises(HTTPException) as excinfo,
    ):
        await start_run(
            retry_request,
            "thread-1",
            _make_start_run_request(peer),
            idempotency_key="http-run:same",
        )

    assert excinfo.value.status_code == 409
    assert "different request" in str(excinfo.value.detail)


@pytest.mark.anyio
@pytest.mark.parametrize("strategy", ["interrupt", "rollback"])
async def test_memory_store_reuses_live_idempotency_key_before_foreign_lease_conflict(strategy):
    from deerflow.config.run_ownership_config import RunOwnershipConfig

    store = MemoryRunStore()
    ownership = RunOwnershipConfig(
        lease_seconds=30,
        grace_seconds=10,
        heartbeat_enabled=True,
    )
    owner = RunManager(
        store=store,
        worker_id="worker-a",
        run_ownership_config=ownership,
    )
    peer = RunManager(
        store=store,
        worker_id="worker-b",
        run_ownership_config=ownership,
    )
    first = await owner.create_or_reject(
        "thread-1",
        multitask_strategy=strategy,
        idempotency_key="http-run:same",
    )

    reused = await peer.create_or_reject(
        "thread-1",
        multitask_strategy=strategy,
        idempotency_key="http-run:same",
    )

    assert reused.run_id == first.run_id
    assert reused.idempotency_reused is True
    assert reused.store_only is True
    stored = await store.get(first.run_id)
    assert stored is not None
    assert stored["status"] == "pending"
    assert stored["owner_worker_id"] == "worker-a"


@pytest.mark.anyio
async def test_keyed_input_remains_compatible_with_legacy_explicit_atomic_signature():
    from deerflow.runtime import RunIdempotencyUnsupported

    class LegacyAtomicMemoryStore(MemoryRunStore):
        async def create_thread_operation_atomic(
            self,
            run_id,
            *,
            thread_id,
            owner_worker_id,
            lease_expires_at,
            operation_kind="run",
            multitask_strategy="reject",
            assistant_id=None,
            user_id=None,
            model_name=None,
            metadata=None,
            kwargs=None,
            created_at=None,
            grace_seconds=10,
            idempotency_key=None,
        ):
            return await super().create_thread_operation_atomic(
                run_id,
                thread_id=thread_id,
                owner_worker_id=owner_worker_id,
                lease_expires_at=lease_expires_at,
                operation_kind=operation_kind,
                multitask_strategy=multitask_strategy,
                assistant_id=assistant_id,
                user_id=user_id,
                model_name=model_name,
                metadata=metadata,
                kwargs=kwargs,
                created_at=created_at,
                grace_seconds=grace_seconds,
                idempotency_key=idempotency_key,
            )

    store = LegacyAtomicMemoryStore()
    manager = RunManager(store=store, worker_id="worker-a")
    peer = RunManager(store=store, worker_id="worker-b")

    record = await manager.create_or_reject(
        "thread-1",
        idempotency_key="http-run:key",
    )

    assert record.idempotency_key == "http-run:key"
    persisted = await store.get(record.run_id)
    assert persisted is not None
    assert persisted["idempotency_key"] == "http-run:key"
    reused = await peer.create_or_reject("thread-1", idempotency_key="http-run:key")
    assert reused.run_id == record.run_id
    assert reused.idempotency_reused is True

    with pytest.raises(RunIdempotencyUnsupported, match="does not support keyed resume"):
        await manager.create_or_reject(
            "thread-2",
            idempotency_key="http-run:resume",
            idempotency_request={
                "version": 1,
                "kind": "resume",
                "sha256": "a" * 64,
            },
        )


@pytest.mark.anyio
async def test_keyed_resume_rejects_ambiguous_kwargs_store_before_admission():
    from deerflow.runtime import RunIdempotencyUnsupported

    class DroppingKwargsStore(MemoryRunStore):
        async def create_thread_operation_atomic(self, run_id, **kwargs):
            kwargs.pop("idempotency_request", None)
            return await super().create_thread_operation_atomic(run_id, **kwargs)

    store = DroppingKwargsStore()
    manager = RunManager(store=store)

    with pytest.raises(RunIdempotencyUnsupported, match="does not support keyed resume"):
        await manager.create_or_reject(
            "thread-1",
            idempotency_key="http-run:resume",
            idempotency_request={
                "version": 1,
                "kind": "resume",
                "sha256": "a" * 64,
            },
        )

    assert await store.list_by_thread("thread-1") == []


@pytest.mark.anyio
async def test_keyed_resume_allows_explicit_kwargs_store_capability():
    class ForwardingKwargsStore(MemoryRunStore):
        supports_idempotency_request_kwargs = True

        async def create_thread_operation_atomic(self, run_id, **kwargs):
            return await super().create_thread_operation_atomic(run_id, **kwargs)

    store = ForwardingKwargsStore()
    manager = RunManager(store=store)
    identity = {
        "version": 1,
        "kind": "resume",
        "sha256": "a" * 64,
    }

    record = await manager.create_or_reject(
        "thread-1",
        idempotency_key="http-run:resume",
        idempotency_request=identity,
    )

    persisted = await store.get(record.run_id)
    assert persisted is not None
    assert persisted["idempotency_request"] == identity


@pytest.mark.anyio
@pytest.mark.parametrize("multitask_strategy", ["reject", "interrupt", "rollback"])
async def test_keyed_input_rejects_ambiguous_kwargs_store_before_admission(multitask_strategy):
    from deerflow.runtime import RunIdempotencyUnsupported

    class DroppingKwargsStore(MemoryRunStore):
        atomic_called = False

        async def create_thread_operation_atomic(self, run_id, **kwargs):
            self.atomic_called = True
            kwargs.pop("idempotency_key", None)
            return await super().create_thread_operation_atomic(run_id, **kwargs)

    store = DroppingKwargsStore()
    manager = RunManager(store=store)

    with pytest.raises(RunIdempotencyUnsupported, match="does not support keyed run"):
        await manager.create_or_reject(
            "thread-1",
            idempotency_key="http-run:input",
            multitask_strategy=multitask_strategy,
        )

    assert store.atomic_called is False
    assert await store.list_by_thread("thread-1") == []
    assert await manager.list_by_thread("thread-1") == []


@pytest.mark.anyio
@pytest.mark.parametrize("with_resume_identity", [False, True])
async def test_explicit_resume_parameter_does_not_imply_key_support(with_resume_identity):
    from deerflow.runtime import RunIdempotencyUnsupported

    class DroppingKeyStore(MemoryRunStore):
        async def create_thread_operation_atomic(self, run_id, *, idempotency_request=None, **kwargs):
            kwargs.pop("idempotency_key", None)
            return await super().create_thread_operation_atomic(run_id, idempotency_request=idempotency_request, **kwargs)

    store = DroppingKeyStore()
    manager = RunManager(store=store)
    identity = {"version": 1, "kind": "resume", "sha256": "a" * 64} if with_resume_identity else None

    with pytest.raises(RunIdempotencyUnsupported, match="does not support keyed run"):
        await manager.create_or_reject("thread-1", idempotency_key="http-run:key", idempotency_request=identity)

    assert await store.list_by_thread("thread-1") == []
    assert await manager.list_by_thread("thread-1") == []


@pytest.mark.anyio
@pytest.mark.parametrize("with_resume_identity", [False, True])
@pytest.mark.parametrize("declaration", ["replaced_parent_method", "child_flag", "mixin_flag"])
async def test_kwargs_store_capability_must_belong_to_implementation_owner(declaration, with_resume_identity):
    from deerflow.runtime import RunIdempotencyUnsupported

    class OptedInParent(MemoryRunStore):
        supports_idempotency_request_kwargs = True

        async def create_thread_operation_atomic(self, run_id, **kwargs):
            return await super().create_thread_operation_atomic(run_id, **kwargs)

    class ReplacingChild(OptedInParent):
        async def create_thread_operation_atomic(self, run_id, **kwargs):
            kwargs.pop("idempotency_key", None)
            kwargs.pop("idempotency_request", None)
            return await super().create_thread_operation_atomic(run_id, **kwargs)

    class UnoptedParent(MemoryRunStore):
        async def create_thread_operation_atomic(self, run_id, **kwargs):
            kwargs.pop("idempotency_key", None)
            kwargs.pop("idempotency_request", None)
            return await super().create_thread_operation_atomic(run_id, **kwargs)

    class FlagOnlyChild(UnoptedParent):
        supports_idempotency_request_kwargs = True

    class FlagMixin:
        supports_idempotency_request_kwargs = True

    class MixinStore(FlagMixin, UnoptedParent):
        async def create_thread_operation_atomic(self, run_id, **kwargs):
            return await super().create_thread_operation_atomic(run_id, **kwargs)

    stores = {"replaced_parent_method": ReplacingChild, "child_flag": FlagOnlyChild, "mixin_flag": MixinStore}
    store = stores[declaration]()
    manager = RunManager(store=store)
    identity = {"version": 1, "kind": "resume", "sha256": "a" * 64} if with_resume_identity else None

    with pytest.raises(RunIdempotencyUnsupported):
        await manager.create_or_reject("thread-1", idempotency_key="http-run:key", idempotency_request=identity)

    assert await store.list_by_thread("thread-1") == []
    assert await manager.list_by_thread("thread-1") == []


@pytest.mark.anyio
@pytest.mark.parametrize("with_resume_identity", [False, True])
async def test_kwargs_store_inherits_capability_with_unchanged_implementation(with_resume_identity):
    class OptedInParent(MemoryRunStore):
        supports_idempotency_request_kwargs = True

        async def create_thread_operation_atomic(self, run_id, **kwargs):
            return await super().create_thread_operation_atomic(run_id, **kwargs)

    class InheritingStore(OptedInParent):
        pass

    store = InheritingStore()
    owner = RunManager(store=store, worker_id="worker-a")
    peer = RunManager(store=store, worker_id="worker-b")
    identity = {"version": 1, "kind": "resume", "sha256": "a" * 64} if with_resume_identity else None

    record = await owner.create_or_reject("thread-1", idempotency_key="http-run:key", idempotency_request=identity)
    reused = await peer.create_or_reject("thread-1", idempotency_key="http-run:key", idempotency_request=identity)

    assert reused.run_id == record.run_id
    assert reused.idempotency_reused is True
    persisted = await store.get(record.run_id)
    assert persisted is not None
    assert persisted["idempotency_key"] == "http-run:key"
    assert persisted["idempotency_request"] == identity
    assert len(await store.list_by_thread("thread-1")) == 1


@pytest.mark.anyio
async def test_start_run_returns_503_for_ambiguous_key_only_store(_stub_app_config):
    class DroppingKeyStore(MemoryRunStore):
        async def create_thread_operation_atomic(self, run_id, **kwargs):
            kwargs.pop("idempotency_key", None)
            return await super().create_thread_operation_atomic(run_id, **kwargs)

    store = DroppingKeyStore()
    manager = RunManager(store=store)

    with (
        patch("app.gateway.services.resolve_agent_factory", return_value=object()),
        patch("app.gateway.services.run_agent", side_effect=AssertionError("worker must not attach")),
        pytest.raises(HTTPException) as excinfo,
    ):
        await services.start_run(
            RunCreateRequest(input={"messages": [{"role": "user", "content": "hello"}]}),
            "thread-1",
            _make_start_run_request(manager),
            idempotency_key="http-run:input",
        )

    assert excinfo.value.status_code == 503
    assert "does not support keyed run" in str(excinfo.value.detail)
    assert await store.list_by_thread("thread-1") == []
    assert await manager.list_by_thread("thread-1") == []


@pytest.mark.anyio
async def test_wait_retry_after_later_run_does_not_return_later_checkpoint(monkeypatch):
    """Complete two runs, then retry the first key: /wait must not return run B."""
    first_input = {"messages": [{"role": "user", "content": "one"}]}
    later_input = {"messages": [{"role": "user", "content": "two"}]}
    store = MemoryRunStore()
    manager = RunManager(store=store, worker_id="worker-a")
    first = await manager.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:first",
        kwargs={"input": first_input, "config": None},
    )
    await manager.set_status(first.run_id, RunStatus.success)
    later = await manager.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:later",
        kwargs={"input": later_input, "config": None},
    )
    await manager.set_status(later.run_id, RunStatus.success)
    reused = await manager.create_or_reject(
        "thread-1",
        user_id=None,
        idempotency_key="http-run:first",
        kwargs={"input": first_input, "config": None},
    )
    assert reused.run_id == first.run_id
    assert reused.idempotency_reused is True
    assert reused.store_only is False
    assert reused.status == RunStatus.success

    async def fake_start_run(body, thread_id, request, *, idempotency_key=None, require_existing_thread=False):
        del body, thread_id, request, idempotency_key, require_existing_thread
        return reused

    async def fake_aget(config):
        del config
        return SimpleNamespace(
            config={"configurable": {"checkpoint_id": "cp-later"}},
            values={"messages": [{"type": "ai", "content": "LATER_RUN_RESULT"}]},
        )

    monkeypatch.setattr(thread_runs, "start_run", fake_start_run)
    monkeypatch.setattr(
        services,
        "build_checkpoint_state_accessor",
        lambda *args, **kwargs: (SimpleNamespace(aget=fake_aget), {}),
    )
    monkeypatch.setattr(thread_runs, "serialize_channel_values_for_api", lambda values: values)

    app = make_authed_test_app(user_factory=lambda: _user("alice@example.com"))
    app.include_router(thread_runs.router)
    app.state.stream_bridge = _LocalBridge()
    app.state.run_manager = manager

    with TestClient(app) as client:
        response = client.post(
            "/api/threads/thread-1/runs/wait",
            json={"input": first_input},
            headers={"Idempotency-Key": "first"},
        )

    assert response.status_code == 200, response.text
    assert response.json() == {"status": "success", "error": None}
    assert "LATER_RUN_RESULT" not in response.text