deer-flow/backend/tests/test_terminal_store_contract.py
Hyeonsang Cho 085b52b5f3
fix(gateway): match conversation reference URLs on the addressed host (#8216)
* fix(gateway): match conversation reference URLs on the addressed host

A conversation URL reference was accepted only when its (scheme,
netloc) equalled Gateway's request.url. Behind TLS that URL is often
http: Docker Compose and Helm run uvicorn without FORWARDED_ALLOW_IPS,
so nginx's X-Forwarded-Proto from another container is ignored, and
nginx.local.conf resends its own $scheme. The browser's https chat link
then got a 422, and so did an explicit :443 or a differently cased host.

Resolve the run request's origin with _request_origin(), as the CSRF
check does (Forwarded / X-Forwarded-Host / Host), normalize the
reference's origin the same way, and compare host and port only. The
scheme cannot tell two deployments on the same host apart, and URLs
remain owner-bound local selectors that are never fetched.

* test(gateway): cover forwarded port, IPv6 and fragment for conversation URLs

Name X-Forwarded-Port in the API.md reference contract: it is appended
when the forwarded host has no port, so a stray value causes a 422.

* test(runtime): synchronize PostgreSQL lease expiration with row lock

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
2026-11-14 22:17:14 +08:00

197 lines
9.9 KiB
Python

"""Execute the terminal/lease contract against memory and real SQL backends."""
import os
import uuid
from datetime import UTC, datetime, timedelta
import pytest
from sqlalchemy import text
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from support.postgres import asyncpg_test_url
from deerflow.persistence.run import RunRepository
from deerflow.persistence.run.model import RunChangeClockRow, RunRow
from deerflow.runtime.runs.store.memory import MemoryRunStore
@pytest.fixture(params=["memory", "sqlite", "postgresql"])
async def store(request, tmp_path):
if request.param == "memory":
yield MemoryRunStore()
return
schema = None
if request.param == "postgresql":
uri = os.getenv("TEST_POSTGRES_URI")
if not uri:
pytest.skip("set TEST_POSTGRES_URI for real PostgreSQL terminal/lease contracts")
schema = f"r2_{uuid.uuid4().hex}"
engine = create_async_engine(asyncpg_test_url(uri), connect_args={"server_settings": {"search_path": schema}})
else:
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'r2.db'}")
try:
async with engine.begin() as conn:
if schema:
await conn.execute(text(f'CREATE SCHEMA "{schema}"'))
await conn.run_sync(lambda c: RunRow.metadata.create_all(c, tables=[RunRow.__table__, RunChangeClockRow.__table__]))
yield RunRepository(async_sessionmaker(engine, expire_on_commit=False))
finally:
if schema:
async with engine.begin() as conn:
await conn.execute(text(f'DROP SCHEMA "{schema}" CASCADE'))
await engine.dispose()
async def put(store, *, expired=False):
expiry = (datetime.now(UTC) + timedelta(seconds=-1 if expired else 30)).isoformat()
await store.put("run", thread_id="thread", status="running", owner_worker_id="owner", lease_expires_at=expiry)
return expiry
@pytest.mark.anyio
async def test_expired_lease_cannot_be_resurrected(store):
expired = await put(store, expired=True)
renewal = await store.renew_lease("run", owner_worker_id="owner", lease_expires_at=(datetime.now(UTC) + timedelta(seconds=60)).isoformat())
assert not renewal.renewed
assert datetime.fromisoformat((await store.get("run"))["lease_expires_at"]) == datetime.fromisoformat(expired)
@pytest.mark.anyio
@pytest.mark.parametrize("expired,owner", [(True, "owner"), (False, "peer")])
async def test_terminal_cas_requires_live_owner(store, expired, owner):
await put(store, expired=expired)
result = await store.finalize_if_not_cancelled("run", status="success", owner_worker_id=owner)
assert not result.finalized
assert (await store.get("run"))["status"] == "running"
@pytest.mark.anyio
async def test_cancel_wins_and_metadata_cannot_bypass_cas(store):
await put(store)
assert await store.request_cancel("run", action="rollback") == "rollback"
result = await store.finalize_if_not_cancelled("run", status="success", owner_worker_id="owner")
assert not result.finalized
assert result.cancel_action == "rollback"
assert await store.update_run_completion("run", status="success", require_terminal=True, owner_worker_id="owner", total_tokens=5) is False
assert (await store.get("run"))["status"] == "running"
@pytest.mark.anyio
async def test_confirmed_terminal_allows_only_matching_metadata(store):
await put(store)
assert (await store.finalize_if_not_cancelled("run", status="success", owner_worker_id="owner")).finalized
assert await store.update_run_completion("run", status="error", require_terminal=True, owner_worker_id="owner") is False
assert await store.update_run_completion("run", status="success", require_terminal=True, owner_worker_id="peer") is False
assert await store.update_run_completion("run", status="success", require_terminal=True, owner_worker_id="owner", total_tokens=5) is True
assert (await store.get("run"))["total_tokens"] == 5
@pytest.mark.anyio
async def test_terminal_metadata_needs_matching_owner_and_outcome_but_not_live_lease(store):
expired = (datetime.now(UTC) - timedelta(seconds=1)).isoformat()
await store.put("run", thread_id="thread", status="success", owner_worker_id="owner", lease_expires_at=expired)
assert await store.update_run_completion("run", status="success", require_terminal=True, owner_worker_id="owner", total_tokens=5)
assert await store.update_run_completion("run", status="success", require_terminal=True, owner_worker_id="peer", total_tokens=10) is False
assert await store.update_run_completion("run", status="error", require_terminal=True, owner_worker_id="owner", total_tokens=10) is False
row = await store.get("run")
assert row["status"] == "success"
assert row["total_tokens"] == 5
assert datetime.fromisoformat(row["lease_expires_at"]) == datetime.fromisoformat(expired)
@pytest.mark.anyio
@pytest.mark.parametrize("method", ["renew_lease", "update_lease", "finalize_if_not_cancelled", "update_status"])
async def test_postgres_checks_database_time_after_row_lock(store, monkeypatch, method):
"""A transaction begun before expiry cannot write after waiting past it."""
if not isinstance(store, RunRepository) or store._sf.kw["bind"].dialect.name != "postgresql":
pytest.skip("requires PostgreSQL row locks and clock_timestamp")
import asyncio
from sqlalchemy import func, select, update
expiry = datetime.now(UTC) + timedelta(hours=1)
await store.put("locked", thread_id="locked-thread", status="running", owner_worker_id="owner", lease_expires_at=expiry.isoformat())
entered = asyncio.get_running_loop().create_future()
lock = store._lock_owned_row
async def observe_transaction_start(session, run_id):
pid, transaction_started_at = (await session.execute(select(func.pg_backend_pid(), func.transaction_timestamp()))).one()
entered.set_result((pid, transaction_started_at))
await lock(session, run_id)
monkeypatch.setattr(store, "_lock_owned_row", observe_transaction_start)
async with asyncio.timeout(10), asyncio.TaskGroup() as tasks:
async with store._sf() as blocker:
await blocker.execute(select(RunRow.run_id).where(RunRow.run_id == "locked").with_for_update())
blocker_pid = await blocker.scalar(select(func.pg_backend_pid()))
if method in ("renew_lease", "update_lease"):
operation = tasks.create_task(getattr(store, method)("locked", owner_worker_id="owner", lease_expires_at=(expiry + timedelta(seconds=30)).isoformat()))
elif method == "update_status":
operation = tasks.create_task(store.update_status("locked", "error", owner_worker_id="owner"))
else:
operation = tasks.create_task(store.finalize_if_not_cancelled("locked", status="success", owner_worker_id="owner"))
pid, transaction_started_at = await entered
while blocker_pid not in await blocker.scalar(select(func.pg_blocking_pids(pid))):
await asyncio.sleep(0.01)
# Expire the lease while the writer is blocked. Its transaction clock
# still sees a live lease, but the database clock has reached expiry.
expired_at = await blocker.scalar(update(RunRow).where(RunRow.run_id == "locked").values(lease_expires_at=func.clock_timestamp()).returning(RunRow.lease_expires_at))
assert transaction_started_at < expired_at < expiry
await blocker.commit()
result = await operation
if method == "renew_lease":
assert not result.renewed
elif method == "finalize_if_not_cancelled":
assert not result.finalized
else:
assert result is False
row = await store.get("locked")
assert row["status"] == "running"
assert datetime.fromisoformat(row["lease_expires_at"]) == expired_at
@pytest.mark.anyio
async def test_durable_cancel_does_not_authorize_expired_terminal_writer(store):
await put(store, expired=True)
assert await store.request_cancel("run", action="rollback") == "rollback"
result = await store.finalize_if_not_cancelled("run", status="success", owner_worker_id="owner")
assert not result.finalized
assert result.cancel_action is None
@pytest.mark.anyio
async def test_renewal_cannot_write_an_already_expired_target(store):
original = await put(store)
renewal = await store.renew_lease("run", owner_worker_id="owner", lease_expires_at=(datetime.now(UTC) - timedelta(seconds=1)).isoformat())
assert not renewal.renewed
assert datetime.fromisoformat((await store.get("run"))["lease_expires_at"]) == datetime.fromisoformat(original)
@pytest.mark.anyio
async def test_strict_completion_cannot_rewrite_terminal_outcome_fields(store):
await put(store)
verdict = {"satisfied": True}
assert (await store.finalize_if_not_cancelled("run", status="success", owner_worker_id="owner", goal_verdict=verdict)).finalized
assert await store.update_run_completion("run", status="success", require_terminal=True, owner_worker_id="owner", error="changed", goal_verdict={"satisfied": False}, total_tokens=5)
row = await store.get("run")
assert row.get("error") is None
assert row["goal_verdict"] == verdict
assert row["total_tokens"] == 5
@pytest.mark.anyio
async def test_terminal_cancel_row_does_not_report_active_cancellation_win(store):
await put(store)
assert await store.request_cancel("run", action="interrupt") == "interrupt"
assert await store.update_status("run", "interrupted", owner_worker_id="owner")
result = await store.finalize_if_not_cancelled("run", status="success", owner_worker_id="owner")
assert not result.finalized
assert result.cancel_action is None
@pytest.mark.anyio
async def test_owned_status_success_cannot_bypass_durable_cancel(store):
await put(store)
assert await store.request_cancel("run", action="interrupt") == "interrupt"
assert await store.update_status("run", "success", owner_worker_id="owner") is False
assert (await store.get("run"))["status"] == "running"