fix(gateway): serialize the readiness database probe (#8277)

* fix(gateway): serialize the readiness database probe

/health/ready is public, and the module promises that connection-opening
probes run behind a per-kind gate. The ORM database probe never did: it
checked a connection out of the repositories' own engine pool on every
request, so concurrent probe traffic could hold the whole pool (15
connections by default) and leave request handlers waiting.

Run it behind _probe_gate("database") like the checkpointer, stream
bridge and provisioner probes. The gate is per kind, so one request's
probes still run concurrently.

* test(gateway): assert separate probe gates by rendezvous, not wall clock

Each probe now waits until the other has entered its gate, so a shared
gate fails through the readiness deadline instead of a timing margin.
This commit is contained in:
Hyeonsang Cho 2026-11-16 16:35:18 +09:00 committed by GitHub
parent c5d072af91
commit 54dc5024a6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 136 additions and 10 deletions

View File

@ -33,8 +33,8 @@ process-local backend (``memory``) has nothing to probe and reports
unreachable. Connection-opening probes are serialized behind strict
per-process gates, one per probe kind: the route is public through the
``/health`` auth prefix, so unlimited concurrent requests must never translate
into unlimited new PostgreSQL connections, Redis pool growth or a connection
flood at the provisioner.
into unlimited new PostgreSQL connections, an ORM or Redis pool held by probe
traffic or a connection flood at the provisioner.
"""
from __future__ import annotations
@ -123,19 +123,27 @@ PROBE_UNREACHABLE = "unreachable"
async def check_database_health() -> str:
"""Probe the persistence engine; return one of the PROBE_* values."""
"""Probe the persistence engine; return one of the PROBE_* values.
The probe checks a connection out of the application's own engine pool,
the one every repository uses. Probes are serialized so unauthenticated
probe traffic holds at most one of those connections per process instead
of the whole pool, which would leave request handlers waiting for
database connections.
"""
engine = get_engine()
if engine is None:
# backend=memory, or the engine has not been initialized yet: there is
# no database to probe.
return PROBE_NOT_CONFIGURED
try:
async with asyncio.timeout(_PROBE_TIMEOUT_SECONDS):
async with engine.connect() as connection:
await connection.execute(text("SELECT 1"))
except Exception:
logger.warning("Readiness database probe failed", exc_info=True)
return PROBE_UNREACHABLE
async with _probe_gate("database"):
try:
async with asyncio.timeout(_PROBE_TIMEOUT_SECONDS):
async with engine.connect() as connection:
await connection.execute(text("SELECT 1"))
except Exception:
logger.warning("Readiness database probe failed", exc_info=True)
return PROBE_UNREACHABLE
return PROBE_OK

View File

@ -112,6 +112,124 @@ async def test_check_database_health_unreachable(monkeypatch):
assert await check_database_health() == PROBE_UNREACHABLE
class _TrackingEngine:
"""Fake engine that records how many probe connections are open at once."""
def __init__(self, *, delay: float = 0.02, hang: bool = False) -> None:
self._delay = delay
self._hang = hang
self.active = 0
self.max_active = 0
def connect(self):
@asynccontextmanager
async def _connect():
self.active += 1
self.max_active = max(self.max_active, self.active)
try:
await asyncio.sleep(30 if self._hang else self._delay)
yield _FakeConnection()
finally:
self.active -= 1
return _connect()
@pytest.mark.anyio
async def test_concurrent_database_probes_are_serialized(monkeypatch):
"""The ORM probe shares the repositories' pool, so probe traffic may hold one connection at most."""
engine = _TrackingEngine()
monkeypatch.setattr("app.gateway.health.get_engine", lambda: engine)
results = await asyncio.gather(*(check_database_health() for _ in range(8)))
assert results == [PROBE_OK] * 8
assert engine.max_active == 1
@pytest.mark.anyio
async def test_concurrent_readiness_requests_check_out_one_pooled_database_connection(monkeypatch, tmp_path):
"""Through a real SQLAlchemy pool, concurrent requests never check out more than one connection."""
from sqlalchemy import event
from sqlalchemy.ext.asyncio import create_async_engine
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'app.db'}")
checked_out = 0
max_checked_out = 0
@event.listens_for(engine.sync_engine, "checkout")
def _checkout(*args):
nonlocal checked_out, max_checked_out
checked_out += 1
max_checked_out = max(max_checked_out, checked_out)
@event.listens_for(engine.sync_engine, "checkin")
def _checkin(*args):
nonlocal checked_out
checked_out -= 1
monkeypatch.setattr("app.gateway.health.get_engine", lambda: engine)
try:
results = await asyncio.gather(*(readiness_payload(CheckpointerConfig(type="memory"), stream_bridge=MemoryStreamBridge()) for _ in range(50)))
finally:
await engine.dispose()
assert {(status_code, payload["database"]) for status_code, payload in results} == {(200, PROBE_OK)}
assert max_checked_out == 1
@pytest.mark.anyio
async def test_database_and_checkpointer_probes_use_separate_gates(monkeypatch, tmp_path):
"""One request's database and checkpointer probes still run side by side."""
database_started = asyncio.Event()
checkpointer_started = asyncio.Event()
class _RendezvousEngine:
def connect(self):
@asynccontextmanager
async def _connect():
database_started.set()
await checkpointer_started.wait()
yield _FakeConnection()
return _connect()
async def _rendezvous_sqlite(conn_string: str | None) -> str:
checkpointer_started.set()
await database_started.wait()
return PROBE_OK
# Each probe waits until the other has entered its gate. With one shared
# gate neither can, so the deadline reports both probes unreachable.
monkeypatch.setattr(health_module, "_READINESS_DEADLINE_SECONDS", 0.5)
monkeypatch.setattr("app.gateway.health.get_engine", lambda: _RendezvousEngine())
monkeypatch.setattr(health_module, "_probe_sqlite_backend", _rendezvous_sqlite)
status_code, payload = await readiness_payload(CheckpointerConfig(type="sqlite", connection_string=str(tmp_path / "store.db")), stream_bridge=MemoryStreamBridge())
assert status_code == 200
assert payload["database"] == PROBE_OK
assert payload["checkpointer"] == PROBE_OK
@pytest.mark.anyio
async def test_database_probe_cancelled_by_the_deadline_releases_its_gate(monkeypatch):
"""A probe the endpoint deadline cancels must not leave later probes waiting on its gate."""
monkeypatch.setattr(health_module, "_READINESS_DEADLINE_SECONDS", 0.05)
monkeypatch.setattr("app.gateway.health.get_engine", lambda: _TrackingEngine(hang=True))
status_code, payload = await readiness_payload(CheckpointerConfig(type="memory"), stream_bridge=MemoryStreamBridge())
assert status_code == 503
assert payload["database"] == PROBE_UNREACHABLE
monkeypatch.setattr("app.gateway.health.get_engine", lambda: _FakeEngine())
status_code, payload = await readiness_payload(CheckpointerConfig(type="memory"), stream_bridge=MemoryStreamBridge())
assert status_code == 200
assert payload["database"] == PROBE_OK
@pytest.mark.anyio
async def test_readiness_payload_ready_when_database_ok_and_memory_backend(monkeypatch):
monkeypatch.setattr("app.gateway.health.get_engine", lambda: _FakeEngine())