mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-16 08:20:34 +00:00
fix(gateway): serialize the readiness database probe (#8277)
* fix(gateway): serialize the readiness database probe
/health/ready is public, and the module promises that connection-opening
probes run behind a per-kind gate. The ORM database probe never did: it
checked a connection out of the repositories' own engine pool on every
request, so concurrent probe traffic could hold the whole pool (15
connections by default) and leave request handlers waiting.
Run it behind _probe_gate("database") like the checkpointer, stream
bridge and provisioner probes. The gate is per kind, so one request's
probes still run concurrently.
* test(gateway): assert separate probe gates by rendezvous, not wall clock
Each probe now waits until the other has entered its gate, so a shared
gate fails through the readiness deadline instead of a timing margin.
This commit is contained in:
parent
c5d072af91
commit
54dc5024a6
@ -33,8 +33,8 @@ process-local backend (``memory``) has nothing to probe and reports
|
||||
unreachable. Connection-opening probes are serialized behind strict
|
||||
per-process gates, one per probe kind: the route is public through the
|
||||
``/health`` auth prefix, so unlimited concurrent requests must never translate
|
||||
into unlimited new PostgreSQL connections, Redis pool growth or a connection
|
||||
flood at the provisioner.
|
||||
into unlimited new PostgreSQL connections, an ORM or Redis pool held by probe
|
||||
traffic or a connection flood at the provisioner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@ -123,19 +123,27 @@ PROBE_UNREACHABLE = "unreachable"
|
||||
|
||||
|
||||
async def check_database_health() -> str:
|
||||
"""Probe the persistence engine; return one of the PROBE_* values."""
|
||||
"""Probe the persistence engine; return one of the PROBE_* values.
|
||||
|
||||
The probe checks a connection out of the application's own engine pool,
|
||||
the one every repository uses. Probes are serialized so unauthenticated
|
||||
probe traffic holds at most one of those connections per process instead
|
||||
of the whole pool, which would leave request handlers waiting for
|
||||
database connections.
|
||||
"""
|
||||
engine = get_engine()
|
||||
if engine is None:
|
||||
# backend=memory, or the engine has not been initialized yet: there is
|
||||
# no database to probe.
|
||||
return PROBE_NOT_CONFIGURED
|
||||
try:
|
||||
async with asyncio.timeout(_PROBE_TIMEOUT_SECONDS):
|
||||
async with engine.connect() as connection:
|
||||
await connection.execute(text("SELECT 1"))
|
||||
except Exception:
|
||||
logger.warning("Readiness database probe failed", exc_info=True)
|
||||
return PROBE_UNREACHABLE
|
||||
async with _probe_gate("database"):
|
||||
try:
|
||||
async with asyncio.timeout(_PROBE_TIMEOUT_SECONDS):
|
||||
async with engine.connect() as connection:
|
||||
await connection.execute(text("SELECT 1"))
|
||||
except Exception:
|
||||
logger.warning("Readiness database probe failed", exc_info=True)
|
||||
return PROBE_UNREACHABLE
|
||||
return PROBE_OK
|
||||
|
||||
|
||||
|
||||
@ -112,6 +112,124 @@ async def test_check_database_health_unreachable(monkeypatch):
|
||||
assert await check_database_health() == PROBE_UNREACHABLE
|
||||
|
||||
|
||||
class _TrackingEngine:
|
||||
"""Fake engine that records how many probe connections are open at once."""
|
||||
|
||||
def __init__(self, *, delay: float = 0.02, hang: bool = False) -> None:
|
||||
self._delay = delay
|
||||
self._hang = hang
|
||||
self.active = 0
|
||||
self.max_active = 0
|
||||
|
||||
def connect(self):
|
||||
@asynccontextmanager
|
||||
async def _connect():
|
||||
self.active += 1
|
||||
self.max_active = max(self.max_active, self.active)
|
||||
try:
|
||||
await asyncio.sleep(30 if self._hang else self._delay)
|
||||
yield _FakeConnection()
|
||||
finally:
|
||||
self.active -= 1
|
||||
|
||||
return _connect()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_concurrent_database_probes_are_serialized(monkeypatch):
|
||||
"""The ORM probe shares the repositories' pool, so probe traffic may hold one connection at most."""
|
||||
engine = _TrackingEngine()
|
||||
monkeypatch.setattr("app.gateway.health.get_engine", lambda: engine)
|
||||
|
||||
results = await asyncio.gather(*(check_database_health() for _ in range(8)))
|
||||
|
||||
assert results == [PROBE_OK] * 8
|
||||
assert engine.max_active == 1
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_concurrent_readiness_requests_check_out_one_pooled_database_connection(monkeypatch, tmp_path):
|
||||
"""Through a real SQLAlchemy pool, concurrent requests never check out more than one connection."""
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'app.db'}")
|
||||
checked_out = 0
|
||||
max_checked_out = 0
|
||||
|
||||
@event.listens_for(engine.sync_engine, "checkout")
|
||||
def _checkout(*args):
|
||||
nonlocal checked_out, max_checked_out
|
||||
checked_out += 1
|
||||
max_checked_out = max(max_checked_out, checked_out)
|
||||
|
||||
@event.listens_for(engine.sync_engine, "checkin")
|
||||
def _checkin(*args):
|
||||
nonlocal checked_out
|
||||
checked_out -= 1
|
||||
|
||||
monkeypatch.setattr("app.gateway.health.get_engine", lambda: engine)
|
||||
try:
|
||||
results = await asyncio.gather(*(readiness_payload(CheckpointerConfig(type="memory"), stream_bridge=MemoryStreamBridge()) for _ in range(50)))
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
assert {(status_code, payload["database"]) for status_code, payload in results} == {(200, PROBE_OK)}
|
||||
assert max_checked_out == 1
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_database_and_checkpointer_probes_use_separate_gates(monkeypatch, tmp_path):
|
||||
"""One request's database and checkpointer probes still run side by side."""
|
||||
database_started = asyncio.Event()
|
||||
checkpointer_started = asyncio.Event()
|
||||
|
||||
class _RendezvousEngine:
|
||||
def connect(self):
|
||||
@asynccontextmanager
|
||||
async def _connect():
|
||||
database_started.set()
|
||||
await checkpointer_started.wait()
|
||||
yield _FakeConnection()
|
||||
|
||||
return _connect()
|
||||
|
||||
async def _rendezvous_sqlite(conn_string: str | None) -> str:
|
||||
checkpointer_started.set()
|
||||
await database_started.wait()
|
||||
return PROBE_OK
|
||||
|
||||
# Each probe waits until the other has entered its gate. With one shared
|
||||
# gate neither can, so the deadline reports both probes unreachable.
|
||||
monkeypatch.setattr(health_module, "_READINESS_DEADLINE_SECONDS", 0.5)
|
||||
monkeypatch.setattr("app.gateway.health.get_engine", lambda: _RendezvousEngine())
|
||||
monkeypatch.setattr(health_module, "_probe_sqlite_backend", _rendezvous_sqlite)
|
||||
|
||||
status_code, payload = await readiness_payload(CheckpointerConfig(type="sqlite", connection_string=str(tmp_path / "store.db")), stream_bridge=MemoryStreamBridge())
|
||||
|
||||
assert status_code == 200
|
||||
assert payload["database"] == PROBE_OK
|
||||
assert payload["checkpointer"] == PROBE_OK
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_database_probe_cancelled_by_the_deadline_releases_its_gate(monkeypatch):
|
||||
"""A probe the endpoint deadline cancels must not leave later probes waiting on its gate."""
|
||||
monkeypatch.setattr(health_module, "_READINESS_DEADLINE_SECONDS", 0.05)
|
||||
monkeypatch.setattr("app.gateway.health.get_engine", lambda: _TrackingEngine(hang=True))
|
||||
|
||||
status_code, payload = await readiness_payload(CheckpointerConfig(type="memory"), stream_bridge=MemoryStreamBridge())
|
||||
|
||||
assert status_code == 503
|
||||
assert payload["database"] == PROBE_UNREACHABLE
|
||||
|
||||
monkeypatch.setattr("app.gateway.health.get_engine", lambda: _FakeEngine())
|
||||
status_code, payload = await readiness_payload(CheckpointerConfig(type="memory"), stream_bridge=MemoryStreamBridge())
|
||||
|
||||
assert status_code == 200
|
||||
assert payload["database"] == PROBE_OK
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_readiness_payload_ready_when_database_ok_and_memory_backend(monkeypatch):
|
||||
monkeypatch.setattr("app.gateway.health.get_engine", lambda: _FakeEngine())
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user