deer-flow/backend/tests/blocking_io/test_channel_leader_lease.py
Nan Gao 8d5f5fd0a8
feat(channels): share encrypted runtime channel config across Gateway instances (#6988)
* feat(channels): share encrypted runtime channel config across Gateway instances

IM bot credentials entered in the web UI lived in each host's plaintext
channels/runtime-config.json, and the router cached the merged config on
app.state for the life of the process. With several Gateway instances a
save or disconnect served by an instance that does not hold the
provider's leader lease never reached the transport holder: it kept the
old credentials until it next started the channel, and a disconnect
could leave the old bot connected.

- New shared table channel_runtime_configs (migration 0044): one row per
  provider with the config block encrypted under DEER_FLOW_CREDENTIALS_KEY,
  a plaintext disconnect tombstone, and a version bumped by every save.
- SqlChannelRuntimeConfigStore replaces the JSON file for sqlite/postgres
  (memory keeps the file); it is synchronous on app_sync_sqlalchemy_url
  because its callers already run in worker threads.
- The router re-reads the store on every request instead of caching the
  merged config, and refuses (503) to save credentials without a key.
- The leader-lease supervisor compares the versions on each renewal tick
  and restarts or stops the holder's transport when a provider's config
  changed; a takeover reads a disconnect instead of reviving cached
  credentials.
- The first start imports an existing runtime-config.json (rows already
  in the table win) and deletes the plaintext file.

* docs(changelog): reference #6988 for the shared runtime channel config

* ci: rerun checks after #6999 fixed the frontend guidance budget on main

* fix(channels): start a held transport that is down when its runtime config changes

When the lease holder's transport was down (a start that failed on the
old credentials), applying a peer-saved config only updated the cached
config, and the transport started on the next lease round: two renewal
ticks instead of the one the docs promise. The apply step now starts it
on the held lease right away, with the lease round's backoff bookkeeping.
2026-10-18 11:10:31 +08:00

146 lines
6.4 KiB
Python

"""Regression anchors: the IM leader lease must not block the event loop.
``ChannelService`` acquires, renews and releases ``channel_leases`` rows on the
Gateway event loop: at startup, on every supervision tick (every
``lease_seconds // 3`` seconds per provider), from readiness requests and at
shutdown. Every statement goes through the async session (aiosqlite /
asyncpg) and every ``wait_for`` bound is a loop timer. The same tick reads the
runtime config versions and re-reads a changed provider's config through the
synchronous runtime config store, both in worker threads. If any of it
regresses onto the loop — a synchronous driver call, a blocking clock or
lock — the strict Blockbuster gate raises ``BlockingError`` here.
The engine and schema are built off the loop: that is fixture work, not the
production path under test.
"""
from __future__ import annotations
import asyncio
from collections.abc import AsyncIterator
from pathlib import Path
import pytest
import pytest_asyncio
import sqlalchemy as sa
from cryptography.fernet import Fernet
from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, create_async_engine
from sqlalchemy.orm import sessionmaker
# Pre-import so lazy imports inside the service are cached no-ops under the gate.
import deerflow.persistence.models # noqa: F401
from app.channels.leadership import ChannelLeadership, RenewOutcome
from app.channels.runtime_config_store import SqlChannelRuntimeConfigStore
from app.channels.service import ChannelService
from app.channels.store import JsonChannelStore
from deerflow import reflection
from deerflow.config.app_config import AppConfig, reset_app_config, set_app_config
from deerflow.config.credentials_key import CredentialsCipher
from deerflow.persistence.base import Base
from deerflow.persistence.channel_leases import ChannelLeaseRow, SqlChannelLeaseRepository
from deerflow.persistence.channel_runtime_configs import ChannelRuntimeConfigRow, SqlChannelRuntimeConfigRepository
pytestmark = pytest.mark.asyncio
def _build_sqlite_engine(path: Path) -> AsyncEngine:
sync_engine = sa.create_engine(f"sqlite:///{path.as_posix()}")
try:
Base.metadata.create_all(sync_engine, tables=[ChannelLeaseRow.__table__])
finally:
sync_engine.dispose()
return create_async_engine(f"sqlite+aiosqlite:///{path.as_posix()}")
@pytest_asyncio.fixture
async def repo(tmp_path) -> AsyncIterator[SqlChannelLeaseRepository]:
engine = await asyncio.to_thread(_build_sqlite_engine, tmp_path / "leases.db")
try:
yield SqlChannelLeaseRepository(async_sessionmaker(engine, expire_on_commit=False))
finally:
await engine.dispose()
class _Transport:
def __init__(self, *, bus: object, config: dict) -> None:
self.config = config
self.is_running = False
async def start(self) -> None:
self.is_running = True
async def stop(self) -> None:
self.is_running = False
async def test_lease_round_trip_does_not_block_loop(repo) -> None:
holder = ChannelLeadership(repo, owner_id="A", lease_seconds=30, grace_seconds=10)
peer = ChannelLeadership(repo, owner_id="B", lease_seconds=30, grace_seconds=10)
assert await holder.acquire("telegram") is True
assert await peer.acquire("telegram") is False # refusal plus the observation read
assert await holder.renew("telegram", transport_running=True) is RenewOutcome.RENEWED
await holder.release("telegram")
assert await peer.acquire("telegram") is True
assert [state.owner_id for state in await repo.list_all()] == ["B"]
async def test_service_supervision_and_shutdown_do_not_block_loop(repo, tmp_path, monkeypatch) -> None:
"""The production caller: startup acquisition, a supervision round, and the releasing shutdown."""
monkeypatch.setattr(reflection, "resolve_class", lambda *_args, **_kwargs: _Transport)
store_path = await asyncio.to_thread(lambda: tmp_path / "store.json")
service = ChannelService(
channels_config={"telegram": {"enabled": True}, "github": {"enabled": True}},
store=JsonChannelStore(store_path),
leadership=ChannelLeadership(repo, owner_id="A", lease_seconds=30, grace_seconds=10),
)
async def noop() -> None:
return None
service.manager.start = noop
service.manager.stop = noop
service._start_supervisor = lambda: None
await service.start()
await service._supervise_once()
assert service.get_status()["channels"]["telegram"]["leader"] == "A"
await service.stop()
assert (await repo.get("telegram")).live is False
def _build_runtime_store(path: Path) -> SqlChannelRuntimeConfigStore:
engine = sa.create_engine(f"sqlite:///{path.as_posix()}")
Base.metadata.create_all(engine, tables=[ChannelRuntimeConfigRow.__table__])
cipher = CredentialsCipher.from_keys([Fernet.generate_key()])
store = SqlChannelRuntimeConfigStore(SqlChannelRuntimeConfigRepository(sessionmaker(engine, expire_on_commit=False)), cipher=cipher)
store.set_provider_config("telegram", {"enabled": True, "bot_token": "t-1"})
return store
async def test_supervision_applies_a_peer_runtime_config_save_without_blocking_loop(repo, tmp_path, monkeypatch) -> None:
monkeypatch.setattr(reflection, "resolve_class", lambda *_args, **_kwargs: _Transport)
set_app_config(AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}, "channel_connections": {"enabled": True, "telegram": {"enabled": True}}}))
runtime_store = await asyncio.to_thread(_build_runtime_store, tmp_path / "runtime.db")
store_path = await asyncio.to_thread(lambda: tmp_path / "store.json")
service = ChannelService(
channels_config={"telegram": {"enabled": True, "bot_token": "t-1"}},
store=JsonChannelStore(store_path),
leadership=ChannelLeadership(repo, owner_id="A", lease_seconds=30, grace_seconds=10),
runtime_config_store=runtime_store,
)
async def noop() -> None:
return None
service.manager.start = noop
service.manager.stop = noop
service._start_supervisor = lambda: None
try:
await service.start()
await asyncio.to_thread(runtime_store.set_provider_config, "telegram", {"enabled": True, "bot_token": "t-2"})
await service._supervise_once()
await asyncio.gather(*list(service._lease_actions.values()))
assert service.get_channel("telegram").config["bot_token"] == "t-2"
await service.stop()
finally:
reset_app_config()