mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 11:20:35 +00:00
* feat(channels): arbitrate IM transports with a per-provider leader lease Every Gateway instance connected every enabled IM platform with the same bot credentials, so channels could not run on more than one instance (Telegram 409 storms, Discord answering twice, scattered per-process state). channels.leader_lease was reserved by #6746; this implements it. - channel_leases table (migration 0041): one row per provider. Acquire is one dialect-native upsert decided against the database clock and bumps a fencing epoch; renew extends only a live lease at the holder's epoch; release sets the expiry to 0 so a peer takes over on its next tick. - app/channels/leadership.py: the holder's local monotonic deadline (send time of the last confirmed renewal + lease_seconds); holds() turns false one renewal interval early, so the transport stops at least grace_seconds before a peer may connect. DB calls are bounded. - ChannelService: start fence in _start_channel (no DB call) behind every start path; fixed-rate supervisor renews/acquires and runs transport starts/stops as per-provider tasks; takeover reloads the provider's saved config; failed starts back off; shutdown releases only leases whose transport stopped. get_status reports leader/leader_here and the holder's transport state on every instance. - Notification worker leaves rows of providers another instance holds. - Gate: the enabled=false refusal now recommends auto/true; an explicit true with database.backend memory is refused when a channel is enabled. - Docs, Helm, deployment guides, AGENTS.md and CHANGELOG updated. * docs(changelog): link the channel leader lease entry to #6843 * fix(channels): see peer-held leases for providers absent from local config Review on #6843: a provider connected from the web UI on a peer never enters this instance's channel config, so the notification worker here still claimed its rows (and parked them for 900s) and GET /api/channels/ reported no leader for it. Every supervision round, and startup, now also reads the whole lease table once (bounded), and notification exclusions and status use those observations: a provider whose live lease a peer holds is left to it even without local config. Running that read concurrently with renewals exposed that, through SQLAlchemy's aiosqlite adapter, an UPDATE ... RETURNING racing a read on another connection can hand back one of its own bind values instead of the column (stored rows stay correct; plain sqlite3 and raw aiosqlite do not show it). A lease must never take such a row for a confirmed renewal, so acquire/renew/release now decide by affected-row count and acquire reads its epoch back in the same transaction. --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
161 lines
6.9 KiB
Python
161 lines
6.9 KiB
Python
"""Migration 0042: the shared ``channel_leases`` table (per-provider IM transport leader lease)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
import sqlalchemy as sa
|
|
from alembic import command
|
|
from alembic.autogenerate import compare_metadata
|
|
from alembic.migration import MigrationContext
|
|
from alembic.script import ScriptDirectory
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
from support.postgres import asyncpg_test_url
|
|
|
|
from deerflow.persistence import bootstrap
|
|
from deerflow.persistence.base import Base
|
|
from deerflow.persistence.channel_leases import ChannelLeaseRow
|
|
from deerflow.persistence.postgres_schema import build_asyncpg_connect_args
|
|
|
|
REVISION = "0042_channel_leases"
|
|
PREVIOUS = "0041_im_notice_preferences"
|
|
CURRENT_HEAD = "0042_channel_leases"
|
|
TABLE = "channel_leases"
|
|
COLUMNS = {"provider", "owner_id", "epoch", "lease_expires_at", "transport_running", "updated_at"}
|
|
pytestmark = pytest.mark.asyncio
|
|
|
|
|
|
async def test_0042_remains_in_the_single_migration_chain():
|
|
script = ScriptDirectory(str(bootstrap._MIGRATIONS_DIR))
|
|
assert script.get_heads() == [CURRENT_HEAD]
|
|
assert script.get_revision(REVISION).down_revision == PREVIOUS
|
|
# alembic_version.version_num is VARCHAR(32).
|
|
assert len(REVISION) <= 32
|
|
|
|
|
|
def _engine(tmp_path, backend):
|
|
if backend == "sqlite":
|
|
return create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'migration.db'}"), None
|
|
uri = os.environ.get("TEST_POSTGRES_URI")
|
|
if not uri:
|
|
pytest.skip("requires TEST_POSTGRES_URI (real Postgres migration 0042)")
|
|
schema = f"channel_leases_{uuid.uuid4().hex}"
|
|
return create_async_engine(asyncpg_test_url(uri), connect_args=build_asyncpg_connect_args(schema)), schema
|
|
|
|
|
|
async def _shape(engine) -> dict:
|
|
async with engine.connect() as conn:
|
|
|
|
def read(sync):
|
|
inspector = sa.inspect(sync)
|
|
tables = set(inspector.get_table_names())
|
|
if TABLE not in tables:
|
|
return {"tables": tables}
|
|
return {
|
|
"tables": tables,
|
|
"columns": {col["name"]: col for col in inspector.get_columns(TABLE)},
|
|
"pk": inspector.get_pk_constraint(TABLE),
|
|
}
|
|
|
|
return await conn.run_sync(read)
|
|
|
|
|
|
async def _orm_diff(engine) -> list:
|
|
"""Alembic's own drift check, restricted to the table this revision owns."""
|
|
|
|
def only_leases(obj, name, type_, reflected, compare_to):
|
|
if type_ == "table":
|
|
return name == TABLE
|
|
return getattr(getattr(obj, "table", None), "name", TABLE) == TABLE
|
|
|
|
async with engine.connect() as conn:
|
|
|
|
def diff(sync):
|
|
context = MigrationContext.configure(sync, opts={"include_object": only_leases, "compare_type": True, "compare_server_default": True})
|
|
return compare_metadata(context, Base.metadata)
|
|
|
|
return await conn.run_sync(diff)
|
|
|
|
|
|
async def _drop_schema(engine, schema):
|
|
if schema:
|
|
async with engine.begin() as conn:
|
|
await conn.execute(sa.text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE'))
|
|
|
|
|
|
@pytest.mark.parametrize("backend", ["sqlite", "postgres"])
|
|
async def test_0042_creates_the_table_matching_the_orm_and_downgrades_cleanly(tmp_path, backend):
|
|
engine, schema = _engine(tmp_path, backend)
|
|
cfg = bootstrap._get_alembic_config(engine, postgres_schema=schema or "")
|
|
try:
|
|
if schema:
|
|
async with engine.begin() as conn:
|
|
await conn.execute(sa.text(f'CREATE SCHEMA "{schema}"'))
|
|
await asyncio.to_thread(command.upgrade, cfg, PREVIOUS)
|
|
assert TABLE not in (await _shape(engine))["tables"]
|
|
|
|
await asyncio.to_thread(command.upgrade, cfg, REVISION)
|
|
shape = await _shape(engine)
|
|
assert set(shape["columns"]) == COLUMNS
|
|
assert shape["pk"]["constrained_columns"] == ["provider"]
|
|
assert shape["pk"]["name"] in ("pk_channel_leases", None) # SQLite does not report constraint names
|
|
assert all(column["nullable"] is False for column in shape["columns"].values())
|
|
# ``make migrate-rev`` would propose nothing: the revision matches the ORM model.
|
|
assert await _orm_diff(engine) == []
|
|
|
|
async with engine.begin() as conn:
|
|
await conn.execute(sa.insert(ChannelLeaseRow).values(provider="telegram", owner_id="host:worker", epoch=1, lease_expires_at=1_700_000_030.0, transport_running=True, updated_at=1_700_000_000.0))
|
|
with pytest.raises(sa.exc.IntegrityError):
|
|
async with engine.begin() as conn:
|
|
await conn.execute(sa.insert(ChannelLeaseRow).values(provider="telegram", owner_id="other", epoch=1, lease_expires_at=1.0, transport_running=False, updated_at=1.0))
|
|
|
|
await asyncio.to_thread(command.downgrade, cfg, PREVIOUS)
|
|
assert TABLE not in (await _shape(engine))["tables"]
|
|
await asyncio.to_thread(command.upgrade, cfg, REVISION)
|
|
assert TABLE in (await _shape(engine))["tables"]
|
|
finally:
|
|
await _drop_schema(engine, schema)
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.parametrize("backend", ["sqlite", "postgres"])
|
|
async def test_0042_completes_after_create_all_and_downgrades_after_a_manual_drop(tmp_path, backend):
|
|
"""The table already exists (an interrupted earlier attempt): upgrade still lands, and vice versa."""
|
|
engine, schema = _engine(tmp_path, backend)
|
|
cfg = bootstrap._get_alembic_config(engine, postgres_schema=schema or "")
|
|
try:
|
|
if schema:
|
|
async with engine.begin() as conn:
|
|
await conn.execute(sa.text(f'CREATE SCHEMA "{schema}"'))
|
|
await asyncio.to_thread(command.upgrade, cfg, PREVIOUS)
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all, tables=[ChannelLeaseRow.__table__])
|
|
|
|
await asyncio.to_thread(command.upgrade, cfg, REVISION)
|
|
async with engine.connect() as conn:
|
|
assert (await conn.execute(sa.text("SELECT version_num FROM alembic_version"))).scalar_one() == REVISION
|
|
|
|
async with engine.begin() as conn:
|
|
await conn.execute(sa.text(f"DROP TABLE {TABLE}"))
|
|
await asyncio.to_thread(command.downgrade, cfg, PREVIOUS)
|
|
async with engine.connect() as conn:
|
|
assert (await conn.execute(sa.text("SELECT version_num FROM alembic_version"))).scalar_one() == PREVIOUS
|
|
finally:
|
|
await _drop_schema(engine, schema)
|
|
await engine.dispose()
|
|
|
|
|
|
async def test_bootstrap_provisions_the_table_on_an_empty_database(tmp_path):
|
|
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'bootstrap.db'}")
|
|
try:
|
|
await bootstrap.bootstrap_schema(engine, backend="sqlite")
|
|
await bootstrap.bootstrap_schema(engine, backend="sqlite")
|
|
assert set((await _shape(engine))["columns"]) == COLUMNS
|
|
async with engine.connect() as conn:
|
|
assert (await conn.execute(sa.text("SELECT version_num FROM alembic_version"))).scalar_one() == CURRENT_HEAD
|
|
finally:
|
|
await engine.dispose()
|