mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 11:20:35 +00:00
* feat(channels): share encrypted runtime channel config across Gateway instances IM bot credentials entered in the web UI lived in each host's plaintext channels/runtime-config.json, and the router cached the merged config on app.state for the life of the process. With several Gateway instances a save or disconnect served by an instance that does not hold the provider's leader lease never reached the transport holder: it kept the old credentials until it next started the channel, and a disconnect could leave the old bot connected. - New shared table channel_runtime_configs (migration 0044): one row per provider with the config block encrypted under DEER_FLOW_CREDENTIALS_KEY, a plaintext disconnect tombstone, and a version bumped by every save. - SqlChannelRuntimeConfigStore replaces the JSON file for sqlite/postgres (memory keeps the file); it is synchronous on app_sync_sqlalchemy_url because its callers already run in worker threads. - The router re-reads the store on every request instead of caching the merged config, and refuses (503) to save credentials without a key. - The leader-lease supervisor compares the versions on each renewal tick and restarts or stops the holder's transport when a provider's config changed; a takeover reads a disconnect instead of reviving cached credentials. - The first start imports an existing runtime-config.json (rows already in the table win) and deletes the plaintext file. * docs(changelog): reference #6988 for the shared runtime channel config * ci: rerun checks after #6999 fixed the frontend guidance budget on main * fix(channels): start a held transport that is down when its runtime config changes When the lease holder's transport was down (a start that failed on the old credentials), applying a peer-saved config only updated the cached config, and the transport started on the next lease round: two renewal ticks instead of the one the docs promise. The apply step now starts it on the held lease right away, with the lease round's backoff bookkeeping.
112 lines
5.4 KiB
Python
112 lines
5.4 KiB
Python
"""Synchronous SQL repository for ``channel_runtime_configs`` (SQLite or PostgreSQL).
|
|
|
|
The callers of the runtime config store are synchronous and already run off the
|
|
event loop (``ChannelService.from_app_config`` and its config reloads run in
|
|
worker threads, the Gateway router offloads every store call), so this
|
|
repository uses a synchronous session factory over the application database,
|
|
the same split as the ``agent_storage.backend: db`` store
|
|
(``deerflow.persistence.agents.sql.get_sync_sessionmaker``).
|
|
|
|
- ``save`` is one dialect-native upsert that bumps ``version``: ``INSERT ...
|
|
ON CONFLICT (provider) DO UPDATE SET ..., version = version + 1``. Two
|
|
instances saving the same provider serialize on the row, so each save gets
|
|
its own version. The new version is read back inside the same transaction.
|
|
- ``insert_missing`` imports rows with ``ON CONFLICT DO NOTHING``: a provider
|
|
already in the table is never overwritten, so two instances importing the
|
|
same legacy file at once neither duplicate nor clobber a row.
|
|
|
|
Ciphertext is opaque here; encryption lives in the application store
|
|
(``app.channels.runtime_config_store``). Database errors propagate.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Sequence
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
|
from sqlalchemy.orm import Session, sessionmaker
|
|
|
|
from deerflow.persistence.channel_runtime_configs.model import ChannelRuntimeConfigRow, StoredChannelRuntimeConfig
|
|
|
|
|
|
def _insert(session: Session):
|
|
name = session.get_bind().dialect.name
|
|
if name == "postgresql":
|
|
return pg_insert
|
|
if name == "sqlite":
|
|
return sqlite_insert
|
|
raise ValueError(f"Unsupported channel runtime config database dialect: {name}")
|
|
|
|
|
|
def _stored(row: ChannelRuntimeConfigRow) -> StoredChannelRuntimeConfig:
|
|
return StoredChannelRuntimeConfig(provider=row.provider, config_ciphertext=row.config_ciphertext, disconnected=bool(row.disconnected), version=int(row.version))
|
|
|
|
|
|
class SqlChannelRuntimeConfigRepository:
|
|
"""Persistence facade for ``channel_runtime_configs``; every call blocks, so run it off the event loop."""
|
|
|
|
def __init__(self, session_factory: sessionmaker[Session]) -> None:
|
|
self._sf = session_factory
|
|
|
|
def list_all(self) -> list[StoredChannelRuntimeConfig]:
|
|
with self._sf() as session:
|
|
rows = session.scalars(select(ChannelRuntimeConfigRow).order_by(ChannelRuntimeConfigRow.provider)).all()
|
|
return [_stored(row) for row in rows]
|
|
|
|
def get(self, provider: str) -> StoredChannelRuntimeConfig | None:
|
|
with self._sf() as session:
|
|
row = session.get(ChannelRuntimeConfigRow, provider)
|
|
return _stored(row) if row is not None else None
|
|
|
|
def versions(self) -> dict[str, int]:
|
|
"""``provider -> version`` for every row, without reading the ciphertext."""
|
|
with self._sf() as session:
|
|
rows = session.execute(select(ChannelRuntimeConfigRow.provider, ChannelRuntimeConfigRow.version)).all()
|
|
return {row.provider: int(row.version) for row in rows}
|
|
|
|
def count(self) -> int:
|
|
with self._sf() as session:
|
|
return int(session.scalar(select(func.count()).select_from(ChannelRuntimeConfigRow)) or 0)
|
|
|
|
def save(self, provider: str, *, config_ciphertext: str | None, disconnected: bool, now: float) -> int:
|
|
"""Create or replace ``provider``'s row and return its new version (1 for a new row)."""
|
|
with self._sf() as session:
|
|
insert = _insert(session)
|
|
stmt = insert(ChannelRuntimeConfigRow).values(provider=provider, config_ciphertext=config_ciphertext, disconnected=disconnected, version=1, updated_at=now)
|
|
stmt = stmt.on_conflict_do_update(
|
|
index_elements=[ChannelRuntimeConfigRow.provider],
|
|
set_={
|
|
"config_ciphertext": config_ciphertext,
|
|
"disconnected": disconnected,
|
|
"version": ChannelRuntimeConfigRow.version + 1,
|
|
"updated_at": now,
|
|
},
|
|
)
|
|
session.execute(stmt)
|
|
# Our own write, read inside the transaction that still holds it.
|
|
version = session.scalar(select(ChannelRuntimeConfigRow.version).where(ChannelRuntimeConfigRow.provider == provider))
|
|
session.commit()
|
|
return int(version)
|
|
|
|
def insert_missing(self, rows: Sequence[StoredChannelRuntimeConfig], *, now: float) -> int:
|
|
"""Insert the rows whose provider has none yet (at version 1); returns how many were inserted."""
|
|
if not rows:
|
|
return 0
|
|
inserted = 0
|
|
with self._sf() as session:
|
|
insert = _insert(session)
|
|
for row in rows:
|
|
stmt = (
|
|
insert(ChannelRuntimeConfigRow)
|
|
.values(provider=row.provider, config_ciphertext=row.config_ciphertext, disconnected=row.disconnected, version=1, updated_at=now)
|
|
.on_conflict_do_nothing(index_elements=[ChannelRuntimeConfigRow.provider])
|
|
.returning(ChannelRuntimeConfigRow.provider)
|
|
)
|
|
# Counted from RETURNING (a row only when inserted): psycopg
|
|
# reports rowcount -1 for ON CONFLICT DO NOTHING.
|
|
inserted += len(session.execute(stmt).all())
|
|
session.commit()
|
|
return inserted
|