mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
e8c7df9042
* feat(channels): arbitrate IM transports with a per-provider leader lease Every Gateway instance connected every enabled IM platform with the same bot credentials, so channels could not run on more than one instance (Telegram 409 storms, Discord answering twice, scattered per-process state). channels.leader_lease was reserved by #6746; this implements it. - channel_leases table (migration 0041): one row per provider. Acquire is one dialect-native upsert decided against the database clock and bumps a fencing epoch; renew extends only a live lease at the holder's epoch; release sets the expiry to 0 so a peer takes over on its next tick. - app/channels/leadership.py: the holder's local monotonic deadline (send time of the last confirmed renewal + lease_seconds); holds() turns false one renewal interval early, so the transport stops at least grace_seconds before a peer may connect. DB calls are bounded. - ChannelService: start fence in _start_channel (no DB call) behind every start path; fixed-rate supervisor renews/acquires and runs transport starts/stops as per-provider tasks; takeover reloads the provider's saved config; failed starts back off; shutdown releases only leases whose transport stopped. get_status reports leader/leader_here and the holder's transport state on every instance. - Notification worker leaves rows of providers another instance holds. - Gate: the enabled=false refusal now recommends auto/true; an explicit true with database.backend memory is refused when a channel is enabled. - Docs, Helm, deployment guides, AGENTS.md and CHANGELOG updated. * docs(changelog): link the channel leader lease entry to #6843 * fix(channels): see peer-held leases for providers absent from local config Review on #6843: a provider connected from the web UI on a peer never enters this instance's channel config, so the notification worker here still claimed its rows (and parked them for 900s) and GET /api/channels/ reported no leader for it. Every supervision round, and startup, now also reads the whole lease table once (bounded), and notification exclusions and status use those observations: a provider whose live lease a peer holds is left to it even without local config. Running that read concurrently with renewals exposed that, through SQLAlchemy's aiosqlite adapter, an UPDATE ... RETURNING racing a read on another connection can hand back one of its own bind values instead of the column (stored rows stay correct; plain sqlite3 and raw aiosqlite do not show it). A lease must never take such a row for a confirmed renewal, so acquire/renew/release now decide by affected-row count and acquire reads its epoch back in the same transaction. --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
113 lines
4.1 KiB
Python
113 lines
4.1 KiB
Python
"""Router tests for legacy IM channel management endpoints."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
from uuid import UUID
|
|
|
|
from _router_auth_helpers import make_authed_test_app
|
|
from fastapi.testclient import TestClient
|
|
|
|
from app.gateway.auth.models import User
|
|
from app.gateway.routers import channels
|
|
|
|
|
|
def _admin_user() -> User:
|
|
return User(
|
|
id=UUID("11111111-2222-3333-4444-555555555555"),
|
|
email="admin@example.com",
|
|
password_hash="x",
|
|
system_role="admin",
|
|
)
|
|
|
|
|
|
def _non_admin_user() -> User:
|
|
return User(
|
|
id=UUID("99999999-8888-7777-6666-555555555555"),
|
|
email="user@example.com",
|
|
password_hash="x",
|
|
system_role="user",
|
|
)
|
|
|
|
|
|
def test_restart_channel_requires_admin(monkeypatch):
|
|
service = SimpleNamespace(restart_channel=AsyncMock(return_value=True))
|
|
monkeypatch.setattr("app.channels.service.get_channel_service", lambda: service)
|
|
app = make_authed_test_app(user_factory=_non_admin_user)
|
|
app.include_router(channels.router)
|
|
|
|
with TestClient(app) as client:
|
|
response = client.post("/api/channels/slack/restart")
|
|
|
|
assert response.status_code == 403
|
|
assert "Admin privileges" in response.json()["detail"]
|
|
service.restart_channel.assert_not_awaited()
|
|
|
|
|
|
def test_restart_channel_allows_admin(monkeypatch):
|
|
service = SimpleNamespace(restart_channel=AsyncMock(return_value=True))
|
|
monkeypatch.setattr("app.channels.service.get_channel_service", lambda: service)
|
|
app = make_authed_test_app(user_factory=_admin_user)
|
|
app.include_router(channels.router)
|
|
|
|
with TestClient(app) as client:
|
|
response = client.post("/api/channels/slack/restart")
|
|
|
|
assert response.status_code == 200
|
|
assert response.json() == {
|
|
"success": True,
|
|
"message": "Channel slack restarted successfully",
|
|
}
|
|
service.restart_channel.assert_awaited_once_with("slack")
|
|
|
|
|
|
def test_get_channels_status_remains_read_only(monkeypatch):
|
|
service = SimpleNamespace(
|
|
get_status=lambda: {
|
|
"service_running": True,
|
|
"channels": {
|
|
"slack": {
|
|
"enabled": True,
|
|
"running": True,
|
|
}
|
|
},
|
|
}
|
|
)
|
|
monkeypatch.setattr("app.channels.service.get_channel_service", lambda: service)
|
|
app = make_authed_test_app(user_factory=_non_admin_user)
|
|
app.include_router(channels.router)
|
|
|
|
with TestClient(app) as client:
|
|
response = client.get("/api/channels/")
|
|
|
|
assert response.status_code == 200
|
|
assert response.json()["service_running"] is True
|
|
assert response.json()["channels"]["slack"]["running"] is True
|
|
|
|
|
|
def test_restart_on_a_non_holder_names_the_lease_holder(monkeypatch):
|
|
"""channels.leader_lease: a restart that reaches an instance without the provider's lease says where it runs."""
|
|
service = SimpleNamespace(restart_channel=AsyncMock(return_value=False), leader_of=lambda name: "pod-b:worker", lease_owner_id="pod-a:worker")
|
|
monkeypatch.setattr("app.channels.service.get_channel_service", lambda: service)
|
|
app = make_authed_test_app(user_factory=_admin_user)
|
|
app.include_router(channels.router)
|
|
|
|
with TestClient(app) as client:
|
|
response = client.post("/api/channels/telegram/restart")
|
|
|
|
assert response.status_code == 200
|
|
assert response.json() == {"success": False, "message": "Channel telegram is connected by another Gateway instance (pod-b:worker); it can only be restarted there"}
|
|
|
|
|
|
def test_a_failed_restart_on_the_holder_keeps_the_generic_message(monkeypatch):
|
|
service = SimpleNamespace(restart_channel=AsyncMock(return_value=False), leader_of=lambda name: "pod-a:worker", lease_owner_id="pod-a:worker")
|
|
monkeypatch.setattr("app.channels.service.get_channel_service", lambda: service)
|
|
app = make_authed_test_app(user_factory=_admin_user)
|
|
app.include_router(channels.router)
|
|
|
|
with TestClient(app) as client:
|
|
response = client.post("/api/channels/telegram/restart")
|
|
|
|
assert response.json() == {"success": False, "message": "Failed to restart channel telegram"}
|