diff --git a/backend/app/channels/AGENTS.md b/backend/app/channels/AGENTS.md index 243e64b5b..b48de56a8 100644 --- a/backend/app/channels/AGENTS.md +++ b/backend/app/channels/AGENTS.md @@ -10,7 +10,7 @@ Bridges external messaging platforms (Feishu, Slack, Telegram, Discord, DingTalk - `manager.py` - Core dispatcher: creates threads via `client.threads.create()`, routes commands including `/goal` (setting a goal persists it through Gateway and then routes the objective as a chat turn), keeps Slack/Discord on `client.runs.wait()`, uses `client.runs.stream(["messages-tuple", "values"])` for Feishu/Telegram incremental outbound updates, serializes same-thread Feishu turns in-manager when the channel's `ChannelRunPolicy.serialize_thread_runs=True` so rapid follow-ups queue instead of tripping the runtime busy reply, and switches to `client.runs.create()` (fire-and-forget, returns once the run is `pending`) for channels whose `ChannelRunPolicy.fire_and_forget=True` so long autonomous runs do not hit the SDK default 300s `httpx.ReadTimeout` A swallowed streaming failure publishes its final outbound before releasing the inbound dedupe key, so a provider redelivery can retry without overtaking the terminal reply. **What may be published from the stream is an allowlist, not a denylist** (`_accumulate_stream_text`): only assistant message types — LangChain serializes `AIMessage.type` as `"ai"` and `AIMessageChunk.type` as `"AIMessageChunk"`, plus the OpenAI-style `"assistant"` spelling for foreign runtimes — become displayable text. The previous rule rejected only payloads whose `type` contained `"tool"` and therefore published everything else, which leaked DeerFlow's hidden model context to every streaming IM channel: `DynamicContextMiddleware` injects recalled memory as a hidden `HumanMessage` (`type == "human"`) and rewrites the user's own turn into a new `HumanMessage`, `DurableContextMiddleware` injects a hidden `` `HumanMessage`, and LangGraph fans those state writes out on the `messages-tuple` stream. Proved live on a Buzz relay, which published a `` fact block and, in another run, a verbatim echo of the user's own message as the assistant's reply. Matching is by prefix (`ai` / `assistant`), never substring, because ordinary words contain `"ai"` (`chain`, `domain`). The message type is resolved by `_stream_payload_type`, which handles both the `model_dump()` shape DeerFlow's own gateway emits and LangChain's `to_json()` constructor shape (whose top-level `type` is the literal `"constructor"`, with the class name at the tail of the `id` path). A bare `str` payload is no longer accepted at all: it carries no type information, so it cannot be attributed to the assistant, and nothing in DeerFlow produces one (`runtime/serialization.py::serialize_messages_tuple` always emits `[message_dict, metadata]`). -- `base.py` - Abstract `Channel` base class (start/stop/send lifecycle). Provider callbacks that submit coroutines from SDK threads through `run_coroutine_threadsafe()` must use `_submit_threadsafe_coroutine()`: submission is closed atomically with shutdown, and `stop()` must call `_close_and_drain_threadsafe_futures()` before tearing down SDK resources. +- `base.py` - Abstract `Channel` base class (start/stop/send lifecycle). Provider callbacks that submit coroutines from SDK threads must use `_submit_threadsafe_coroutine()`: it creates and retains the real `asyncio.Task` on the owner loop instead of treating `run_coroutine_threadsafe()`'s proxy Future as a completion signal. Submission is closed atomically with shutdown, and `stop()` must call `_close_and_drain_threadsafe_futures()` before tearing down SDK resources. - `service.py` - Manages lifecycle of all configured channels from `config.yaml`. Shutdown closes manager admission first and keeps transports alive until every manager worker/follow-up watcher has exited. A successful manager stop therefore owns no live handler; if the Gateway's outer timeout cancels shutdown, the service retains its channel objects and global singleton so unfinished resources are not detached and cleanup can be retried. - `slack.py` / `feishu.py` / `telegram.py` / `discord.py` / `dingtalk.py` - Platform-specific implementations (`feishu.py` tracks the running card `message_id` in memory and patches the same card in place; `telegram.py` accepts inbound text/photos/documents, preserves media captions, hands token-free attachment bytes to the shared upload pipeline, edits the "Working on it..." stream target in place via `editMessageText`, and can optionally send final Markdown replies as Rich Messages through `channels.telegram.rich_messages`; `discord.py` registers typing loops before inbound handling yields and `_start_typing()` refuses work once `_running` is false; because `stop()` runs on the main loop while typing tasks belong to `_discord_loop`, normal cross-thread cancellation, awaiting, and map cleanup are scheduled there with a bounded wait, while `_run_client()` drains tasks in its `finally` block before an exception/disconnect can make that loop unusable; an already-stopped foreign loop must never have its tasks awaited from the main loop; `dingtalk.py` optionally uses AI Card streaming for in-place updates when `card_template_id` is configured, and overrides `receive_file` to download inbound images (`picture`/`richText`) and documents (`file`) by `downloadCode` into the thread uploads bucket, mirroring `feishu.py`) - `buzz.py` - Buzz (Nostr relay) implementation: one NIP-42-authenticated websocket, pubkey-allowlist + mention/DM/thread-follow gating, streaming replies via in-place kind-40003 edits; requires the `buzz` dependency extra. **Subscription model** (operator-facing version in [IM_CHANNEL_CONNECTIONS.md](../../docs/IM_CHANNEL_CONNECTIONS.md#buzz-subscription-model)): the relay fans kind-9 chat events out **only** to channel-scoped subscriptions, proved against a live relay — `REQ {"kinds":[9]}` is accepted and answered with `EOSE` but never receives an event (the connector authenticated and then sat silent forever), `REQ {"kinds":[9],"#h":[uuid]}` works, and a multi-value `#h` matches nothing, so it is strictly one REQ per channel (the same shape as Buzz's own `buzz-acp` harness). Every connection therefore rebuilds three kinds of subscription after NIP-42 auth: `buzz-discovery` (`{"kinds":[39000]}`, a historical query returning exactly the channels this identity belongs to, one stored event each then `EOSE` — adding `#p` returns zero, do not "narrow" it), `buzz-membership` (`{"kinds":[44100,44101],"#p":[us],"since":}`, the relay-signed member-added/member-removed notifications whose `p` tag names the affected member and `h` tag the channel), and one `buzz-chat-` per discovered channel. Chat subscriptions open as each kind-39000 arrives; the discovery `EOSE` is the completeness barrier that retries any that failed and warns when discovery found nothing. A kind-44100 for our pubkey subscribes to the new channel **live** (then re-issues discovery so its name/type reach the DM-detection cache, but only when that channel's metadata is actually missing — an unconditional refresh is how a burst of 44100s multiplied into one discovery pass each); a kind-44101 issues `buzz_nostr.close_frame` for exactly that channel's subscription and drops its metadata. **The membership subscription is scoped to LIVE events** and this is load-bearing: buzz-relay *stores* 44100/44101 and serves history newest-first (default limit 2000), so an unscoped filter replayed the whole membership history on every connect — every stored add read as live, re-running discovery once each (M+1 discovery passes × N stored kind-39000 events per connect, observed live as two `channel discovery complete` lines and one channel logged `` because the 44100 path subscribed before its metadata arrived), re-subscribing channels we have since been removed from, and letting a stored 44101 transiently unsubscribe a channel we are still in. `since` is anchored at the moment the socket opened (`_session_started_at`) minus `MEMBERSHIP_LOOKBACK_SECONDS` (60) of slack, which covers both relay clock skew and a membership change published *during* the connect/auth handshake; the slack can only cost an idempotent replay of the last minute. **A relay `CLOSED` frame is recovered, not merely forgotten** — every subscription on the socket fails silently when dropped, so `_handle_closed` re-issues it, bounded by `MAX_RESUBSCRIBE_ATTEMPTS` (3) per subscription id per connection *and per auth epoch* (`_resubscribe_attempts` is reset on session start, on re-auth, and by `stop()`, so pre-auth rejections — which the auth branch already recovers wholesale — never spend the authenticated session's budget). The first retry is immediate (the common case is a one-off hiccup); later ones back off 1s then 2s, awaited inline in the read loop rather than in a background task that could outlive its own socket. **`auth-required:` before this socket has completed its NIP-42 handshake is the expected bootstrap sequence, not a refusal**, and `_handle_closed` short-circuits it ahead of every recovery path: the connector opens its control REQs immediately in case the relay serves unauthenticated reads, a closed relay answers `auth-required:` plus an `AUTH` challenge, and the auth branch re-opens everything. That case is logged at DEBUG and consumes neither the permanent-refusal branch nor the retry budget (a chat subscription is still dropped from `_chat_subscriptions`, since it genuinely is not subscribed and discovery is what re-opens it). Treating it as permanent produced an operator-facing warning claiming discovery/membership tracking was DOWN in the same run where discovery then completed, every channel was subscribed, and a brand-new channel's kind-44100 was picked up one second later. The boundary is the per-socket `_auth_completed` flag, set once the signed AUTH event has been sent and cleared on session entry, session exit, and `stop()`; the same reason *after* that stays loud, and says the subscription is down until the relay's next AUTH challenge or the next reconnect rather than borrowing the non-auth wording. Then `_is_transient_close` decides whether to retry at all: NIP-01/NIP-42 `auth-required:`/`restricted:`/`blocked:`/`mute:`/`invalid:`/`pow:` prefixes and buzz-relay's own removal/revocation prose are permanent (do not fight the relay over a channel that is no longer ours; a post-auth `auth-required:` is recovered by the AUTH branch, not by re-issuing), `rate-limited:`/`error:`/no-reason-at-all and anything unrecognized are transient — the default resolves toward "keep listening" because going silently deaf is the failure this exists to remove, and the attempt budget bounds a wrong guess. A chat `CLOSED` is only ever recovered for a channel already in `_chat_subscriptions`: a `CLOSED` is relay-supplied, so acting on an unknown one would let a relay induce a subscription just by naming a channel. Every subscription that goes unlistened is logged at WARNING, never INFO. Subscription ids are deterministic per channel precisely so one can be replaced or closed without disturbing the others on the socket, and `_chat_subscriptions` is per-socket state cleared on session end, on re-auth (a pre-auth REQ may have been rejected), and by `stop()`. Three bounds on remote-fed state: `MAX_CACHED_CHANNELS` (512) caps the kind-39000 metadata cache, the watermark map, and the resubscribe-attempt map; `MAX_CHANNEL_SUBSCRIPTIONS` (256, well under buzz-relay's own 1024-per-connection ceiling) caps live chat subscriptions — at the cap new channels are refused and named in a warning rather than evicting a working subscription. **Known bound (documented, not fixed):** the relay caps historical delivery at 2000 events per subscription, newest-first, even with a `since`, so >2000 unread messages in a *single* channel across a disconnect loses the oldest — the relay never sends them and the watermark advances past them. That is the one remaining path that can skip; everything else fails toward replay. **Trust model** (operator-facing version in [IM_CHANNEL_CONNECTIONS.md](../../docs/IM_CHANNEL_CONNECTIONS.md#buzz-trust-model)): every inbound `EVENT` is authenticated at the single `handle_relay_frame` choke point — the NIP-01 id is recomputed from the delivered payload and the BIP-340 Schnorr signature verified against the claimed `pubkey` (`buzz_nostr.verify_event`, pure and total: malformed input returns `False`, never raises) — so `ev["pubkey"]`, the authorization principal for both the allowlist and the `/connect` bind, cannot be forged by a relay the DeerFlow operator does not run. What remains trusted is the *authorship* of kind-39000 channel metadata: any member can sign one, and because per-channel subscriptions are now driven by discovery, a forged kind-39000 has two effects rather than one — it can mark a channel `type: "dm"` (relaxing `require_mention` for that channel) **and** it can induce a chat subscription for a channel of the forger's choosing, since the channels we listen to are exactly the channels we hold metadata for. Neither makes anything be *acted on*: `allowed_users` and per-event signature verification are independent gates, so an induced subscription only means the relay reads its own traffic back to a subscriber that drops it, bounded by `MAX_CHANNEL_SUBSCRIPTIONS` (which refuses rather than evicts, so it cannot displace a real channel). Same for a forged kind-44100, except its `p` tag is re-checked locally so it must at least name us. Closing this needs a configured trusted relay pubkey, which `relay_url` is not. `allowed_users` is deny-by-default (empty = nobody, unlike siblings' empty = everyone), so `start()` logs a WARNING when it is empty and each drop logs at DEBUG. The resubscribe cursor (`since`) is **per channel**, advances only for events that were actually processed, and never past `now + MAX_FUTURE_SKEW_SECONDS`, because it is peer-supplied (`created_at`) and a single future-dated event otherwise made the connector permanently deaf. Per channel rather than global is the safety-critical half: subscriptions are per channel, so one shared cursor is the newest event seen in *any* channel and a busy channel would drag it past a quiet channel's unread messages, skipping them on the next reconnect — measured on a live relay, three channels of one identity sat ~28h apart. Per-channel cursors can only ever cost duplicate delivery (absorbed by the manager's `event_id` dedupe), and an evicted cursor degrades to "no `since`", i.e. the relay's default backlog — both fail toward replay, never toward a miss. Streaming tracks every oversize chunk index (`_stream_targets` for chunk 0, `_stream_tails` for the rest), since the manager republishes cumulative text and reposting `chunks[1:]` per update flooded the channel; all of it is per-connection state cleared by `stop()`, and the remote-fed kind-39000 cache is capped at `MAX_CACHED_CHANNELS`. **`send()` refuses outright to publish text carrying a hidden model-context wrapper** (``, ``, `` — `_HIDDEN_CONTEXT_MARKERS`), logging at ERROR and clearing the stream bookkeeping on a blocked `is_final`. This is defense in depth behind the manager's allowlist, and it lives here rather than in a sibling connector because on Buzz a leak is permanent: every streaming update is an immutable public Nostr event, so a corrective edit only changes what clients render while the original leaked event stays on the relay. Matching is on the literal opening tag, so a reply that merely talks about memory is still published diff --git a/backend/app/channels/base.py b/backend/app/channels/base.py index 62f0a3bf6..58e633cbd 100644 --- a/backend/app/channels/base.py +++ b/backend/app/channels/base.py @@ -9,6 +9,7 @@ from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable, Coroutine from concurrent.futures import CancelledError as FutureCancelledError from concurrent.futures import Future +from dataclasses import dataclass from typing import Any, TypeVar from app.channels.commands import extract_connect_code @@ -29,6 +30,18 @@ logger = logging.getLogger(__name__) T = TypeVar("T") +@dataclass(eq=False, slots=True) +class _ThreadsafeSubmission: + coroutine: Coroutine[Any, Any, Any] + loop: asyncio.AbstractEventLoop + name: str + msg_id: Any + reservation: InboundReservation | None + completion: Future[Any] + task: asyncio.Task[Any] | None = None + cancel_requested: bool = False + + class Channel(ABC): """Base class for all IM channel implementations. @@ -48,9 +61,9 @@ class Channel(ABC): # Provider SDK callbacks often run on a dedicated thread and submit # preparation work to the Gateway loop. Submission and shutdown share # this lock so stop() cannot miss a future created concurrently. - self._threadsafe_futures: set[Future[Any]] = set() - self._threadsafe_futures_lock = threading.Lock() - self._threadsafe_future_intake_open = True + self._threadsafe_submissions: set[_ThreadsafeSubmission] = set() + self._threadsafe_submissions_lock = threading.Lock() + self._threadsafe_submission_intake_open = True @property def is_running(self) -> bool: @@ -139,12 +152,10 @@ class Channel(ABC): def _open_threadsafe_future_intake(self) -> None: """Allow a newly started provider to submit work to its main loop.""" - with self._threadsafe_futures_lock: - pending = [future for future in self._threadsafe_futures if not future.done()] - if pending: + with self._threadsafe_submissions_lock: + if self._threadsafe_submissions: raise RuntimeError(f"cannot restart {self.name} while cross-thread work is still running") - self._threadsafe_futures.clear() - self._threadsafe_future_intake_open = True + self._threadsafe_submission_intake_open = True def _submit_threadsafe_coroutine( self, @@ -154,60 +165,111 @@ class Channel(ABC): name: str, msg_id: Any, reservation: InboundReservation | None = None, - ) -> Future[T] | None: - """Submit and retain provider-thread work until completion or shutdown.""" + ) -> bool: + """Submit provider-thread work while retaining its real asyncio Task.""" - with self._threadsafe_futures_lock: - if not self._threadsafe_future_intake_open or loop is None or not loop.is_running(): + with self._threadsafe_submissions_lock: + if not self._threadsafe_submission_intake_open or loop is None or not loop.is_running(): coroutine.close() if reservation is not None: reservation.release() - return None - try: - future = asyncio.run_coroutine_threadsafe(coroutine, loop) - except RuntimeError: - coroutine.close() - if reservation is not None: - reservation.release() - return None - self._threadsafe_futures.add(future) + return False - future.add_done_callback( - lambda completed: self._finalize_threadsafe_future( - completed, + submission = _ThreadsafeSubmission( + coroutine=coroutine, + loop=loop, name=name, msg_id=msg_id, reservation=reservation, + completion=Future(), ) - ) - return future + self._threadsafe_submissions.add(submission) + try: + loop.call_soon_threadsafe(self._start_threadsafe_submission, submission) + except RuntimeError: + self._threadsafe_submissions.discard(submission) + coroutine.close() + if reservation is not None: + reservation.release() + return False + return True - def _finalize_threadsafe_future( + def _start_threadsafe_submission(self, submission: _ThreadsafeSubmission) -> None: + """Create the owned Task on its event loop or finish a pre-start cancel.""" + task: asyncio.Task[Any] | None = None + startup_error: BaseException | None = None + with self._threadsafe_submissions_lock: + if submission.cancel_requested: + self._threadsafe_submissions.discard(submission) + cancelled_before_start = True + else: + cancelled_before_start = False + try: + task = submission.loop.create_task(submission.coroutine) + except BaseException as exc: + self._threadsafe_submissions.discard(submission) + startup_error = exc + else: + submission.task = task + + if cancelled_before_start: + submission.coroutine.close() + if submission.reservation is not None: + submission.reservation.release() + submission.completion.cancel() + return + + if startup_error is not None: + submission.coroutine.close() + if submission.reservation is not None: + submission.reservation.release() + submission.completion.set_exception(startup_error) + self._log_future_error(submission.completion, submission.name, submission.msg_id) + return + + assert task is not None + task.add_done_callback(lambda completed: self._finalize_threadsafe_submission(submission, completed)) + + def _finalize_threadsafe_submission( self, - future: Future[Any], - *, - name: str, - msg_id: Any, - reservation: InboundReservation | None, + submission: _ThreadsafeSubmission, + task: asyncio.Task[Any], ) -> None: - with self._threadsafe_futures_lock: - self._threadsafe_futures.discard(future) - if reservation is not None: - reservation.release() - self._log_future_error(future, name, msg_id) + with self._threadsafe_submissions_lock: + self._threadsafe_submissions.discard(submission) + if submission.reservation is not None: + submission.reservation.release() + + if task.cancelled(): + submission.completion.cancel() + else: + try: + submission.completion.set_result(task.result()) + except BaseException as exc: + submission.completion.set_exception(exc) + self._log_future_error(submission.completion, submission.name, submission.msg_id) async def _close_and_drain_threadsafe_futures(self) -> None: - """Close cross-thread submission, then cancel and await owned futures.""" - with self._threadsafe_futures_lock: - self._threadsafe_future_intake_open = False - futures = tuple(self._threadsafe_futures) + """Close submission, then cancel and await the owned asyncio Tasks.""" + with self._threadsafe_submissions_lock: + self._threadsafe_submission_intake_open = False + submissions = tuple(self._threadsafe_submissions) + tasks: list[tuple[asyncio.AbstractEventLoop, asyncio.Task[Any]]] = [] + for submission in submissions: + submission.cancel_requested = True + if submission.task is not None: + tasks.append((submission.loop, submission.task)) - for future in futures: - future.cancel() - if futures: - await asyncio.gather(*(asyncio.wrap_future(future) for future in futures), return_exceptions=True) - with self._threadsafe_futures_lock: - self._threadsafe_futures.difference_update(future for future in futures if future.done()) + for loop, task in tasks: + try: + loop.call_soon_threadsafe(task.cancel) + except RuntimeError: + logger.warning("[%s] event loop closed before cross-thread task cancellation", self.name) + if submissions: + await asyncio.gather( + *(asyncio.shield(asyncio.wrap_future(submission.completion)) for submission in submissions), + return_exceptions=True, + ) def _pending_connect_code(self, text: str) -> str | None: """Return the one-time bind code if *text* is a ``/connect `` command diff --git a/backend/app/channels/dingtalk.py b/backend/app/channels/dingtalk.py index 2dff8c339..d8d9e3b8d 100644 --- a/backend/app/channels/dingtalk.py +++ b/backend/app/channels/dingtalk.py @@ -398,7 +398,7 @@ class DingTalkChannel(Channel): connect_code = self._pending_connect_code(text) if connect_code: if self._main_loop and self._main_loop.is_running(): - future = self._submit_threadsafe_coroutine( + scheduled = self._submit_threadsafe_coroutine( self._bind_connection_from_connect_code( conversation_type=conversation_type, sender_staff_id=sender_staff_id, @@ -410,7 +410,7 @@ class DingTalkChannel(Channel): name="bind_connection", msg_id=msg_id, ) - if future is None: + if not scheduled: logger.info("[DingTalk] main loop stopped before channel connection bind could be scheduled") else: logger.warning("[DingTalk] main loop not running, cannot bind channel connection") @@ -492,14 +492,14 @@ class DingTalkChannel(Channel): with self._incoming_messages_lock: self._incoming_messages[source_key] = message logger.info("[DingTalk] publishing inbound message to bus (type=%s, msg_id=%s)", msg_type.value, msg_id) - future = self._submit_threadsafe_coroutine( + scheduled = self._submit_threadsafe_coroutine( self._prepare_inbound(chat_id, inbound, reservation=reservation), self._main_loop, name="prepare_inbound", msg_id=msg_id, reservation=reservation, ) - if future is None: + if not scheduled: logger.info("[DingTalk] main loop stopped before reserved inbound could be scheduled") else: logger.warning("[DingTalk] main loop not running, cannot publish inbound message") diff --git a/backend/app/channels/feishu.py b/backend/app/channels/feishu.py index 417bbe9c3..191b53892 100644 --- a/backend/app/channels/feishu.py +++ b/backend/app/channels/feishu.py @@ -844,7 +844,7 @@ class FeishuChannel(Channel): if reservation is None: return logger.info("[Feishu] publishing inbound message to bus (type=%s, msg_id=%s)", inbound.msg_type.value, msg_id) - future = self._submit_threadsafe_coroutine( + scheduled = self._submit_threadsafe_coroutine( self._prepare_inbound( msg_id, inbound, @@ -856,20 +856,20 @@ class FeishuChannel(Channel): msg_id=msg_id, reservation=reservation, ) - if future is None: + if not scheduled: logger.info("[Feishu] main loop stopped before reserved inbound could be scheduled") else: logger.warning("[Feishu] main loop not running, cannot publish inbound message") def _schedule_batch_flush(self, key: tuple[str, str], source_message_id: str) -> None: if self._main_loop and self._main_loop.is_running(): - future = self._submit_threadsafe_coroutine( + scheduled = self._submit_threadsafe_coroutine( self._flush_pending_inbound_batch_after(key, source_message_id), self._main_loop, name="flush_inbound_batch", msg_id=source_message_id, ) - if future is None: + if not scheduled: logger.info("[Feishu] main loop stopped before inbound batch flush could be scheduled") else: logger.warning("[Feishu] main loop not running, cannot flush inbound batch") @@ -1115,7 +1115,7 @@ class FeishuChannel(Channel): connect_code = self._pending_connect_code(text) if connect_code: if self._main_loop and self._main_loop.is_running(): - future = self._submit_threadsafe_coroutine( + scheduled = self._submit_threadsafe_coroutine( self._bind_connection_from_connect_code( message_id=msg_id, chat_id=chat_id, @@ -1126,7 +1126,7 @@ class FeishuChannel(Channel): name="bind_connection", msg_id=msg_id, ) - if future is None: + if not scheduled: logger.info("[Feishu] main loop stopped before channel connection bind could be scheduled") else: logger.warning("[Feishu] main loop not running, cannot bind channel connection") diff --git a/backend/app/channels/slack.py b/backend/app/channels/slack.py index 3e7b17152..e2ecdd272 100644 --- a/backend/app/channels/slack.py +++ b/backend/app/channels/slack.py @@ -338,7 +338,7 @@ class SlackChannel(Channel): connect_code = self._pending_connect_code(text) if connect_code: if self._loop and self._loop.is_running(): - future = self._submit_threadsafe_coroutine( + scheduled = self._submit_threadsafe_coroutine( self._bind_connection_from_connect_code( event=event, team_id=str(team_id or ""), @@ -348,7 +348,7 @@ class SlackChannel(Channel): name="bind_connection", msg_id=event.get("ts"), ) - if future is None: + if not scheduled: logger.info("[Slack] main loop stopped before channel connection bind could be scheduled") return @@ -398,14 +398,14 @@ class SlackChannel(Channel): # thread; no coroutine/Future waits for queue capacity. self._loop.call_soon_threadsafe(self._commit_reserved_inbound, reservation, inbound) else: - future = self._submit_threadsafe_coroutine( + scheduled = self._submit_threadsafe_coroutine( self._publish_inbound_with_connection(inbound, reservation=reservation, team_id=team_id), self._loop, name="publish_inbound", msg_id=event.get("ts", thread_ts), reservation=reservation, ) - if future is None: + if not scheduled: logger.info("[Slack] main loop stopped before reserved inbound could be scheduled") except RuntimeError: reservation.release() diff --git a/backend/app/channels/telegram.py b/backend/app/channels/telegram.py index 2c9e2e123..9e6877663 100644 --- a/backend/app/channels/telegram.py +++ b/backend/app/channels/telegram.py @@ -857,7 +857,7 @@ class TelegramChannel(Channel): return try: inbound = await self._attach_connection_identity(inbound) - future = self._submit_threadsafe_coroutine( + scheduled = self._submit_threadsafe_coroutine( self._process_incoming_with_reply( chat_id, update.message.message_id, @@ -869,7 +869,7 @@ class TelegramChannel(Channel): msg_id=update.message.message_id, reservation=reservation, ) - if future is None: + if not scheduled: logger.info("[Telegram] main loop stopped before reserved command could be scheduled") except Exception: reservation.release() @@ -928,7 +928,7 @@ class TelegramChannel(Channel): return try: inbound = await self._attach_connection_identity(inbound) - future = self._submit_threadsafe_coroutine( + scheduled = self._submit_threadsafe_coroutine( self._process_incoming_with_reply( chat_id, update.message.message_id, @@ -940,7 +940,7 @@ class TelegramChannel(Channel): msg_id=update.message.message_id, reservation=reservation, ) - if future is None: + if not scheduled: logger.info("[Telegram] main loop stopped before reserved inbound could be scheduled") except Exception: reservation.release() diff --git a/backend/tests/test_channel_intake_backpressure.py b/backend/tests/test_channel_intake_backpressure.py index 20dd86c7d..ec79323ff 100644 --- a/backend/tests/test_channel_intake_backpressure.py +++ b/backend/tests/test_channel_intake_backpressure.py @@ -375,6 +375,7 @@ async def test_provider_stop_drains_cross_thread_preparation_futures() -> None: for channel in providers: started = asyncio.Event() cancelled = asyncio.Event() + release_after_cancel = asyncio.Event() finished = asyncio.Event() async def preparation() -> None: @@ -383,29 +384,108 @@ async def test_provider_stop_drains_cross_thread_preparation_futures() -> None: await asyncio.Event().wait() except asyncio.CancelledError: cancelled.set() - raise + await release_after_cancel.wait() finally: finished.set() channel._running = True channel._main_loop = loop channel._open_threadsafe_future_intake() - future = await asyncio.to_thread( + scheduled = await asyncio.to_thread( channel._submit_threadsafe_coroutine, preparation(), loop, name="test_preparation", msg_id="message-1", ) - assert future is not None + assert scheduled is True await asyncio.wait_for(started.wait(), timeout=1) - await asyncio.wait_for(channel.stop(), timeout=1) + stop_task = asyncio.create_task(channel.stop()) + await asyncio.wait_for(cancelled.wait(), timeout=1) + await asyncio.sleep(0) + + assert not stop_task.done() + assert not finished.is_set() + + release_after_cancel.set() + await asyncio.wait_for(stop_task, timeout=1) - assert cancelled.is_set() assert finished.is_set() - assert future.done() - assert channel._threadsafe_futures == set() + assert channel._threadsafe_submissions == set() + + +@pytest.mark.asyncio +async def test_shutdown_closes_cross_thread_submission_before_task_start() -> None: + channel = SlackChannel(MessageBus(), config={}) + channel._open_threadsafe_future_intake() + coroutine_started = False + + async def preparation() -> None: + nonlocal coroutine_started + coroutine_started = True + + scheduled = channel._submit_threadsafe_coroutine( + preparation(), + asyncio.get_running_loop(), + name="test_preparation", + msg_id="message-1", + ) + assert scheduled is True + + await asyncio.wait_for(channel._close_and_drain_threadsafe_futures(), timeout=1) + + assert coroutine_started is False + assert channel._threadsafe_submissions == set() + + +@pytest.mark.asyncio +async def test_cancelled_cross_thread_drain_remains_retryable() -> None: + channel = SlackChannel(MessageBus(), config={}) + channel._open_threadsafe_future_intake() + started = asyncio.Event() + cancellation_seen = asyncio.Event() + release_after_cancel = asyncio.Event() + finished = asyncio.Event() + + async def preparation() -> None: + started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + cancellation_seen.set() + await release_after_cancel.wait() + finally: + finished.set() + + assert channel._submit_threadsafe_coroutine( + preparation(), + asyncio.get_running_loop(), + name="test_preparation", + msg_id="message-1", + ) + await asyncio.wait_for(started.wait(), timeout=1) + submission = next(iter(channel._threadsafe_submissions)) + + first_drain = asyncio.create_task(channel._close_and_drain_threadsafe_futures()) + await asyncio.wait_for(cancellation_seen.wait(), timeout=1) + first_drain.cancel() + with pytest.raises(asyncio.CancelledError): + await first_drain + + assert not finished.is_set() + assert channel._threadsafe_submissions + assert not submission.completion.done() + + second_drain = asyncio.create_task(channel._close_and_drain_threadsafe_futures()) + await asyncio.sleep(0) + assert not second_drain.done() + + release_after_cancel.set() + await asyncio.wait_for(second_drain, timeout=1) + + assert finished.is_set() + assert channel._threadsafe_submissions == set() def test_channel_service_threads_intake_limits_into_bus_and_worker_pool() -> None: diff --git a/backend/tests/test_channels.py b/backend/tests/test_channels.py index 46806644f..5a21ff491 100644 --- a/backend/tests/test_channels.py +++ b/backend/tests/test_channels.py @@ -7844,9 +7844,9 @@ class TestSlackSendRetry: class TestSlackAllowedUsers: @staticmethod - def _submit_coro(coro, loop): + def _submit_coro(coro, loop, **_kwargs): coro.close() - return MagicMock() + return True @staticmethod def _immediate_loop(): @@ -7946,8 +7946,9 @@ class TestSlackAllowedUsers: "ts": "1710000000.000100", } - with patch( - "app.channels.slack.asyncio.run_coroutine_threadsafe", + with patch.object( + channel, + "_submit_threadsafe_coroutine", side_effect=self._submit_coro, ) as submit: channel._handle_message_event(event)