mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 21:26:39 +00:00
* fix(channels): make Feishu and DingTalk transports stoppable in-process
FeishuChannel.stop() and DingTalkChannel.stop() could not end their SDK
threads. Both SDKs run on a thread-private event loop that only returned
on a fatal error: lark-oapi's ws.Client.start() parks forever in
run_until_complete(_select()) and has no public stop, and the channel
never kept the client handle; dingtalk-stream's start_forever() is a
while-True loop that also catches CancelledError, and the disconnect()
the channel called does not exist on the SDK. So every stop joined for
5s, raised ChannelStopTimeout, and ChannelService retained the instance
while the platform connection stayed alive on the old process. That
burns the full budget on shutdown and on POST /api/channels/{name}/restart,
and it blocks the per-provider leader lease: a demoted leader could not
release the transport before a peer took over.
Both channels now publish the SDK thread's loop and client under a lock;
stop() schedules a shutdown on that loop and then joins. Feishu disarms
lark's _auto_reconnect before closing the socket and stopping the loop,
because start() otherwise swallows the interrupted run_until_complete in
its reconnect branch and dials again. DingTalk drives its own copy of the
SDK connect loop that re-checks the running flag before every dial, runs
the blocking ticket request off the loop, cancels keepalive with the
socket, and logs the endpoint rather than the ticket-bearing payload. A
shared shutdown_private_loop cancels the SDK's leftover tasks with a
bounded drain and closes the loop; a stop requested before the client
exists never dials. ChannelStopTimeout now only fires for a thread that
genuinely wedges inside an SDK call. Telegram already stopped this way.
Tests: a fake lark ws.Client with the SDK's start() shape (including the
reconnect branch that swallows a bare loop stop), a real local websockets
server that observes the DingTalk close, the stop-before-client race for
both, and a blocking-io anchor that stop() ends a live private loop
without blocking the Gateway loop. All eight were red on main.
* docs(changelog): link the stoppable-transports entry to #6798
* fix(channels): release the socket on a stalled close and bound the DingTalk ticket request
Review follow-ups on #6798 (Totoro-qaq):
Feishu: the bounded _disconnect() wait cancelled websockets' close before
its own close-timeout abort ran, and lark clears _conn in its finally, so
a peer that never answers the close frame left the socket to garbage
collection. Worse, a stop landing while _connect() still held lark's
_lock snapshotted _conn as None. _shutdown_ws now settles the in-flight
dial by taking that lock under one shared close deadline, snapshots the
connection, runs _disconnect() under the same deadline, and aborts the
transport itself (abort_connection_transport in base.py) when the
handshake did not finish, yielding once so connection_lost runs before
loop.stop(). DingTalk's websocket.close() path gets the same abort.
DingTalk: the SDK's open_connection() calls requests.post with no
timeout, and running it in an executor let a stalled ticket endpoint
block interpreter exit (executor workers are joined at shutdown). The
channel now sends the same request through httpx on the stream loop
under a 30s deadline, so stop() cancels it and no worker thread lingers.
Tests drive the real SDK clients against loopback peers: a WebSocket
peer that completes the handshake and never ACKs the close (both
channels assert the connection's socket reports fileno() == -1 right
after stop(); red with the abort disabled), a real websockets server for
the clean path, and a loopback ticket server (ok / 500 / never responds)
with a no-lingering-non-daemon-thread check and a request-shape check.
143 lines
5.4 KiB
Python
143 lines
5.4 KiB
Python
"""Loopback peers for exercising SDK transports at the socket boundary.
|
|
|
|
Both run on the calling test's event loop and only ever bind ``127.0.0.1:0``.
|
|
|
|
- ``StalledWebSocketServer`` completes the WebSocket opening handshake, then
|
|
reads and discards every frame without answering, so a client's close
|
|
handshake never gets its ACK. ``closed`` is set once the peer observes EOF,
|
|
i.e. once the client actually released the TCP connection.
|
|
- ``LocalHttpServer`` answers every request with a fixed status and JSON body,
|
|
or (``stall=True``) accepts the connection and never responds. Each request
|
|
body is recorded so tests can assert on what the SDK sent.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
from collections.abc import Callable
|
|
from typing import Any
|
|
|
|
_WS_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
|
|
|
|
|
|
async def wait_until(predicate: Callable[[], Any], *, timeout: float = 5.0, interval: float = 0.01) -> Any:
|
|
"""Poll ``predicate`` (which may read another thread's state) until it is truthy."""
|
|
loop = asyncio.get_running_loop()
|
|
deadline = loop.time() + timeout
|
|
while True:
|
|
value = predicate()
|
|
if value:
|
|
return value
|
|
if loop.time() >= deadline:
|
|
raise TimeoutError("condition not met in time")
|
|
await asyncio.sleep(interval)
|
|
|
|
|
|
def transport_socket(connection: Any) -> Any:
|
|
"""The socket object under a ``websockets`` connection, for release checks after stop."""
|
|
return connection.transport.get_extra_info("socket")
|
|
|
|
|
|
def _header(request: bytes, name: str) -> str:
|
|
prefix = name.lower() + ":"
|
|
for line in request.decode("latin-1").split("\r\n"):
|
|
if line.lower().startswith(prefix):
|
|
return line.split(":", 1)[1].strip()
|
|
return ""
|
|
|
|
|
|
class StalledWebSocketServer:
|
|
def __init__(self) -> None:
|
|
self.accepted = asyncio.Event()
|
|
self.closed = asyncio.Event()
|
|
self.port = 0
|
|
self._server: asyncio.base_events.Server | None = None
|
|
|
|
async def __aenter__(self) -> StalledWebSocketServer:
|
|
self._server = await asyncio.start_server(self._handle, "127.0.0.1", 0)
|
|
self.port = self._server.sockets[0].getsockname()[1]
|
|
return self
|
|
|
|
async def __aexit__(self, *exc_info: object) -> None:
|
|
assert self._server is not None
|
|
self._server.close()
|
|
try:
|
|
await asyncio.wait_for(self._server.wait_closed(), timeout=5)
|
|
except TimeoutError:
|
|
pass
|
|
|
|
@property
|
|
def url(self) -> str:
|
|
return f"ws://127.0.0.1:{self.port}/"
|
|
|
|
async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
|
try:
|
|
request = await reader.readuntil(b"\r\n\r\n")
|
|
key = _header(request, "Sec-WebSocket-Key")
|
|
accept = base64.b64encode(hashlib.sha1((key + _WS_GUID).encode("ascii")).digest()).decode("ascii")
|
|
writer.write(("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + f"Sec-WebSocket-Accept: {accept}\r\n\r\n").encode("ascii"))
|
|
await writer.drain()
|
|
self.accepted.set()
|
|
while await reader.read(65536):
|
|
pass # swallow pings, data and the close frame; never answer
|
|
except (asyncio.IncompleteReadError, ConnectionError):
|
|
pass
|
|
finally:
|
|
self.closed.set()
|
|
writer.close()
|
|
|
|
|
|
class LocalHttpServer:
|
|
def __init__(self, *, status: int = 200, body: Any = None, stall: bool = False) -> None:
|
|
self.status = status
|
|
self.body = body if body is not None else {}
|
|
self.stall = stall
|
|
self.requests: list[dict[str, Any]] = []
|
|
self.accepted = asyncio.Event()
|
|
self.closed = asyncio.Event()
|
|
self.port = 0
|
|
self._server: asyncio.base_events.Server | None = None
|
|
|
|
async def __aenter__(self) -> LocalHttpServer:
|
|
self._server = await asyncio.start_server(self._handle, "127.0.0.1", 0)
|
|
self.port = self._server.sockets[0].getsockname()[1]
|
|
return self
|
|
|
|
async def __aexit__(self, *exc_info: object) -> None:
|
|
assert self._server is not None
|
|
self._server.close()
|
|
try:
|
|
await asyncio.wait_for(self._server.wait_closed(), timeout=5)
|
|
except TimeoutError:
|
|
pass
|
|
|
|
@property
|
|
def url(self) -> str:
|
|
return f"http://127.0.0.1:{self.port}/open"
|
|
|
|
async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
|
try:
|
|
head = await reader.readuntil(b"\r\n\r\n")
|
|
length = int(_header(head, "Content-Length") or 0)
|
|
raw = await reader.readexactly(length) if length else b""
|
|
try:
|
|
self.requests.append(json.loads(raw) if raw else {})
|
|
except ValueError:
|
|
self.requests.append({"raw": raw.decode("latin-1")})
|
|
self.accepted.set()
|
|
if self.stall:
|
|
while await reader.read(65536):
|
|
pass # hold the request open until the client gives up
|
|
return
|
|
payload = json.dumps(self.body).encode("utf-8")
|
|
writer.write((f"HTTP/1.1 {self.status} X\r\nContent-Type: application/json\r\nContent-Length: {len(payload)}\r\nConnection: close\r\n\r\n").encode("ascii") + payload)
|
|
await writer.drain()
|
|
except (asyncio.IncompleteReadError, ConnectionError):
|
|
pass
|
|
finally:
|
|
self.closed.set()
|
|
writer.close()
|