mirror of
https://github.com/OpenBMB/ChatDev.git
synced 2026-11-18 07:50:38 +00:00
fix: avoid blocking websocket owner loop
This commit is contained in:
parent
a2ea4a263c
commit
9ae144a39b
@ -178,9 +178,8 @@ class WebSocketManager:
|
||||
which spins up a *new* event loop, causing ``RuntimeError: … attached to a
|
||||
different loop`` or silent delivery failures.
|
||||
|
||||
The fix: always schedule the coroutine on the loop that owns the sockets
|
||||
via ``asyncio.run_coroutine_threadsafe`` and wait for the result with a
|
||||
short timeout so the caller knows if delivery failed.
|
||||
Worker threads submit to the owner loop and wait with a short timeout.
|
||||
Calls from the owner loop schedule a task without blocking that loop.
|
||||
"""
|
||||
loop = self._owner_loop
|
||||
if loop is None or loop.is_closed():
|
||||
@ -190,6 +189,14 @@ class WebSocketManager:
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
current_loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
current_loop = None
|
||||
if current_loop is loop:
|
||||
loop.create_task(self.send_message(session_id, message))
|
||||
return
|
||||
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.send_message(session_id, message), loop
|
||||
)
|
||||
|
||||
@ -3,18 +3,20 @@
|
||||
Verifies that send_message_sync correctly delivers messages when called
|
||||
from worker threads (the common case during workflow execution).
|
||||
|
||||
The test avoids importing the full server stack (which has circular import
|
||||
issues) by patching only the WebSocketManager class directly.
|
||||
Load the manager in isolation without leaving dependency stubs in sys.modules.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import List
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
@ -23,40 +25,32 @@ import pytest
|
||||
# Isolate WebSocketManager from the circular-import chain
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Stub out heavy modules so we can import websocket_manager in isolation
|
||||
_stubs = {}
|
||||
for mod_name in (
|
||||
"check", "check.check",
|
||||
"runtime", "runtime.sdk", "runtime.bootstrap", "runtime.bootstrap.schema",
|
||||
"server.services.workflow_run_service",
|
||||
"server.services.message_handler",
|
||||
"server.services.attachment_service",
|
||||
"server.services.session_execution",
|
||||
"server.services.session_store",
|
||||
"server.services.artifact_events",
|
||||
):
|
||||
if mod_name not in sys.modules:
|
||||
_stubs[mod_name] = MagicMock()
|
||||
sys.modules[mod_name] = _stubs[mod_name]
|
||||
|
||||
from server.services.websocket_manager import WebSocketManager # noqa: E402
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_manager() -> WebSocketManager:
|
||||
"""Create a WebSocketManager with minimal mocks."""
|
||||
session_store = MagicMock()
|
||||
session_store.has_session.return_value = False
|
||||
session_store.get_session.return_value = None
|
||||
return WebSocketManager(
|
||||
session_store=session_store,
|
||||
session_controller=MagicMock(),
|
||||
attachment_service=MagicMock(),
|
||||
workflow_run_service=MagicMock(),
|
||||
@pytest.fixture
|
||||
def make_manager(monkeypatch):
|
||||
service_name = "server.services.workflow_run_service"
|
||||
service_stub = ModuleType(service_name)
|
||||
service_stub.WorkflowRunService = MagicMock()
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"websocket_manager_under_test",
|
||||
Path(__file__).resolve().parents[1] / "server/services/websocket_manager.py",
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
# Only the workflow runner needs isolation. In particular, keep the real
|
||||
# in-memory session store so snapshots and reconnect behavior are realistic.
|
||||
with monkeypatch.context() as isolated:
|
||||
isolated.setitem(sys.modules, service_name, service_stub)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
def create_manager():
|
||||
return module.WebSocketManager(
|
||||
session_store=module.WorkflowSessionStore(),
|
||||
session_controller=MagicMock(),
|
||||
attachment_service=MagicMock(),
|
||||
workflow_run_service=MagicMock(),
|
||||
)
|
||||
|
||||
return create_manager
|
||||
|
||||
|
||||
class FakeWebSocket:
|
||||
@ -73,6 +67,9 @@ class FakeWebSocket:
|
||||
self.sent.append(data)
|
||||
self.send_threads.append(threading.get_ident())
|
||||
|
||||
async def close(self, **kwargs) -> None:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
@ -81,9 +78,9 @@ class FakeWebSocket:
|
||||
class TestSendMessageSync:
|
||||
"""send_message_sync must deliver messages regardless of calling thread."""
|
||||
|
||||
def test_send_from_main_thread(self):
|
||||
def test_send_from_main_thread(self, make_manager):
|
||||
"""Message sent from the main (event-loop) thread is delivered."""
|
||||
manager = _make_manager()
|
||||
manager = make_manager()
|
||||
ws = FakeWebSocket()
|
||||
delivered = []
|
||||
|
||||
@ -97,13 +94,17 @@ class TestSendMessageSync:
|
||||
await asyncio.sleep(0.05)
|
||||
delivered.extend(ws.sent)
|
||||
|
||||
asyncio.run(run())
|
||||
# Waiting on a thread-safe Future from its own loop would deadlock.
|
||||
# Fail immediately if that path is used, rather than waiting 10 seconds.
|
||||
with patch.object(concurrent.futures.Future, "result") as blocking_wait:
|
||||
asyncio.run(run())
|
||||
blocking_wait.assert_not_called()
|
||||
assert len(delivered) == 1
|
||||
assert '"test"' in delivered[0]
|
||||
|
||||
def test_send_from_worker_thread(self):
|
||||
def test_send_from_worker_thread(self, make_manager):
|
||||
"""Message sent from a background (worker) thread is delivered on the owner loop."""
|
||||
manager = _make_manager()
|
||||
manager = make_manager()
|
||||
ws = FakeWebSocket()
|
||||
worker_errors: List[Exception] = []
|
||||
|
||||
@ -136,14 +137,14 @@ class TestSendMessageSync:
|
||||
asyncio.run(run())
|
||||
assert not worker_errors, f"Worker thread raised: {worker_errors}"
|
||||
|
||||
def test_concurrent_workers_no_lost_messages(self):
|
||||
def test_concurrent_workers_no_lost_messages(self, make_manager):
|
||||
"""Multiple concurrent workers should each have their message delivered.
|
||||
|
||||
In production, the event loop is free while workers run (the main coroutine
|
||||
awaits ``run_in_executor``). We replicate that by polling workers via
|
||||
``asyncio.sleep`` so the loop can process the scheduled sends.
|
||||
"""
|
||||
manager = _make_manager()
|
||||
manager = make_manager()
|
||||
ws = FakeWebSocket()
|
||||
num_workers = 8
|
||||
|
||||
@ -152,7 +153,6 @@ class TestSendMessageSync:
|
||||
ws.sent.clear()
|
||||
|
||||
barrier = threading.Barrier(num_workers)
|
||||
done_count = threading.atomic(0) if hasattr(threading, "atomic") else None
|
||||
done_flags = [False] * num_workers
|
||||
|
||||
def worker(idx: int):
|
||||
@ -179,12 +179,35 @@ class TestSendMessageSync:
|
||||
assert len(ws.sent) == num_workers, (
|
||||
f"Expected {num_workers} messages, got {len(ws.sent)}"
|
||||
)
|
||||
assert sorted(json.loads(message)["idx"] for message in ws.sent) == list(range(num_workers))
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
def test_send_after_disconnect_does_not_crash(self):
|
||||
|
||||
class TestSessionReconnect:
|
||||
def test_reconnect_replays_messages_and_serializes_snapshot(self, make_manager):
|
||||
manager = make_manager()
|
||||
session = manager.session_store.create_session(
|
||||
session_id="reconnect", yaml_file="test.yaml", task_prompt="test",
|
||||
)
|
||||
buffered = {"type": "test", "data": "buffered"}
|
||||
session.append_message(buffered)
|
||||
ws = FakeWebSocket()
|
||||
|
||||
async def run():
|
||||
assert await manager.connect(ws, session_id=session.session_id) == session.session_id
|
||||
|
||||
asyncio.run(run())
|
||||
messages = [json.loads(message) for message in ws.sent]
|
||||
assert [message["type"] for message in messages] == ["connection", "test", "session_resumed"]
|
||||
assert messages[1] == buffered
|
||||
assert messages[2]["data"]["session_id"] == session.session_id
|
||||
assert messages[2]["data"]["status"] == "idle"
|
||||
assert messages[2]["data"]["message_count"] == 1
|
||||
|
||||
def test_send_after_disconnect_does_not_crash(self, make_manager):
|
||||
"""Sending after disconnection should not raise."""
|
||||
manager = _make_manager()
|
||||
manager = make_manager()
|
||||
ws = FakeWebSocket()
|
||||
|
||||
async def run():
|
||||
@ -197,9 +220,9 @@ class TestSendMessageSync:
|
||||
|
||||
asyncio.run(run()) # no exception == pass
|
||||
|
||||
def test_send_before_any_connection_no_crash(self):
|
||||
def test_send_before_any_connection_no_crash(self, make_manager):
|
||||
"""Calling send_message_sync before any connect() should not crash."""
|
||||
manager = _make_manager()
|
||||
manager = make_manager()
|
||||
# _owner_loop is None
|
||||
manager.send_message_sync("nonexistent", {"type": "orphan"})
|
||||
# Should log a warning, not crash
|
||||
@ -208,8 +231,8 @@ class TestSendMessageSync:
|
||||
class TestOwnerLoopCapture:
|
||||
"""The manager must capture the event loop on first connect."""
|
||||
|
||||
def test_owner_loop_captured_on_connect(self):
|
||||
manager = _make_manager()
|
||||
def test_owner_loop_captured_on_connect(self, make_manager):
|
||||
manager = make_manager()
|
||||
ws = FakeWebSocket()
|
||||
|
||||
async def run():
|
||||
@ -219,9 +242,9 @@ class TestOwnerLoopCapture:
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
def test_owner_loop_stable_across_connections(self):
|
||||
def test_owner_loop_stable_across_connections(self, make_manager):
|
||||
"""Subsequent connects should not reset the owner loop."""
|
||||
manager = _make_manager()
|
||||
manager = make_manager()
|
||||
ws1 = FakeWebSocket()
|
||||
ws2 = FakeWebSocket()
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user