fix(channels): rollback partial service startup across cancellation (#5537)

This commit is contained in:
NanPan 2026-09-19 10:29:02 +08:00 committed by GitHub
parent f33b4fb4bf
commit 74ab3cf818
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 77 additions and 3 deletions

View File

@ -15,6 +15,7 @@ from app.channels.manager import DEFAULT_CHANNEL_MAX_CONCURRENCY, DEFAULT_CHANNE
from app.channels.message_bus import DEFAULT_INBOUND_QUEUE_MAXSIZE, MessageBus
from app.channels.runtime_config_store import merge_runtime_channel_configs
from app.channels.store import ChannelStore
from deerflow.utils.file_io import await_drained
logger = logging.getLogger(__name__)
@ -573,9 +574,30 @@ async def start_channel_service(
# from_app_config reads the JSON channel store and runtime config files;
# keep that disk IO off the event loop. asyncio.to_thread forwards both
# args and kwargs to the target callable.
_channel_service = await asyncio.to_thread(ChannelService.from_app_config, app_config, get_stream_bridge=get_stream_bridge)
await _channel_service.start()
return _channel_service
service = await asyncio.to_thread(ChannelService.from_app_config, app_config, get_stream_bridge=get_stream_bridge)
_channel_service = service
async def rollback_failed_start() -> None:
global _channel_service
await service.stop()
if _channel_service is service:
_channel_service = None
try:
await service.start()
except BaseException:
try:
await await_drained(rollback_failed_start())
except asyncio.CancelledError:
# A repeated caller cancellation arrives only after the owned
# rollback has drained; preserve cancellation semantics.
raise
except Exception:
# Retain the singleton when cleanup itself fails so shutdown can
# retry it instead of orphaning partially-started resources.
logger.exception("Failed to stop ChannelService after startup failure; retaining singleton")
raise
return service
async def stop_channel_service() -> None:

View File

@ -0,0 +1,52 @@
import asyncio
from unittest.mock import MagicMock, patch
import pytest
@pytest.mark.asyncio
async def test_cancelled_start_drains_partial_service_and_clears_singleton() -> None:
import app.channels.service as service_module
start_entered = asyncio.Event()
stop_entered = asyncio.Event()
allow_stop = asyncio.Event()
class FakeService:
async def start(self) -> None:
start_entered.set()
await asyncio.Event().wait()
async def stop(self) -> None:
stop_entered.set()
await allow_stop.wait()
fake = FakeService()
service_module._channel_service = None
config = MagicMock()
try:
with patch.object(service_module.ChannelService, "from_app_config", return_value=fake):
task = asyncio.create_task(service_module.start_channel_service(config))
await asyncio.wait_for(start_entered.wait(), timeout=1)
task.cancel()
for _ in range(5):
await asyncio.sleep(0)
assert stop_entered.is_set(), "cancelled startup did not begin partial-service cleanup"
assert not task.done(), "startup returned before owned partial cleanup finished"
task.cancel()
for _ in range(5):
await asyncio.sleep(0)
assert not task.done(), "repeated cancellation abandoned partial-service cleanup"
allow_stop.set()
with pytest.raises(asyncio.CancelledError):
await task
assert service_module.get_channel_service() is None
finally:
allow_stop.set()
service_module._channel_service = None