mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-19 19:16:17 +00:00
fix(channels): rollback partial service startup across cancellation (#5537)
This commit is contained in:
parent
f33b4fb4bf
commit
74ab3cf818
@ -15,6 +15,7 @@ from app.channels.manager import DEFAULT_CHANNEL_MAX_CONCURRENCY, DEFAULT_CHANNE
|
||||
from app.channels.message_bus import DEFAULT_INBOUND_QUEUE_MAXSIZE, MessageBus
|
||||
from app.channels.runtime_config_store import merge_runtime_channel_configs
|
||||
from app.channels.store import ChannelStore
|
||||
from deerflow.utils.file_io import await_drained
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -573,9 +574,30 @@ async def start_channel_service(
|
||||
# from_app_config reads the JSON channel store and runtime config files;
|
||||
# keep that disk IO off the event loop. asyncio.to_thread forwards both
|
||||
# args and kwargs to the target callable.
|
||||
_channel_service = await asyncio.to_thread(ChannelService.from_app_config, app_config, get_stream_bridge=get_stream_bridge)
|
||||
await _channel_service.start()
|
||||
return _channel_service
|
||||
service = await asyncio.to_thread(ChannelService.from_app_config, app_config, get_stream_bridge=get_stream_bridge)
|
||||
_channel_service = service
|
||||
|
||||
async def rollback_failed_start() -> None:
|
||||
global _channel_service
|
||||
await service.stop()
|
||||
if _channel_service is service:
|
||||
_channel_service = None
|
||||
|
||||
try:
|
||||
await service.start()
|
||||
except BaseException:
|
||||
try:
|
||||
await await_drained(rollback_failed_start())
|
||||
except asyncio.CancelledError:
|
||||
# A repeated caller cancellation arrives only after the owned
|
||||
# rollback has drained; preserve cancellation semantics.
|
||||
raise
|
||||
except Exception:
|
||||
# Retain the singleton when cleanup itself fails so shutdown can
|
||||
# retry it instead of orphaning partially-started resources.
|
||||
logger.exception("Failed to stop ChannelService after startup failure; retaining singleton")
|
||||
raise
|
||||
return service
|
||||
|
||||
|
||||
async def stop_channel_service() -> None:
|
||||
|
||||
52
backend/tests/test_channel_service_startup_cancellation.py
Normal file
52
backend/tests/test_channel_service_startup_cancellation.py
Normal file
@ -0,0 +1,52 @@
|
||||
import asyncio
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_start_drains_partial_service_and_clears_singleton() -> None:
|
||||
import app.channels.service as service_module
|
||||
|
||||
start_entered = asyncio.Event()
|
||||
stop_entered = asyncio.Event()
|
||||
allow_stop = asyncio.Event()
|
||||
|
||||
class FakeService:
|
||||
async def start(self) -> None:
|
||||
start_entered.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
async def stop(self) -> None:
|
||||
stop_entered.set()
|
||||
await allow_stop.wait()
|
||||
|
||||
fake = FakeService()
|
||||
service_module._channel_service = None
|
||||
config = MagicMock()
|
||||
|
||||
try:
|
||||
with patch.object(service_module.ChannelService, "from_app_config", return_value=fake):
|
||||
task = asyncio.create_task(service_module.start_channel_service(config))
|
||||
await asyncio.wait_for(start_entered.wait(), timeout=1)
|
||||
|
||||
task.cancel()
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert stop_entered.is_set(), "cancelled startup did not begin partial-service cleanup"
|
||||
assert not task.done(), "startup returned before owned partial cleanup finished"
|
||||
|
||||
task.cancel()
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0)
|
||||
assert not task.done(), "repeated cancellation abandoned partial-service cleanup"
|
||||
|
||||
allow_stop.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
assert service_module.get_channel_service() is None
|
||||
finally:
|
||||
allow_stop.set()
|
||||
service_module._channel_service = None
|
||||
Loading…
x
Reference in New Issue
Block a user