mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-02 00:20:40 +00:00
aabfdeebc9
* fix(streaming): make Redis stream END publication idempotent * fix(streaming): align Redis test fake with Lua END checks Remove the unreachable owned missing-stream branch, mirror tail-only data checks in both fake script paths, and pin legacy END and owned missing-buffer behavior against real Redis. * fix(streaming): gate legacy END scans after writer rollout * fix(helm): synchronize embedded config version with schema * test(runtime): use real stream bridge config in close coverage Keep new configuration defaults in lifecycle fixtures and surface owner-task failures instead of masking them as event timeouts. Drain every helper task and cover context entry/close failures. --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
109 lines
3.6 KiB
Python
109 lines
3.6 KiB
Python
"""Async stream bridge factory.
|
|
|
|
Provides an **async context manager** aligned with
|
|
:func:`deerflow.runtime.checkpointer.async_provider.make_checkpointer`.
|
|
|
|
Usage (e.g. FastAPI lifespan)::
|
|
|
|
from deerflow.agents.stream_bridge import make_stream_bridge
|
|
|
|
async with make_stream_bridge() as bridge:
|
|
app.state.stream_bridge = bridge
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import logging
|
|
import os
|
|
from collections.abc import AsyncIterator
|
|
|
|
from deerflow.config.app_config import AppConfig
|
|
from deerflow.config.stream_bridge_config import StreamBridgeConfig, get_stream_bridge_config
|
|
from deerflow.utils.file_io import await_drained
|
|
|
|
from .base import DEFAULT_HEARTBEAT_INTERVAL_SECONDS, StreamBridge
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_ENV_REDIS_URL = "DEER_FLOW_STREAM_BRIDGE_REDIS_URL"
|
|
|
|
|
|
def _resolve_config(app_config: AppConfig | None) -> StreamBridgeConfig | None:
|
|
if app_config is None:
|
|
config = get_stream_bridge_config()
|
|
else:
|
|
config = app_config.stream_bridge
|
|
|
|
if config is None:
|
|
redis_url = os.getenv(_ENV_REDIS_URL)
|
|
if redis_url:
|
|
return StreamBridgeConfig(type="redis", redis_url=redis_url)
|
|
return config
|
|
|
|
|
|
def _resolve_redis_url(config: StreamBridgeConfig) -> str:
|
|
return config.redis_url or os.getenv(_ENV_REDIS_URL) or os.getenv("REDIS_URL") or "redis://localhost:6379/0"
|
|
|
|
|
|
@contextlib.asynccontextmanager
|
|
async def make_stream_bridge(app_config: AppConfig | None = None) -> AsyncIterator[StreamBridge]:
|
|
"""Async context manager that yields a :class:`StreamBridge`.
|
|
|
|
Falls back to :class:`MemoryStreamBridge` when no configuration is
|
|
provided and nothing is set globally.
|
|
"""
|
|
config = _resolve_config(app_config)
|
|
|
|
if config is None or config.type == "memory":
|
|
from deerflow.runtime.stream_bridge.memory import MemoryStreamBridge
|
|
|
|
maxsize = config.queue_maxsize if config is not None else 256
|
|
heartbeat_interval = config.heartbeat_interval_seconds if config is not None else DEFAULT_HEARTBEAT_INTERVAL_SECONDS
|
|
bridge = MemoryStreamBridge(
|
|
queue_maxsize=maxsize,
|
|
heartbeat_interval=heartbeat_interval,
|
|
)
|
|
logger.info(
|
|
"Stream bridge initialised: memory (queue_maxsize=%d, heartbeat_interval_seconds=%.1f)",
|
|
maxsize,
|
|
heartbeat_interval,
|
|
)
|
|
try:
|
|
yield bridge
|
|
finally:
|
|
await await_drained(bridge.close())
|
|
return
|
|
|
|
if config.type == "redis":
|
|
from deerflow.runtime.stream_bridge.redis import RedisStreamBridge
|
|
|
|
redis_url = _resolve_redis_url(config)
|
|
bridge = RedisStreamBridge(
|
|
redis_url=redis_url,
|
|
queue_maxsize=config.queue_maxsize,
|
|
heartbeat_interval=config.heartbeat_interval_seconds,
|
|
max_connections=config.max_connections,
|
|
stream_ttl_seconds=config.stream_ttl_seconds,
|
|
legacy_end_scan_enabled=config.legacy_end_scan_enabled,
|
|
producer_lease_recovery_enabled=getattr(
|
|
config,
|
|
"producer_lease_recovery_enabled",
|
|
False,
|
|
),
|
|
)
|
|
logger.info(
|
|
"Stream bridge initialised: redis (queue_maxsize=%d, heartbeat_interval_seconds=%.1f, max_connections=%s, stream_ttl_seconds=%d)",
|
|
config.queue_maxsize,
|
|
config.heartbeat_interval_seconds,
|
|
config.max_connections,
|
|
config.stream_ttl_seconds,
|
|
)
|
|
try:
|
|
yield bridge
|
|
finally:
|
|
await await_drained(bridge.close())
|
|
return
|
|
|
|
raise ValueError(f"Unknown stream bridge type: {config.type!r}")
|