mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-08-18 02:38:54 +00:00
72f033fbbe
* feat: add redis stream bridge * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * fix(gateway): address redis stream bridge review Redis was imported eagerly through deerflow.runtime and declared as a hard dependency, which made memory-only installs load redis.asyncio at startup and left the lazy factory import ineffective. Move redis behind an optional extra, remove the public eager re-export, and keep make_stream_bridge as the only runtime import path with an actionable install hint when the extra is missing. Because Docker deployments now default the stream bridge to Redis via DEER_FLOW_STREAM_BRIDGE_REDIS_URL, install the redis extra explicitly in Docker/dev container flows and teach the local uv-extra detector to infer redis from both stream_bridge.type and the Redis URL env var. This keeps Docker working while preserving slim non-Docker installs. Harden the Redis bridge by batching XREAD replay, replacing brittle ResponseError string matching with a single fallback to 0-0 for malformed Last-Event-ID values, documenting connection/retention/fail-hard behavior, and adding fake plus opt-in real Redis coverage for XADD/XREAD, replay, invalid IDs, and MAXLEN trimming. * fix(config): bump config version for stream bridge * fix redis stream bridge terminal handling * fix: repair uv.lock, format redis.py, and align Dockerfile extras test The uv.lock file was missing a closing bracket for the redis extras section, redis.py had a formatting issue caught by ruff, and the Dockerfile extras test did not account for the hardcoded --extra redis flag. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
60 lines
2.3 KiB
Python
60 lines
2.3 KiB
Python
"""Configuration for stream bridge."""
|
|
|
|
from typing import Literal
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
StreamBridgeType = Literal["memory", "redis"]
|
|
|
|
|
|
class StreamBridgeConfig(BaseModel):
|
|
"""Configuration for the stream bridge that connects agent workers to SSE endpoints."""
|
|
|
|
type: StreamBridgeType = Field(
|
|
default="memory",
|
|
description="Stream bridge backend type. 'memory' uses an in-process event log (single-process only). 'redis' uses Redis Streams for multi-worker Docker deployments.",
|
|
)
|
|
redis_url: str | None = Field(
|
|
default=None,
|
|
description="Redis URL for the redis stream bridge type. If omitted, DEER_FLOW_STREAM_BRIDGE_REDIS_URL, REDIS_URL, or redis://localhost:6379/0 is used.",
|
|
)
|
|
queue_maxsize: int = Field(
|
|
default=256,
|
|
description="Maximum number of events retained per run (memory bridge queue size / redis stream MAXLEN).",
|
|
)
|
|
max_connections: int | None = Field(
|
|
default=None,
|
|
description=(
|
|
"Max Redis connections in the pool for the redis stream bridge. Each live SSE "
|
|
"client holds one connection blocked in XREAD ... BLOCK for up to heartbeat_interval "
|
|
"(15s), so hundreds of concurrent clients open hundreds of connections. Leave unset "
|
|
"for redis-py's default (effectively unbounded), or set a ceiling sized for peak "
|
|
"concurrent SSE clients. Only applies to the redis bridge."
|
|
),
|
|
)
|
|
|
|
|
|
# Global configuration instance — None means no stream bridge is configured
|
|
# (falls back to memory with defaults).
|
|
_stream_bridge_config: StreamBridgeConfig | None = None
|
|
|
|
|
|
def get_stream_bridge_config() -> StreamBridgeConfig | None:
|
|
"""Get the current stream bridge configuration, or None if not configured."""
|
|
return _stream_bridge_config
|
|
|
|
|
|
def set_stream_bridge_config(config: StreamBridgeConfig | None) -> None:
|
|
"""Set the stream bridge configuration."""
|
|
global _stream_bridge_config
|
|
_stream_bridge_config = config
|
|
|
|
|
|
def load_stream_bridge_config_from_dict(config_dict: dict | None) -> None:
|
|
"""Load stream bridge configuration from a dictionary."""
|
|
global _stream_bridge_config
|
|
if config_dict is None:
|
|
_stream_bridge_config = None
|
|
return
|
|
_stream_bridge_config = StreamBridgeConfig(**config_dict)
|