diff --git a/backend/app/gateway/deps.py b/backend/app/gateway/deps.py index b4b6c06e72..d69fd2f5d8 100644 --- a/backend/app/gateway/deps.py +++ b/backend/app/gateway/deps.py @@ -526,8 +526,35 @@ async def _retry_recovered_run_stream_end( *, retry_delay: float, cleanup_delay: float, + stream_was_present: bool | None = None, ) -> None: await asyncio.sleep(max(0.0, retry_delay)) + if stream_was_present is True: + stream_exists = getattr(bridge, "stream_exists", None) + if callable(stream_exists): + try: + if not await stream_exists(run_id): + cleanup_expired_producer = getattr(bridge, "cleanup_expired_producer", None) + if callable(cleanup_expired_producer): + try: + await cleanup_expired_producer(run_id) + except Exception: + logger.warning( + "Failed to clean expired stream producer marker after retry for %s", + run_id, + exc_info=True, + ) + logger.debug( + "Skipping recovered stream end retry for %s: stream expired during retry delay", + run_id, + ) + return + except Exception: + logger.debug( + "Failed to recheck recovered stream existence for %s", + run_id, + exc_info=True, + ) await bridge.publish_end(run_id) await bridge.cleanup(run_id, delay=cleanup_delay) @@ -542,7 +569,7 @@ async def _publish_recovered_run_stream_end( """Terminate retained streams for runs recovered as orphaned.""" cleanup_tasks: list[tuple[str, asyncio.Task[None]]] = [] - def schedule_retry(record: RunRecord) -> bool: + def schedule_retry(record: RunRecord, *, stream_was_present: bool | None = None) -> bool: retry_delay = getattr( bridge, "producer_lease_retry_delay_seconds", @@ -556,6 +583,7 @@ async def _publish_recovered_run_stream_end( record.run_id, retry_delay=float(retry_delay), cleanup_delay=cleanup_delay, + stream_was_present=stream_was_present, ) ) task.add_done_callback( @@ -571,9 +599,11 @@ async def _publish_recovered_run_stream_end( for record in recovered_runs: stream_exists = getattr(bridge, "stream_exists", None) + stream_was_present: bool | None = None if stream_exists is not None: try: - if not await stream_exists(record.run_id): + stream_was_present = bool(await stream_exists(record.run_id)) + if not stream_was_present: logger.debug("Skipping recovered stream end for %s: stream already expired", record.run_id) cleanup_expired_producer = getattr( bridge, @@ -591,10 +621,12 @@ async def _publish_recovered_run_stream_end( ) cleaned = False if not cleaned: - schedule_retry(record) + schedule_retry(record, stream_was_present=False) continue except Exception: logger.debug("Failed to check recovered stream existence for %s", record.run_id, exc_info=True) + schedule_retry(record, stream_was_present=stream_was_present) + continue try: await bridge.publish_end(record.run_id) except Exception: @@ -603,7 +635,7 @@ async def _publish_recovered_run_stream_end( record.run_id, exc_info=True, ) - schedule_retry(record) + schedule_retry(record, stream_was_present=stream_was_present) continue task = asyncio.create_task(bridge.cleanup(record.run_id, delay=cleanup_delay)) task.add_done_callback(lambda task, run_id=record.run_id: _log_recovered_stream_cleanup_result(task, run_id)) diff --git a/backend/docs/STREAMING.md b/backend/docs/STREAMING.md index a53e250cc6..137d965d77 100644 --- a/backend/docs/STREAMING.md +++ b/backend/docs/STREAMING.md @@ -143,7 +143,7 @@ sequenceDiagram 关键组件: - `runtime/runs/worker.py::run_agent` — 在 `asyncio.Task` 里跑 `agent.astream()`,把每个 chunk 通过 `serialize(chunk, mode=mode)` 转成 JSON,再 `bridge.publish()`。 -- `runtime/stream_bridge` — 抽象 Queue。`publish/subscribe` 解耦生产者和消费者,支持 `Last-Event-ID` 重连、心跳、多订阅者 fan-out。`stream_bridge.heartbeat_interval_seconds`(默认 15 秒)是 bridge 实例的默认心跳周期,统一作用于 Gateway SSE、`/wait` 和内部 channel watcher;显式传给 `subscribe()` 的值仍可覆盖单次订阅。Memory 和 Redis 都只保留 `queue_maxsize` 条数据事件。`MemoryStreamBridge` 在分配事件 ID 前深拷贝发布的 payload,并在 stream Condition 外为每个订阅者返回独立深拷贝,因此生产者复用 payload 或订阅者修改数据都不会影响保留帧及重连重放;心跳与 END sentinel 保持对象身份。游标早于保留水位线时返回 `StreamGap`,不会从当前最早事件静默部分重放。Redis backend 会在写入数据或首个 END 时刷新 retained stream key TTL;启动恢复与基于 worker lease 的周期恢复共用 Gateway stream terminalization 路径:`RunManager` 先将 orphan run 持久化为 `error` 并写入显式的 `stop_reason=orphan_recovered`,随后 Gateway 发布 `END_SENTINEL` 并安排 stream cleanup。周期扫描、逐行状态写入和 Gateway callback 作为一个受监督的 single-flight 后台 task 执行;慢任务不会堆积,也不会阻塞唯一的 lease heartbeat。shutdown 优先收敛活跃 run,再处理恢复 task;尚未执行的延迟 stream cleanup 会改为立即删除。只有 runtime `yield` 前、无并发请求的启动恢复会把最新受影响 thread 标记为 error;周期恢复不做非原子的 thread 投影。普通 durable terminal status 本身仍不足以证明尾帧已发布;Redis producer 在 durable admission 前取得短租约,经 request task、worker、finalizer 转移并保留到 END。只有显式 `orphan_recovered` 或 durable terminal 加已失效的 tracked producer lease 才能在 heartbeat 上触发 END fallback;lease loss 会阻止后续 stream 写入,active 或 untracked producer 继续等待,避免截断尾帧。Redis END 通过同一个 Lua 原子完成 lease 校验、既有 END 检查和必要写入,因此重复调用、owner/recovery 竞争和 ACK 丢失重试至多追加一个 END;当 END 已位于 stream 尾部时,后续事件写入会被拒绝;旧 stream 中 END 后已有迟到数据时,仍允许继续写入数据。cleanup 后无 owner 的迟到 END 重试是成功 no-op,不会复活已删除的 stream。普通事件只做 O(1) 尾部 END 检查。`stream_bridge.legacy_end_scan_enabled` 默认 `true`:尾部不是 END 时扫描整个保留窗口,包括正常 run 的首次 END;原子 Lua 的耗时随保留条目数量和 payload 总字节数增长,期间会阻塞其他 Redis 客户端。只有全部旧 writer 停止且旧 replay buffer 已过期或清除后,才可设为 `false` 并重启所有 Gateway,使 END 检查也只看尾部。TTL 为 0 时必须显式清除旧 buffer。TTL 仍是 Redis 内存和故障安全网,不是正常的 subscriber 终止机制。 +- `runtime/stream_bridge` — 抽象 Queue。`publish/subscribe` 解耦生产者和消费者,支持 `Last-Event-ID` 重连、心跳、多订阅者 fan-out。`stream_bridge.heartbeat_interval_seconds`(默认 15 秒)是 bridge 实例的默认心跳周期,统一作用于 Gateway SSE、`/wait` 和内部 channel watcher;显式传给 `subscribe()` 的值仍可覆盖单次订阅。Memory 和 Redis 都只保留 `queue_maxsize` 条数据事件。Memory 发布在 Condition 内再次检查捕获的 generation 是否已 retired 或 ended,只有通过检查后才分配事件 ID;cleanup 或 END 先取得锁时,已排队的迟到写入会被丢弃,不会消耗序号,也不会出现在 END 之后。`MemoryStreamBridge` 在分配事件 ID 前深拷贝发布的 payload,并在 stream Condition 外为每个订阅者返回独立深拷贝,因此生产者复用 payload 或订阅者修改数据都不会影响保留帧及重连重放;心跳与 END sentinel 保持对象身份。游标早于保留水位线时返回 `StreamGap`,不会从当前最早事件静默部分重放。Redis backend 会在写入数据或首个 END 时刷新 retained stream key TTL;启动恢复与基于 worker lease 的周期恢复共用 Gateway stream terminalization 路径:`RunManager` 先将 orphan run 持久化为 `error` 并写入显式的 `stop_reason=orphan_recovered`,随后 Gateway 发布 `END_SENTINEL` 并安排 stream cleanup。周期扫描、逐行状态写入和 Gateway callback 作为一个受监督的 single-flight 后台 task 执行;慢任务不会堆积,也不会阻塞唯一的 lease heartbeat。shutdown 优先收敛活跃 run,再处理恢复 task;尚未执行的延迟 stream cleanup 会改为立即删除。只有 runtime `yield` 前、无并发请求的启动恢复会把最新受影响 thread 标记为 error;周期恢复不做非原子的 thread 投影。普通 durable terminal status 本身仍不足以证明尾帧已发布;Redis producer 在 durable admission 前取得短租约,经 request task、worker、finalizer 转移并保留到 END。只有显式 `orphan_recovered` 或 durable terminal 加已失效的 tracked producer lease 才能在 heartbeat 上触发 END fallback;lease loss 会阻止后续 stream 写入,active 或 untracked producer 继续等待,避免截断尾帧。Redis END 通过同一个 Lua 原子完成 lease 校验、既有 END 检查和必要写入,因此重复调用、owner/recovery 竞争和 ACK 丢失重试至多追加一个 END;当 END 已位于 stream 尾部时,后续事件写入会被拒绝;旧 stream 中 END 后已有迟到数据时,仍允许继续写入数据。cleanup 后无 owner 的迟到 END 重试是成功 no-op,不会复活已删除的 stream。普通事件只做 O(1) 尾部 END 检查。`stream_bridge.legacy_end_scan_enabled` 默认 `true`:尾部不是 END 时扫描整个保留窗口,包括正常 run 的首次 END;原子 Lua 的耗时随保留条目数量和 payload 总字节数增长,期间会阻塞其他 Redis 客户端。只有全部旧 writer 停止且旧 replay buffer 已过期或清除后,才可设为 `false` 并重启所有 Gateway,使 END 检查也只看尾部。TTL 为 0 时必须显式清除旧 buffer。TTL 仍是 Redis 内存和故障安全网,不是正常的 subscriber 终止机制。 - `stream_bridge.producer_lease_recovery_enabled` 是上述 producer-lease fallback 的 restart-required activate gate。升级时先保持 `false` 完成所有 writer 的滚动发布并排空旧 run,再在所有 Gateway 上同时设为 `true` 并重启;mixed-version 阶段不得启用。 - `app/gateway/services.py::sse_consumer` — 从 bridge 订阅,格式化为 SSE wire 帧。 - `runtime/serialization.py::serialize` — mode-aware 序列化;`messages` mode 下 `serialize_messages_tuple` 把 `(chunk, metadata)` 转成 `[chunk.model_dump(), metadata]`。 diff --git a/backend/packages/harness/deerflow/runtime/AGENTS.md b/backend/packages/harness/deerflow/runtime/AGENTS.md index ceb09baacd..8355082a0f 100644 --- a/backend/packages/harness/deerflow/runtime/AGENTS.md +++ b/backend/packages/harness/deerflow/runtime/AGENTS.md @@ -34,6 +34,8 @@ make test-blocking-io would overwrite the thread view of an SDK client. Consumers that only handle the root run ignore namespaced frames. +- **Memory writers fence the captured generation.** After waiting for the stream Condition, check `retired` and `ended` before allocating an event ID or appending; rejected writes consume no sequence number, and END is idempotent for that generation. + ## Checkpoints - **The channel mode is fixed for the life of the process.** A reader in `full` diff --git a/backend/packages/harness/deerflow/runtime/stream_bridge/memory.py b/backend/packages/harness/deerflow/runtime/stream_bridge/memory.py index c784e85861..895e19ba03 100644 --- a/backend/packages/harness/deerflow/runtime/stream_bridge/memory.py +++ b/backend/packages/harness/deerflow/runtime/stream_bridge/memory.py @@ -119,8 +119,11 @@ class MemoryStreamBridge(StreamBridge): async def publish(self, run_id: str, event: str, data: Any) -> None: data = deepcopy(data) stream = self._get_or_create_stream(run_id) - entry = StreamEvent(id=self._next_id(run_id), event=event, data=data) async with stream.condition: + # Cleanup or END may win the Condition after this producer captures the generation. + if stream.retired or stream.ended: + return + entry = StreamEvent(id=self._next_id(run_id), event=event, data=data) stream.events.append(entry) if len(stream.events) > self._maxsize: overflow = len(stream.events) - self._maxsize @@ -132,6 +135,8 @@ class MemoryStreamBridge(StreamBridge): async def publish_end(self, run_id: str) -> None: stream = self._get_or_create_stream(run_id) async with stream.condition: + if stream.retired or stream.ended: + return stream.ended = True stream.condition.notify_all() diff --git a/backend/tests/test_gateway_run_recovery.py b/backend/tests/test_gateway_run_recovery.py index 53a748c442..b85af662ec 100644 --- a/backend/tests/test_gateway_run_recovery.py +++ b/backend/tests/test_gateway_run_recovery.py @@ -408,3 +408,22 @@ async def test_sqlite_runtime_does_not_mark_thread_error_when_newer_run_is_succe assert thread_store.status_updates == [] assert stream_bridge.publish_end_calls == ["old-running"] assert stream_bridge.cleanup_calls == [("old-running", 60.0)] + + +@pytest.mark.anyio +async def test_recovered_stream_end_retry_does_not_recreate_stream_cleaned_during_delay(): + stream_bridge = _RetryingRecoveredStreamBridge() + + cleanups = await gateway_deps._publish_recovered_run_stream_end( + stream_bridge, + [SimpleNamespace(run_id="run-1", thread_id="thread-1")], + cleanup_delay=0, + ) + # Model another owner/retention cleanup winning before the delayed retry. + stream_bridge._existing_streams.clear() + await asyncio.gather(*(task for _run_id, task in cleanups)) + + assert stream_bridge.publish_attempts == 1 + assert stream_bridge.publish_end_calls == [] + assert stream_bridge.cleanup_calls == [] + assert stream_bridge.expired_producer_cleanup_calls == ["run-1"] diff --git a/backend/tests/test_memory_stream_retirement.py b/backend/tests/test_memory_stream_retirement.py index d43952f93f..90342924d5 100644 --- a/backend/tests/test_memory_stream_retirement.py +++ b/backend/tests/test_memory_stream_retirement.py @@ -348,3 +348,77 @@ async def test_gateway_wait_finishes_from_terminal_state_after_retirement(store_ finally: await bridge.close() await manager.shutdown() + + +async def _publish_behind_retirement(bridge, run_id, monkeypatch): + """Queue cleanup before a producer write on the captured generation.""" + condition = bridge._streams[run_id].condition + real_acquire = condition.acquire + retirement_queued = asyncio.Event() + publication_queued = asyncio.Event() + tasks = [] + + async def observe_acquire(): + if asyncio.current_task() is tasks[0]: + retirement_queued.set() + else: + publication_queued.set() + return await real_acquire() + + try: + async with condition: + monkeypatch.setattr(condition, "acquire", observe_acquire) + tasks.append(asyncio.create_task(bridge.cleanup(run_id))) + await asyncio.wait_for(retirement_queued.wait(), 1) + tasks.append(asyncio.create_task(bridge.publish(run_id, "late", {"value": "late"}))) + await asyncio.wait_for(publication_queued.wait(), 1) + await asyncio.wait_for(asyncio.shield(tasks[0]), 1) + await asyncio.wait_for(asyncio.shield(tasks[1]), 1) + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_publish_queued_behind_cleanup_is_discarded_from_the_retired_generation(monkeypatch): + bridge = MemoryStreamBridge() + run_id = "retired-publish" + await bridge.publish(run_id, "before", {"value": "before"}) + stream = bridge._streams[run_id] + try: + await _publish_behind_retirement(bridge, run_id, monkeypatch) + + assert stream.retired + assert stream.events == [] + assert not await bridge.stream_exists(run_id) + + await bridge.publish(run_id, "fresh", {"value": "fresh"}) + current = bridge._streams[run_id] + assert current is not stream + assert current.events[0].id.rsplit("-", 1)[-1] == "0" + finally: + await bridge.close() + + +@pytest.mark.asyncio +async def test_publish_after_end_is_ignored_without_consuming_an_event_id(): + bridge = MemoryStreamBridge() + run_id = "terminal-publish" + try: + await bridge.publish(run_id, "before", {"value": "before"}) + stream = bridge._streams[run_id] + event_count = bridge._counters[run_id] + + await bridge.publish_end(run_id) + await bridge.publish_end(run_id) + await bridge.publish(run_id, "late", {"value": "late"}) + + assert bridge._counters[run_id] == event_count + assert [entry.event for entry in stream.events] == ["before"] + + replay = [item async for item in bridge.subscribe(run_id)] + assert [item.event for item in replay[:-1]] == ["before"] + assert replay[-1] is END_SENTINEL + finally: + await bridge.close()