mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
fix(streaming): fence memory writes after retirement or END (#7766)
* Rebase stream lifecycle fix onto upstream main * Preserve gateway config snapshot and recovered stream retention guard
This commit is contained in:
parent
4cd47819a6
commit
13edf25b89
@ -526,8 +526,35 @@ async def _retry_recovered_run_stream_end(
|
||||
*,
|
||||
retry_delay: float,
|
||||
cleanup_delay: float,
|
||||
stream_was_present: bool | None = None,
|
||||
) -> None:
|
||||
await asyncio.sleep(max(0.0, retry_delay))
|
||||
if stream_was_present is True:
|
||||
stream_exists = getattr(bridge, "stream_exists", None)
|
||||
if callable(stream_exists):
|
||||
try:
|
||||
if not await stream_exists(run_id):
|
||||
cleanup_expired_producer = getattr(bridge, "cleanup_expired_producer", None)
|
||||
if callable(cleanup_expired_producer):
|
||||
try:
|
||||
await cleanup_expired_producer(run_id)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to clean expired stream producer marker after retry for %s",
|
||||
run_id,
|
||||
exc_info=True,
|
||||
)
|
||||
logger.debug(
|
||||
"Skipping recovered stream end retry for %s: stream expired during retry delay",
|
||||
run_id,
|
||||
)
|
||||
return
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Failed to recheck recovered stream existence for %s",
|
||||
run_id,
|
||||
exc_info=True,
|
||||
)
|
||||
await bridge.publish_end(run_id)
|
||||
await bridge.cleanup(run_id, delay=cleanup_delay)
|
||||
|
||||
@ -542,7 +569,7 @@ async def _publish_recovered_run_stream_end(
|
||||
"""Terminate retained streams for runs recovered as orphaned."""
|
||||
cleanup_tasks: list[tuple[str, asyncio.Task[None]]] = []
|
||||
|
||||
def schedule_retry(record: RunRecord) -> bool:
|
||||
def schedule_retry(record: RunRecord, *, stream_was_present: bool | None = None) -> bool:
|
||||
retry_delay = getattr(
|
||||
bridge,
|
||||
"producer_lease_retry_delay_seconds",
|
||||
@ -556,6 +583,7 @@ async def _publish_recovered_run_stream_end(
|
||||
record.run_id,
|
||||
retry_delay=float(retry_delay),
|
||||
cleanup_delay=cleanup_delay,
|
||||
stream_was_present=stream_was_present,
|
||||
)
|
||||
)
|
||||
task.add_done_callback(
|
||||
@ -571,9 +599,11 @@ async def _publish_recovered_run_stream_end(
|
||||
|
||||
for record in recovered_runs:
|
||||
stream_exists = getattr(bridge, "stream_exists", None)
|
||||
stream_was_present: bool | None = None
|
||||
if stream_exists is not None:
|
||||
try:
|
||||
if not await stream_exists(record.run_id):
|
||||
stream_was_present = bool(await stream_exists(record.run_id))
|
||||
if not stream_was_present:
|
||||
logger.debug("Skipping recovered stream end for %s: stream already expired", record.run_id)
|
||||
cleanup_expired_producer = getattr(
|
||||
bridge,
|
||||
@ -591,10 +621,12 @@ async def _publish_recovered_run_stream_end(
|
||||
)
|
||||
cleaned = False
|
||||
if not cleaned:
|
||||
schedule_retry(record)
|
||||
schedule_retry(record, stream_was_present=False)
|
||||
continue
|
||||
except Exception:
|
||||
logger.debug("Failed to check recovered stream existence for %s", record.run_id, exc_info=True)
|
||||
schedule_retry(record, stream_was_present=stream_was_present)
|
||||
continue
|
||||
try:
|
||||
await bridge.publish_end(record.run_id)
|
||||
except Exception:
|
||||
@ -603,7 +635,7 @@ async def _publish_recovered_run_stream_end(
|
||||
record.run_id,
|
||||
exc_info=True,
|
||||
)
|
||||
schedule_retry(record)
|
||||
schedule_retry(record, stream_was_present=stream_was_present)
|
||||
continue
|
||||
task = asyncio.create_task(bridge.cleanup(record.run_id, delay=cleanup_delay))
|
||||
task.add_done_callback(lambda task, run_id=record.run_id: _log_recovered_stream_cleanup_result(task, run_id))
|
||||
|
||||
@ -143,7 +143,7 @@ sequenceDiagram
|
||||
关键组件:
|
||||
|
||||
- `runtime/runs/worker.py::run_agent` — 在 `asyncio.Task` 里跑 `agent.astream()`,把每个 chunk 通过 `serialize(chunk, mode=mode)` 转成 JSON,再 `bridge.publish()`。
|
||||
- `runtime/stream_bridge` — 抽象 Queue。`publish/subscribe` 解耦生产者和消费者,支持 `Last-Event-ID` 重连、心跳、多订阅者 fan-out。`stream_bridge.heartbeat_interval_seconds`(默认 15 秒)是 bridge 实例的默认心跳周期,统一作用于 Gateway SSE、`/wait` 和内部 channel watcher;显式传给 `subscribe()` 的值仍可覆盖单次订阅。Memory 和 Redis 都只保留 `queue_maxsize` 条数据事件。`MemoryStreamBridge` 在分配事件 ID 前深拷贝发布的 payload,并在 stream Condition 外为每个订阅者返回独立深拷贝,因此生产者复用 payload 或订阅者修改数据都不会影响保留帧及重连重放;心跳与 END sentinel 保持对象身份。游标早于保留水位线时返回 `StreamGap`,不会从当前最早事件静默部分重放。Redis backend 会在写入数据或首个 END 时刷新 retained stream key TTL;启动恢复与基于 worker lease 的周期恢复共用 Gateway stream terminalization 路径:`RunManager` 先将 orphan run 持久化为 `error` 并写入显式的 `stop_reason=orphan_recovered`,随后 Gateway 发布 `END_SENTINEL` 并安排 stream cleanup。周期扫描、逐行状态写入和 Gateway callback 作为一个受监督的 single-flight 后台 task 执行;慢任务不会堆积,也不会阻塞唯一的 lease heartbeat。shutdown 优先收敛活跃 run,再处理恢复 task;尚未执行的延迟 stream cleanup 会改为立即删除。只有 runtime `yield` 前、无并发请求的启动恢复会把最新受影响 thread 标记为 error;周期恢复不做非原子的 thread 投影。普通 durable terminal status 本身仍不足以证明尾帧已发布;Redis producer 在 durable admission 前取得短租约,经 request task、worker、finalizer 转移并保留到 END。只有显式 `orphan_recovered` 或 durable terminal 加已失效的 tracked producer lease 才能在 heartbeat 上触发 END fallback;lease loss 会阻止后续 stream 写入,active 或 untracked producer 继续等待,避免截断尾帧。Redis END 通过同一个 Lua 原子完成 lease 校验、既有 END 检查和必要写入,因此重复调用、owner/recovery 竞争和 ACK 丢失重试至多追加一个 END;当 END 已位于 stream 尾部时,后续事件写入会被拒绝;旧 stream 中 END 后已有迟到数据时,仍允许继续写入数据。cleanup 后无 owner 的迟到 END 重试是成功 no-op,不会复活已删除的 stream。普通事件只做 O(1) 尾部 END 检查。`stream_bridge.legacy_end_scan_enabled` 默认 `true`:尾部不是 END 时扫描整个保留窗口,包括正常 run 的首次 END;原子 Lua 的耗时随保留条目数量和 payload 总字节数增长,期间会阻塞其他 Redis 客户端。只有全部旧 writer 停止且旧 replay buffer 已过期或清除后,才可设为 `false` 并重启所有 Gateway,使 END 检查也只看尾部。TTL 为 0 时必须显式清除旧 buffer。TTL 仍是 Redis 内存和故障安全网,不是正常的 subscriber 终止机制。
|
||||
- `runtime/stream_bridge` — 抽象 Queue。`publish/subscribe` 解耦生产者和消费者,支持 `Last-Event-ID` 重连、心跳、多订阅者 fan-out。`stream_bridge.heartbeat_interval_seconds`(默认 15 秒)是 bridge 实例的默认心跳周期,统一作用于 Gateway SSE、`/wait` 和内部 channel watcher;显式传给 `subscribe()` 的值仍可覆盖单次订阅。Memory 和 Redis 都只保留 `queue_maxsize` 条数据事件。Memory 发布在 Condition 内再次检查捕获的 generation 是否已 retired 或 ended,只有通过检查后才分配事件 ID;cleanup 或 END 先取得锁时,已排队的迟到写入会被丢弃,不会消耗序号,也不会出现在 END 之后。`MemoryStreamBridge` 在分配事件 ID 前深拷贝发布的 payload,并在 stream Condition 外为每个订阅者返回独立深拷贝,因此生产者复用 payload 或订阅者修改数据都不会影响保留帧及重连重放;心跳与 END sentinel 保持对象身份。游标早于保留水位线时返回 `StreamGap`,不会从当前最早事件静默部分重放。Redis backend 会在写入数据或首个 END 时刷新 retained stream key TTL;启动恢复与基于 worker lease 的周期恢复共用 Gateway stream terminalization 路径:`RunManager` 先将 orphan run 持久化为 `error` 并写入显式的 `stop_reason=orphan_recovered`,随后 Gateway 发布 `END_SENTINEL` 并安排 stream cleanup。周期扫描、逐行状态写入和 Gateway callback 作为一个受监督的 single-flight 后台 task 执行;慢任务不会堆积,也不会阻塞唯一的 lease heartbeat。shutdown 优先收敛活跃 run,再处理恢复 task;尚未执行的延迟 stream cleanup 会改为立即删除。只有 runtime `yield` 前、无并发请求的启动恢复会把最新受影响 thread 标记为 error;周期恢复不做非原子的 thread 投影。普通 durable terminal status 本身仍不足以证明尾帧已发布;Redis producer 在 durable admission 前取得短租约,经 request task、worker、finalizer 转移并保留到 END。只有显式 `orphan_recovered` 或 durable terminal 加已失效的 tracked producer lease 才能在 heartbeat 上触发 END fallback;lease loss 会阻止后续 stream 写入,active 或 untracked producer 继续等待,避免截断尾帧。Redis END 通过同一个 Lua 原子完成 lease 校验、既有 END 检查和必要写入,因此重复调用、owner/recovery 竞争和 ACK 丢失重试至多追加一个 END;当 END 已位于 stream 尾部时,后续事件写入会被拒绝;旧 stream 中 END 后已有迟到数据时,仍允许继续写入数据。cleanup 后无 owner 的迟到 END 重试是成功 no-op,不会复活已删除的 stream。普通事件只做 O(1) 尾部 END 检查。`stream_bridge.legacy_end_scan_enabled` 默认 `true`:尾部不是 END 时扫描整个保留窗口,包括正常 run 的首次 END;原子 Lua 的耗时随保留条目数量和 payload 总字节数增长,期间会阻塞其他 Redis 客户端。只有全部旧 writer 停止且旧 replay buffer 已过期或清除后,才可设为 `false` 并重启所有 Gateway,使 END 检查也只看尾部。TTL 为 0 时必须显式清除旧 buffer。TTL 仍是 Redis 内存和故障安全网,不是正常的 subscriber 终止机制。
|
||||
- `stream_bridge.producer_lease_recovery_enabled` 是上述 producer-lease fallback 的 restart-required activate gate。升级时先保持 `false` 完成所有 writer 的滚动发布并排空旧 run,再在所有 Gateway 上同时设为 `true` 并重启;mixed-version 阶段不得启用。
|
||||
- `app/gateway/services.py::sse_consumer` — 从 bridge 订阅,格式化为 SSE wire 帧。
|
||||
- `runtime/serialization.py::serialize` — mode-aware 序列化;`messages` mode 下 `serialize_messages_tuple` 把 `(chunk, metadata)` 转成 `[chunk.model_dump(), metadata]`。
|
||||
|
||||
@ -34,6 +34,8 @@ make test-blocking-io
|
||||
would overwrite the thread view of an SDK client. Consumers that only handle the
|
||||
root run ignore namespaced frames.
|
||||
|
||||
- **Memory writers fence the captured generation.** After waiting for the stream Condition, check `retired` and `ended` before allocating an event ID or appending; rejected writes consume no sequence number, and END is idempotent for that generation.
|
||||
|
||||
## Checkpoints
|
||||
|
||||
- **The channel mode is fixed for the life of the process.** A reader in `full`
|
||||
|
||||
@ -119,8 +119,11 @@ class MemoryStreamBridge(StreamBridge):
|
||||
async def publish(self, run_id: str, event: str, data: Any) -> None:
|
||||
data = deepcopy(data)
|
||||
stream = self._get_or_create_stream(run_id)
|
||||
entry = StreamEvent(id=self._next_id(run_id), event=event, data=data)
|
||||
async with stream.condition:
|
||||
# Cleanup or END may win the Condition after this producer captures the generation.
|
||||
if stream.retired or stream.ended:
|
||||
return
|
||||
entry = StreamEvent(id=self._next_id(run_id), event=event, data=data)
|
||||
stream.events.append(entry)
|
||||
if len(stream.events) > self._maxsize:
|
||||
overflow = len(stream.events) - self._maxsize
|
||||
@ -132,6 +135,8 @@ class MemoryStreamBridge(StreamBridge):
|
||||
async def publish_end(self, run_id: str) -> None:
|
||||
stream = self._get_or_create_stream(run_id)
|
||||
async with stream.condition:
|
||||
if stream.retired or stream.ended:
|
||||
return
|
||||
stream.ended = True
|
||||
stream.condition.notify_all()
|
||||
|
||||
|
||||
@ -408,3 +408,22 @@ async def test_sqlite_runtime_does_not_mark_thread_error_when_newer_run_is_succe
|
||||
assert thread_store.status_updates == []
|
||||
assert stream_bridge.publish_end_calls == ["old-running"]
|
||||
assert stream_bridge.cleanup_calls == [("old-running", 60.0)]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_recovered_stream_end_retry_does_not_recreate_stream_cleaned_during_delay():
|
||||
stream_bridge = _RetryingRecoveredStreamBridge()
|
||||
|
||||
cleanups = await gateway_deps._publish_recovered_run_stream_end(
|
||||
stream_bridge,
|
||||
[SimpleNamespace(run_id="run-1", thread_id="thread-1")],
|
||||
cleanup_delay=0,
|
||||
)
|
||||
# Model another owner/retention cleanup winning before the delayed retry.
|
||||
stream_bridge._existing_streams.clear()
|
||||
await asyncio.gather(*(task for _run_id, task in cleanups))
|
||||
|
||||
assert stream_bridge.publish_attempts == 1
|
||||
assert stream_bridge.publish_end_calls == []
|
||||
assert stream_bridge.cleanup_calls == []
|
||||
assert stream_bridge.expired_producer_cleanup_calls == ["run-1"]
|
||||
|
||||
@ -348,3 +348,77 @@ async def test_gateway_wait_finishes_from_terminal_state_after_retirement(store_
|
||||
finally:
|
||||
await bridge.close()
|
||||
await manager.shutdown()
|
||||
|
||||
|
||||
async def _publish_behind_retirement(bridge, run_id, monkeypatch):
|
||||
"""Queue cleanup before a producer write on the captured generation."""
|
||||
condition = bridge._streams[run_id].condition
|
||||
real_acquire = condition.acquire
|
||||
retirement_queued = asyncio.Event()
|
||||
publication_queued = asyncio.Event()
|
||||
tasks = []
|
||||
|
||||
async def observe_acquire():
|
||||
if asyncio.current_task() is tasks[0]:
|
||||
retirement_queued.set()
|
||||
else:
|
||||
publication_queued.set()
|
||||
return await real_acquire()
|
||||
|
||||
try:
|
||||
async with condition:
|
||||
monkeypatch.setattr(condition, "acquire", observe_acquire)
|
||||
tasks.append(asyncio.create_task(bridge.cleanup(run_id)))
|
||||
await asyncio.wait_for(retirement_queued.wait(), 1)
|
||||
tasks.append(asyncio.create_task(bridge.publish(run_id, "late", {"value": "late"})))
|
||||
await asyncio.wait_for(publication_queued.wait(), 1)
|
||||
await asyncio.wait_for(asyncio.shield(tasks[0]), 1)
|
||||
await asyncio.wait_for(asyncio.shield(tasks[1]), 1)
|
||||
finally:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_publish_queued_behind_cleanup_is_discarded_from_the_retired_generation(monkeypatch):
|
||||
bridge = MemoryStreamBridge()
|
||||
run_id = "retired-publish"
|
||||
await bridge.publish(run_id, "before", {"value": "before"})
|
||||
stream = bridge._streams[run_id]
|
||||
try:
|
||||
await _publish_behind_retirement(bridge, run_id, monkeypatch)
|
||||
|
||||
assert stream.retired
|
||||
assert stream.events == []
|
||||
assert not await bridge.stream_exists(run_id)
|
||||
|
||||
await bridge.publish(run_id, "fresh", {"value": "fresh"})
|
||||
current = bridge._streams[run_id]
|
||||
assert current is not stream
|
||||
assert current.events[0].id.rsplit("-", 1)[-1] == "0"
|
||||
finally:
|
||||
await bridge.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_publish_after_end_is_ignored_without_consuming_an_event_id():
|
||||
bridge = MemoryStreamBridge()
|
||||
run_id = "terminal-publish"
|
||||
try:
|
||||
await bridge.publish(run_id, "before", {"value": "before"})
|
||||
stream = bridge._streams[run_id]
|
||||
event_count = bridge._counters[run_id]
|
||||
|
||||
await bridge.publish_end(run_id)
|
||||
await bridge.publish_end(run_id)
|
||||
await bridge.publish(run_id, "late", {"value": "late"})
|
||||
|
||||
assert bridge._counters[run_id] == event_count
|
||||
assert [entry.event for entry in stream.events] == ["before"]
|
||||
|
||||
replay = [item async for item in bridge.subscribe(run_id)]
|
||||
assert [item.event for item in replay[:-1]] == ["before"]
|
||||
assert replay[-1] is END_SENTINEL
|
||||
finally:
|
||||
await bridge.close()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user