fix(streaming): fence memory writes after retirement or END (#7766)

* Rebase stream lifecycle fix onto upstream main

* Preserve gateway config snapshot and recovered stream retention guard
This commit is contained in:
NanPan 2026-11-17 09:59:18 +08:00 committed by GitHub
parent 4cd47819a6
commit 13edf25b89
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 138 additions and 6 deletions

View File

@ -526,8 +526,35 @@ async def _retry_recovered_run_stream_end(
*,
retry_delay: float,
cleanup_delay: float,
stream_was_present: bool | None = None,
) -> None:
await asyncio.sleep(max(0.0, retry_delay))
if stream_was_present is True:
stream_exists = getattr(bridge, "stream_exists", None)
if callable(stream_exists):
try:
if not await stream_exists(run_id):
cleanup_expired_producer = getattr(bridge, "cleanup_expired_producer", None)
if callable(cleanup_expired_producer):
try:
await cleanup_expired_producer(run_id)
except Exception:
logger.warning(
"Failed to clean expired stream producer marker after retry for %s",
run_id,
exc_info=True,
)
logger.debug(
"Skipping recovered stream end retry for %s: stream expired during retry delay",
run_id,
)
return
except Exception:
logger.debug(
"Failed to recheck recovered stream existence for %s",
run_id,
exc_info=True,
)
await bridge.publish_end(run_id)
await bridge.cleanup(run_id, delay=cleanup_delay)
@ -542,7 +569,7 @@ async def _publish_recovered_run_stream_end(
"""Terminate retained streams for runs recovered as orphaned."""
cleanup_tasks: list[tuple[str, asyncio.Task[None]]] = []
def schedule_retry(record: RunRecord) -> bool:
def schedule_retry(record: RunRecord, *, stream_was_present: bool | None = None) -> bool:
retry_delay = getattr(
bridge,
"producer_lease_retry_delay_seconds",
@ -556,6 +583,7 @@ async def _publish_recovered_run_stream_end(
record.run_id,
retry_delay=float(retry_delay),
cleanup_delay=cleanup_delay,
stream_was_present=stream_was_present,
)
)
task.add_done_callback(
@ -571,9 +599,11 @@ async def _publish_recovered_run_stream_end(
for record in recovered_runs:
stream_exists = getattr(bridge, "stream_exists", None)
stream_was_present: bool | None = None
if stream_exists is not None:
try:
if not await stream_exists(record.run_id):
stream_was_present = bool(await stream_exists(record.run_id))
if not stream_was_present:
logger.debug("Skipping recovered stream end for %s: stream already expired", record.run_id)
cleanup_expired_producer = getattr(
bridge,
@ -591,10 +621,12 @@ async def _publish_recovered_run_stream_end(
)
cleaned = False
if not cleaned:
schedule_retry(record)
schedule_retry(record, stream_was_present=False)
continue
except Exception:
logger.debug("Failed to check recovered stream existence for %s", record.run_id, exc_info=True)
schedule_retry(record, stream_was_present=stream_was_present)
continue
try:
await bridge.publish_end(record.run_id)
except Exception:
@ -603,7 +635,7 @@ async def _publish_recovered_run_stream_end(
record.run_id,
exc_info=True,
)
schedule_retry(record)
schedule_retry(record, stream_was_present=stream_was_present)
continue
task = asyncio.create_task(bridge.cleanup(record.run_id, delay=cleanup_delay))
task.add_done_callback(lambda task, run_id=record.run_id: _log_recovered_stream_cleanup_result(task, run_id))

View File

@ -143,7 +143,7 @@ sequenceDiagram
关键组件:
- `runtime/runs/worker.py::run_agent` — 在 `asyncio.Task` 里跑 `agent.astream()`,把每个 chunk 通过 `serialize(chunk, mode=mode)` 转成 JSON,再 `bridge.publish()`。
- `runtime/stream_bridge` — 抽象 Queue。`publish/subscribe` 解耦生产者和消费者,支持 `Last-Event-ID` 重连、心跳、多订阅者 fan-out。`stream_bridge.heartbeat_interval_seconds`(默认 15 秒)是 bridge 实例的默认心跳周期,统一作用于 Gateway SSE、`/wait` 和内部 channel watcher;显式传给 `subscribe()` 的值仍可覆盖单次订阅。Memory 和 Redis 都只保留 `queue_maxsize` 条数据事件。`MemoryStreamBridge` 在分配事件 ID 前深拷贝发布的 payload,并在 stream Condition 外为每个订阅者返回独立深拷贝,因此生产者复用 payload 或订阅者修改数据都不会影响保留帧及重连重放;心跳与 END sentinel 保持对象身份。游标早于保留水位线时返回 `StreamGap`,不会从当前最早事件静默部分重放。Redis backend 会在写入数据或首个 END 时刷新 retained stream key TTL;启动恢复与基于 worker lease 的周期恢复共用 Gateway stream terminalization 路径:`RunManager` 先将 orphan run 持久化为 `error` 并写入显式的 `stop_reason=orphan_recovered`,随后 Gateway 发布 `END_SENTINEL` 并安排 stream cleanup。周期扫描、逐行状态写入和 Gateway callback 作为一个受监督的 single-flight 后台 task 执行;慢任务不会堆积,也不会阻塞唯一的 lease heartbeat。shutdown 优先收敛活跃 run,再处理恢复 task;尚未执行的延迟 stream cleanup 会改为立即删除。只有 runtime `yield` 前、无并发请求的启动恢复会把最新受影响 thread 标记为 error;周期恢复不做非原子的 thread 投影。普通 durable terminal status 本身仍不足以证明尾帧已发布;Redis producer 在 durable admission 前取得短租约,经 request task、worker、finalizer 转移并保留到 END。只有显式 `orphan_recovered` 或 durable terminal 加已失效的 tracked producer lease 才能在 heartbeat 上触发 END fallback;lease loss 会阻止后续 stream 写入,active 或 untracked producer 继续等待,避免截断尾帧。Redis END 通过同一个 Lua 原子完成 lease 校验、既有 END 检查和必要写入,因此重复调用、owner/recovery 竞争和 ACK 丢失重试至多追加一个 END;当 END 已位于 stream 尾部时,后续事件写入会被拒绝;旧 stream 中 END 后已有迟到数据时,仍允许继续写入数据。cleanup 后无 owner 的迟到 END 重试是成功 no-op,不会复活已删除的 stream。普通事件只做 O(1) 尾部 END 检查。`stream_bridge.legacy_end_scan_enabled` 默认 `true`:尾部不是 END 时扫描整个保留窗口,包括正常 run 的首次 END;原子 Lua 的耗时随保留条目数量和 payload 总字节数增长,期间会阻塞其他 Redis 客户端。只有全部旧 writer 停止且旧 replay buffer 已过期或清除后,才可设为 `false` 并重启所有 Gateway,使 END 检查也只看尾部。TTL 为 0 时必须显式清除旧 buffer。TTL 仍是 Redis 内存和故障安全网,不是正常的 subscriber 终止机制。
- `runtime/stream_bridge` — 抽象 Queue。`publish/subscribe` 解耦生产者和消费者,支持 `Last-Event-ID` 重连、心跳、多订阅者 fan-out。`stream_bridge.heartbeat_interval_seconds`(默认 15 秒)是 bridge 实例的默认心跳周期,统一作用于 Gateway SSE、`/wait` 和内部 channel watcher;显式传给 `subscribe()` 的值仍可覆盖单次订阅。Memory 和 Redis 都只保留 `queue_maxsize` 条数据事件。Memory 发布在 Condition 内再次检查捕获的 generation 是否已 retired 或 ended,只有通过检查后才分配事件 ID;cleanup 或 END 先取得锁时,已排队的迟到写入会被丢弃,不会消耗序号,也不会出现在 END 之后。`MemoryStreamBridge` 在分配事件 ID 前深拷贝发布的 payload,并在 stream Condition 外为每个订阅者返回独立深拷贝,因此生产者复用 payload 或订阅者修改数据都不会影响保留帧及重连重放;心跳与 END sentinel 保持对象身份。游标早于保留水位线时返回 `StreamGap`,不会从当前最早事件静默部分重放。Redis backend 会在写入数据或首个 END 时刷新 retained stream key TTL;启动恢复与基于 worker lease 的周期恢复共用 Gateway stream terminalization 路径:`RunManager` 先将 orphan run 持久化为 `error` 并写入显式的 `stop_reason=orphan_recovered`,随后 Gateway 发布 `END_SENTINEL` 并安排 stream cleanup。周期扫描、逐行状态写入和 Gateway callback 作为一个受监督的 single-flight 后台 task 执行;慢任务不会堆积,也不会阻塞唯一的 lease heartbeat。shutdown 优先收敛活跃 run,再处理恢复 task;尚未执行的延迟 stream cleanup 会改为立即删除。只有 runtime `yield` 前、无并发请求的启动恢复会把最新受影响 thread 标记为 error;周期恢复不做非原子的 thread 投影。普通 durable terminal status 本身仍不足以证明尾帧已发布;Redis producer 在 durable admission 前取得短租约,经 request task、worker、finalizer 转移并保留到 END。只有显式 `orphan_recovered` 或 durable terminal 加已失效的 tracked producer lease 才能在 heartbeat 上触发 END fallback;lease loss 会阻止后续 stream 写入,active 或 untracked producer 继续等待,避免截断尾帧。Redis END 通过同一个 Lua 原子完成 lease 校验、既有 END 检查和必要写入,因此重复调用、owner/recovery 竞争和 ACK 丢失重试至多追加一个 END;当 END 已位于 stream 尾部时,后续事件写入会被拒绝;旧 stream 中 END 后已有迟到数据时,仍允许继续写入数据。cleanup 后无 owner 的迟到 END 重试是成功 no-op,不会复活已删除的 stream。普通事件只做 O(1) 尾部 END 检查。`stream_bridge.legacy_end_scan_enabled` 默认 `true`:尾部不是 END 时扫描整个保留窗口,包括正常 run 的首次 END;原子 Lua 的耗时随保留条目数量和 payload 总字节数增长,期间会阻塞其他 Redis 客户端。只有全部旧 writer 停止且旧 replay buffer 已过期或清除后,才可设为 `false` 并重启所有 Gateway,使 END 检查也只看尾部。TTL 为 0 时必须显式清除旧 buffer。TTL 仍是 Redis 内存和故障安全网,不是正常的 subscriber 终止机制。
- `stream_bridge.producer_lease_recovery_enabled` 是上述 producer-lease fallback 的 restart-required activate gate。升级时先保持 `false` 完成所有 writer 的滚动发布并排空旧 run,再在所有 Gateway 上同时设为 `true` 并重启;mixed-version 阶段不得启用。
- `app/gateway/services.py::sse_consumer` — 从 bridge 订阅,格式化为 SSE wire 帧。
- `runtime/serialization.py::serialize` — mode-aware 序列化;`messages` mode 下 `serialize_messages_tuple` 把 `(chunk, metadata)` 转成 `[chunk.model_dump(), metadata]`。

View File

@ -34,6 +34,8 @@ make test-blocking-io
would overwrite the thread view of an SDK client. Consumers that only handle the
root run ignore namespaced frames.
- **Memory writers fence the captured generation.** After waiting for the stream Condition, check `retired` and `ended` before allocating an event ID or appending; rejected writes consume no sequence number, and END is idempotent for that generation.
## Checkpoints
- **The channel mode is fixed for the life of the process.** A reader in `full`

View File

@ -119,8 +119,11 @@ class MemoryStreamBridge(StreamBridge):
async def publish(self, run_id: str, event: str, data: Any) -> None:
data = deepcopy(data)
stream = self._get_or_create_stream(run_id)
entry = StreamEvent(id=self._next_id(run_id), event=event, data=data)
async with stream.condition:
# Cleanup or END may win the Condition after this producer captures the generation.
if stream.retired or stream.ended:
return
entry = StreamEvent(id=self._next_id(run_id), event=event, data=data)
stream.events.append(entry)
if len(stream.events) > self._maxsize:
overflow = len(stream.events) - self._maxsize
@ -132,6 +135,8 @@ class MemoryStreamBridge(StreamBridge):
async def publish_end(self, run_id: str) -> None:
stream = self._get_or_create_stream(run_id)
async with stream.condition:
if stream.retired or stream.ended:
return
stream.ended = True
stream.condition.notify_all()

View File

@ -408,3 +408,22 @@ async def test_sqlite_runtime_does_not_mark_thread_error_when_newer_run_is_succe
assert thread_store.status_updates == []
assert stream_bridge.publish_end_calls == ["old-running"]
assert stream_bridge.cleanup_calls == [("old-running", 60.0)]
@pytest.mark.anyio
async def test_recovered_stream_end_retry_does_not_recreate_stream_cleaned_during_delay():
stream_bridge = _RetryingRecoveredStreamBridge()
cleanups = await gateway_deps._publish_recovered_run_stream_end(
stream_bridge,
[SimpleNamespace(run_id="run-1", thread_id="thread-1")],
cleanup_delay=0,
)
# Model another owner/retention cleanup winning before the delayed retry.
stream_bridge._existing_streams.clear()
await asyncio.gather(*(task for _run_id, task in cleanups))
assert stream_bridge.publish_attempts == 1
assert stream_bridge.publish_end_calls == []
assert stream_bridge.cleanup_calls == []
assert stream_bridge.expired_producer_cleanup_calls == ["run-1"]

View File

@ -348,3 +348,77 @@ async def test_gateway_wait_finishes_from_terminal_state_after_retirement(store_
finally:
await bridge.close()
await manager.shutdown()
async def _publish_behind_retirement(bridge, run_id, monkeypatch):
"""Queue cleanup before a producer write on the captured generation."""
condition = bridge._streams[run_id].condition
real_acquire = condition.acquire
retirement_queued = asyncio.Event()
publication_queued = asyncio.Event()
tasks = []
async def observe_acquire():
if asyncio.current_task() is tasks[0]:
retirement_queued.set()
else:
publication_queued.set()
return await real_acquire()
try:
async with condition:
monkeypatch.setattr(condition, "acquire", observe_acquire)
tasks.append(asyncio.create_task(bridge.cleanup(run_id)))
await asyncio.wait_for(retirement_queued.wait(), 1)
tasks.append(asyncio.create_task(bridge.publish(run_id, "late", {"value": "late"})))
await asyncio.wait_for(publication_queued.wait(), 1)
await asyncio.wait_for(asyncio.shield(tasks[0]), 1)
await asyncio.wait_for(asyncio.shield(tasks[1]), 1)
finally:
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
@pytest.mark.asyncio
async def test_publish_queued_behind_cleanup_is_discarded_from_the_retired_generation(monkeypatch):
bridge = MemoryStreamBridge()
run_id = "retired-publish"
await bridge.publish(run_id, "before", {"value": "before"})
stream = bridge._streams[run_id]
try:
await _publish_behind_retirement(bridge, run_id, monkeypatch)
assert stream.retired
assert stream.events == []
assert not await bridge.stream_exists(run_id)
await bridge.publish(run_id, "fresh", {"value": "fresh"})
current = bridge._streams[run_id]
assert current is not stream
assert current.events[0].id.rsplit("-", 1)[-1] == "0"
finally:
await bridge.close()
@pytest.mark.asyncio
async def test_publish_after_end_is_ignored_without_consuming_an_event_id():
bridge = MemoryStreamBridge()
run_id = "terminal-publish"
try:
await bridge.publish(run_id, "before", {"value": "before"})
stream = bridge._streams[run_id]
event_count = bridge._counters[run_id]
await bridge.publish_end(run_id)
await bridge.publish_end(run_id)
await bridge.publish(run_id, "late", {"value": "late"})
assert bridge._counters[run_id] == event_count
assert [entry.event for entry in stream.events] == ["before"]
replay = [item async for item in bridge.subscribe(run_id)]
assert [item.event for item in replay[:-1]] == ["before"]
assert replay[-1] is END_SENTINEL
finally:
await bridge.close()