mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-27 13:40:37 +00:00
4f9a76d5ef
* fix(stream): tolerate ~~~-fenced code and \-prefixed closer when decoding escaped newlines _decode_escaped_newlines_outside_fences only recognised ``` fences, so replies that used ~~~ fences (e.g. code blocks rendered by the model with ~~~python ...) kept literal backslash-n sequences escaped instead of decoding the ones outside fences, and a fence whose closer was split as backslash+tilde across chunks was misread mid-stream. Fixes #7262. - detect the opening fence with (?:```|~~~) and remember which delimiter the fence was opened with, so closing requires the same delimiter - while inside a fence, hold back a closer that begins with a backslash so a fence ending in backslash+tilde mid-stream is not misread; the held tail is revisited on the next push and normalised on flush - keep backtick hold semantics unchanged * fix(mindie): preserve unclosed fences during newline decoding Match one-shot decoding to streaming by choosing the earliest backtick or tilde opener and preserving the remainder when its matching closer is absent. Correct the string-return docstring and cover mixed delimiters across every two-chunk split and character-by-character streaming. --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
54 lines
1.7 KiB
Python
54 lines
1.7 KiB
Python
"""Whole-response and streaming MindIE decoding must protect the same fences."""
|
|
|
|
import pytest
|
|
|
|
from deerflow.models.mindie_provider import (
|
|
_decode_escaped_newlines_outside_fences,
|
|
_EscapedNewlineStreamDecoder,
|
|
)
|
|
|
|
|
|
@pytest.fixture(params=["```", "~~~"], ids=["backticks", "tildes"])
|
|
def fence_delimiters(request):
|
|
outer = request.param
|
|
inner = "~~~" if outer == "```" else "```"
|
|
return outer, inner
|
|
|
|
|
|
@pytest.fixture(params=["", r"intro\n"], ids=["at-start", "after-prose"])
|
|
def prefix(request):
|
|
return request.param
|
|
|
|
|
|
@pytest.fixture(params=[False, True], ids=["unterminated", "closed"])
|
|
def fenced_reply(request, fence_delimiters, prefix):
|
|
outer, inner = fence_delimiters
|
|
# The inner pair must never close or escape an outer fence of either kind.
|
|
code = f"{outer}python\\nouter\\n{inner}json\\ninner\\n{inner}\\ntail\\n"
|
|
suffix = ""
|
|
if request.param:
|
|
code += outer
|
|
suffix = r"\noutro\nend"
|
|
raw = prefix + code + suffix
|
|
expected = prefix.replace("\\n", "\n") + code + suffix.replace("\\n", "\n")
|
|
return raw, expected
|
|
|
|
|
|
def test_complete_and_unterminated_fences_keep_literal_escapes(fenced_reply):
|
|
raw, expected = fenced_reply
|
|
|
|
assert _decode_escaped_newlines_outside_fences(raw) == expected
|
|
|
|
|
|
def test_stream_matches_whole_reply_at_every_chunk_boundary(fenced_reply):
|
|
raw, expected = fenced_reply
|
|
chunkings = [[raw[:split], raw[split:]] for split in range(len(raw) + 1)]
|
|
chunkings.append(list(raw))
|
|
|
|
for chunks in chunkings:
|
|
decoder = _EscapedNewlineStreamDecoder()
|
|
streamed = "".join(decoder.push(chunk) for chunk in chunks) + decoder.flush()
|
|
|
|
assert streamed == expected, chunks
|
|
assert streamed == _decode_escaped_newlines_outside_fences(raw), chunks
|