mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 21:26:39 +00:00
fix(models): preserve escapes in MindIE tilde fences (#7264)
This commit is contained in:
Tsai Yuan
GitHub
parent
3d2ed725f6
commit
9f1fd2fc05
@ -225,11 +225,28 @@ def _decode_escaped_newlines_outside_fences(content: str) -> str:
|
||||
if "\\n" not in content:
|
||||
return content
|
||||
|
||||
parts = re.split(r"(```[\s\S]*?```)", content)
|
||||
for idx, part in enumerate(parts):
|
||||
if part.startswith("```"):
|
||||
continue
|
||||
parts[idx] = part.replace("\\n", "\n")
|
||||
parts: list[str] = []
|
||||
cursor = 0
|
||||
while cursor < len(content):
|
||||
candidates = [(position, delimiter) for delimiter in ("```", "~~~") if (position := content.find(delimiter, cursor)) >= 0]
|
||||
if not candidates:
|
||||
parts.append(content[cursor:].replace("\\n", "\n"))
|
||||
break
|
||||
|
||||
start, delimiter = min(candidates)
|
||||
end = content.find(delimiter, start + len(delimiter))
|
||||
if end < 0:
|
||||
# An unterminated Markdown fence extends to the end of the reply.
|
||||
# Preserve it verbatim, including any complete pair that uses the
|
||||
# other supported delimiter, to match the streaming decoder.
|
||||
parts.append(content[cursor:start].replace("\\n", "\n"))
|
||||
parts.append(content[start:])
|
||||
break
|
||||
|
||||
parts.append(content[cursor:start].replace("\\n", "\n"))
|
||||
end += len(delimiter)
|
||||
parts.append(content[start:end])
|
||||
cursor = end
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
@ -246,7 +263,7 @@ class _EscapedNewlineStreamDecoder:
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._inside_fence = False
|
||||
self._fence_delimiter: str | None = None
|
||||
self._pending = ""
|
||||
|
||||
def push(self, text: str) -> str:
|
||||
@ -255,32 +272,36 @@ class _EscapedNewlineStreamDecoder:
|
||||
self._pending = ""
|
||||
out = []
|
||||
while data:
|
||||
fence = data.find("```")
|
||||
if self._inside_fence:
|
||||
if self._fence_delimiter is not None:
|
||||
fence = data.find(self._fence_delimiter)
|
||||
if fence == -1:
|
||||
# The closing ``` may be split across chunks: emit the
|
||||
# The closing delimiter may be split across chunks: emit the
|
||||
# fenced code verbatim and keep a partial closer pending.
|
||||
hold = len(data) - len(data.rstrip("`"))
|
||||
hold = len(data) - len(data.rstrip(self._fence_delimiter[0]))
|
||||
out.append(data[: len(data) - hold])
|
||||
self._pending = data[len(data) - hold :]
|
||||
break
|
||||
# Fenced code up to and including its closing ``` stays verbatim.
|
||||
out.append(data[: fence + 3])
|
||||
data = data[fence + 3 :]
|
||||
self._inside_fence = False
|
||||
# Fenced code through its matching closer stays verbatim.
|
||||
delimiter_length = len(self._fence_delimiter)
|
||||
out.append(data[: fence + delimiter_length])
|
||||
data = data[fence + delimiter_length :]
|
||||
self._fence_delimiter = None
|
||||
continue
|
||||
if fence == -1:
|
||||
# Outside a fence the tail may still grow into ``` or into an
|
||||
|
||||
candidates = [(position, delimiter) for delimiter in ("```", "~~~") if (position := data.find(delimiter)) >= 0]
|
||||
if not candidates:
|
||||
# Outside a fence the tail may still grow into a delimiter or an
|
||||
# escape split across the boundary; hold it back, decode the rest.
|
||||
hold = len(data) - len(data.rstrip("`\\"))
|
||||
hold = len(data) - len(data.rstrip("`~\\"))
|
||||
out.append(_decode_escaped_newlines_outside_fences(data[: len(data) - hold]))
|
||||
self._pending = data[len(data) - hold :]
|
||||
break
|
||||
# Prose before the opening ``` decodes; the fence itself does not.
|
||||
fence, delimiter = min(candidates)
|
||||
# Prose before the opening delimiter decodes; the fence itself does not.
|
||||
out.append(_decode_escaped_newlines_outside_fences(data[:fence]))
|
||||
out.append(data[fence : fence + 3])
|
||||
data = data[fence + 3 :]
|
||||
self._inside_fence = True
|
||||
out.append(delimiter)
|
||||
data = data[fence + len(delimiter) :]
|
||||
self._fence_delimiter = delimiter
|
||||
return "".join(out)
|
||||
|
||||
def flush(self) -> str:
|
||||
|
||||
@ -503,11 +503,20 @@ class TestPatchResult:
|
||||
patched = model._patch_result_with_tools(result)
|
||||
assert patched.generations[0].message.content == "line1\nline2"
|
||||
|
||||
def test_escaped_newlines_inside_code_fence_preserved(self):
|
||||
@pytest.mark.parametrize("delimiter", ["```", "~~~"], ids=["backticks", "tildes"])
|
||||
def test_escaped_newlines_inside_code_fence_preserved(self, delimiter):
|
||||
model = self._model()
|
||||
result = _make_chat_result('text\\n```json\n{"k":"a\\\\nb"}\n```\\nend')
|
||||
result = _make_chat_result(f'text\\n{delimiter}json\n{{"k":"a\\\\nb"}}\n{delimiter}\\nend')
|
||||
patched = model._patch_result_with_tools(result)
|
||||
assert patched.generations[0].message.content == 'text\n```json\n{"k":"a\\\\nb"}\n```\nend'
|
||||
assert patched.generations[0].message.content == f'text\n{delimiter}json\n{{"k":"a\\\\nb"}}\n{delimiter}\nend'
|
||||
|
||||
@pytest.mark.parametrize(("outer", "inner"), [("~~~", "```"), ("```", "~~~")], ids=["tilde-outer", "backtick-outer"])
|
||||
def test_unterminated_fence_preserves_other_delimiter_pair(self, outer, inner):
|
||||
content = f"intro\\n{outer}python\\nouter\\n{inner}json\\ninner\\n{inner}\\ntail\\n"
|
||||
|
||||
decoded = _decode_escaped_newlines_outside_fences(content)
|
||||
|
||||
assert decoded == f"intro\n{outer}python\\nouter\\n{inner}json\\ninner\\n{inner}\\ntail\\n"
|
||||
|
||||
def test_xml_tool_calls_extracted(self):
|
||||
model = self._model()
|
||||
@ -761,7 +770,8 @@ class TestAStream:
|
||||
assert chunks[0].message.content == "a\nb"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_tools_stream_keeps_escaped_newlines_inside_fences(self):
|
||||
@pytest.mark.parametrize("delimiter", ["```", "~~~"], ids=["backticks", "tildes"])
|
||||
async def test_no_tools_stream_keeps_escaped_newlines_inside_fences(self, delimiter):
|
||||
"""A fence spanning several chunks must still protect its literal `\\n`.
|
||||
|
||||
The one-shot decode needs a complete ```...``` pair to tell fenced code
|
||||
@ -773,7 +783,7 @@ class TestAStream:
|
||||
from langchain_core.messages import AIMessageChunk
|
||||
from langchain_core.outputs import ChatGenerationChunk
|
||||
|
||||
raw_chunks = ['intro\\n```python\\nprint("a\\nb")', "\\n```\\noutro\\nend"]
|
||||
raw_chunks = [f'intro\\n{delimiter}python\\nprint("a\\nb")', f"\\n{delimiter}\\noutro\\nend"]
|
||||
|
||||
async def fake_stream(*args, **kwargs):
|
||||
for text in raw_chunks:
|
||||
@ -802,10 +812,25 @@ class TestAStream:
|
||||
["outside\\", "ncode"],
|
||||
# Fence delimiter itself split across the chunk boundary.
|
||||
["text\\n``", "`python\\nx\\ny", "\\n```\\nend"],
|
||||
# Tilde fence delimiter split across the chunk boundary.
|
||||
["text\\n~~", "~python\\nx\\ny", "\\n~~~\\nend"],
|
||||
# Two fences with prose between them, escaped prose still decodes.
|
||||
["```a\\n1```\\nmid\\n```b\\n2", "```\\ntail\\nend"],
|
||||
# An unterminated tilde fence protects a complete backtick pair.
|
||||
["intro\\n~~~python\\nouter\\n```json", "\\ninner\\n```\\ntail\\n"],
|
||||
# An unterminated backtick fence protects a complete tilde pair.
|
||||
["intro\\n```python\\nouter\\n~~~json", "\\ninner\\n~~~\\ntail\\n"],
|
||||
],
|
||||
ids=[
|
||||
"single-chunk-fence",
|
||||
"fence-spans-chunks",
|
||||
"escape-split-across-chunks",
|
||||
"fence-delimiter-split",
|
||||
"tilde-delimiter-split",
|
||||
"two-fences",
|
||||
"unterminated-tilde-with-backtick-pair",
|
||||
"unterminated-backtick-with-tilde-pair",
|
||||
],
|
||||
ids=["single-chunk-fence", "fence-spans-chunks", "escape-split-across-chunks", "fence-delimiter-split", "two-fences"],
|
||||
)
|
||||
async def test_no_tools_stream_decode_matches_non_streaming(self, raw_chunks):
|
||||
"""The joined native stream equals the one-shot decode of the full reply."""
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user