From 9f1fd2fc05508dba4bfb2be6e67eba8f308f4828 Mon Sep 17 00:00:00 2001 From: Tsai Yuan <136825049+nothingyuancando@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:59:38 +0800 Subject: [PATCH] fix(models): preserve escapes in MindIE tilde fences (#7264) --- .../deerflow/models/mindie_provider.py | 63 ++++++++++++------- backend/tests/test_mindie_provider.py | 37 +++++++++-- 2 files changed, 73 insertions(+), 27 deletions(-) diff --git a/backend/packages/harness/deerflow/models/mindie_provider.py b/backend/packages/harness/deerflow/models/mindie_provider.py index fd30a1b796..fb72a33383 100644 --- a/backend/packages/harness/deerflow/models/mindie_provider.py +++ b/backend/packages/harness/deerflow/models/mindie_provider.py @@ -225,11 +225,28 @@ def _decode_escaped_newlines_outside_fences(content: str) -> str: if "\\n" not in content: return content - parts = re.split(r"(```[\s\S]*?```)", content) - for idx, part in enumerate(parts): - if part.startswith("```"): - continue - parts[idx] = part.replace("\\n", "\n") + parts: list[str] = [] + cursor = 0 + while cursor < len(content): + candidates = [(position, delimiter) for delimiter in ("```", "~~~") if (position := content.find(delimiter, cursor)) >= 0] + if not candidates: + parts.append(content[cursor:].replace("\\n", "\n")) + break + + start, delimiter = min(candidates) + end = content.find(delimiter, start + len(delimiter)) + if end < 0: + # An unterminated Markdown fence extends to the end of the reply. + # Preserve it verbatim, including any complete pair that uses the + # other supported delimiter, to match the streaming decoder. + parts.append(content[cursor:start].replace("\\n", "\n")) + parts.append(content[start:]) + break + + parts.append(content[cursor:start].replace("\\n", "\n")) + end += len(delimiter) + parts.append(content[start:end]) + cursor = end return "".join(parts) @@ -246,7 +263,7 @@ class _EscapedNewlineStreamDecoder: """ def __init__(self) -> None: - self._inside_fence = False + self._fence_delimiter: str | None = None self._pending = "" def push(self, text: str) -> str: @@ -255,32 +272,36 @@ class _EscapedNewlineStreamDecoder: self._pending = "" out = [] while data: - fence = data.find("```") - if self._inside_fence: + if self._fence_delimiter is not None: + fence = data.find(self._fence_delimiter) if fence == -1: - # The closing ``` may be split across chunks: emit the + # The closing delimiter may be split across chunks: emit the # fenced code verbatim and keep a partial closer pending. - hold = len(data) - len(data.rstrip("`")) + hold = len(data) - len(data.rstrip(self._fence_delimiter[0])) out.append(data[: len(data) - hold]) self._pending = data[len(data) - hold :] break - # Fenced code up to and including its closing ``` stays verbatim. - out.append(data[: fence + 3]) - data = data[fence + 3 :] - self._inside_fence = False + # Fenced code through its matching closer stays verbatim. + delimiter_length = len(self._fence_delimiter) + out.append(data[: fence + delimiter_length]) + data = data[fence + delimiter_length :] + self._fence_delimiter = None continue - if fence == -1: - # Outside a fence the tail may still grow into ``` or into an + + candidates = [(position, delimiter) for delimiter in ("```", "~~~") if (position := data.find(delimiter)) >= 0] + if not candidates: + # Outside a fence the tail may still grow into a delimiter or an # escape split across the boundary; hold it back, decode the rest. - hold = len(data) - len(data.rstrip("`\\")) + hold = len(data) - len(data.rstrip("`~\\")) out.append(_decode_escaped_newlines_outside_fences(data[: len(data) - hold])) self._pending = data[len(data) - hold :] break - # Prose before the opening ``` decodes; the fence itself does not. + fence, delimiter = min(candidates) + # Prose before the opening delimiter decodes; the fence itself does not. out.append(_decode_escaped_newlines_outside_fences(data[:fence])) - out.append(data[fence : fence + 3]) - data = data[fence + 3 :] - self._inside_fence = True + out.append(delimiter) + data = data[fence + len(delimiter) :] + self._fence_delimiter = delimiter return "".join(out) def flush(self) -> str: diff --git a/backend/tests/test_mindie_provider.py b/backend/tests/test_mindie_provider.py index d5d8e031a7..a4c62904a4 100644 --- a/backend/tests/test_mindie_provider.py +++ b/backend/tests/test_mindie_provider.py @@ -503,11 +503,20 @@ class TestPatchResult: patched = model._patch_result_with_tools(result) assert patched.generations[0].message.content == "line1\nline2" - def test_escaped_newlines_inside_code_fence_preserved(self): + @pytest.mark.parametrize("delimiter", ["```", "~~~"], ids=["backticks", "tildes"]) + def test_escaped_newlines_inside_code_fence_preserved(self, delimiter): model = self._model() - result = _make_chat_result('text\\n```json\n{"k":"a\\\\nb"}\n```\\nend') + result = _make_chat_result(f'text\\n{delimiter}json\n{{"k":"a\\\\nb"}}\n{delimiter}\\nend') patched = model._patch_result_with_tools(result) - assert patched.generations[0].message.content == 'text\n```json\n{"k":"a\\\\nb"}\n```\nend' + assert patched.generations[0].message.content == f'text\n{delimiter}json\n{{"k":"a\\\\nb"}}\n{delimiter}\nend' + + @pytest.mark.parametrize(("outer", "inner"), [("~~~", "```"), ("```", "~~~")], ids=["tilde-outer", "backtick-outer"]) + def test_unterminated_fence_preserves_other_delimiter_pair(self, outer, inner): + content = f"intro\\n{outer}python\\nouter\\n{inner}json\\ninner\\n{inner}\\ntail\\n" + + decoded = _decode_escaped_newlines_outside_fences(content) + + assert decoded == f"intro\n{outer}python\\nouter\\n{inner}json\\ninner\\n{inner}\\ntail\\n" def test_xml_tool_calls_extracted(self): model = self._model() @@ -761,7 +770,8 @@ class TestAStream: assert chunks[0].message.content == "a\nb" @pytest.mark.asyncio - async def test_no_tools_stream_keeps_escaped_newlines_inside_fences(self): + @pytest.mark.parametrize("delimiter", ["```", "~~~"], ids=["backticks", "tildes"]) + async def test_no_tools_stream_keeps_escaped_newlines_inside_fences(self, delimiter): """A fence spanning several chunks must still protect its literal `\\n`. The one-shot decode needs a complete ```...``` pair to tell fenced code @@ -773,7 +783,7 @@ class TestAStream: from langchain_core.messages import AIMessageChunk from langchain_core.outputs import ChatGenerationChunk - raw_chunks = ['intro\\n```python\\nprint("a\\nb")', "\\n```\\noutro\\nend"] + raw_chunks = [f'intro\\n{delimiter}python\\nprint("a\\nb")', f"\\n{delimiter}\\noutro\\nend"] async def fake_stream(*args, **kwargs): for text in raw_chunks: @@ -802,10 +812,25 @@ class TestAStream: ["outside\\", "ncode"], # Fence delimiter itself split across the chunk boundary. ["text\\n``", "`python\\nx\\ny", "\\n```\\nend"], + # Tilde fence delimiter split across the chunk boundary. + ["text\\n~~", "~python\\nx\\ny", "\\n~~~\\nend"], # Two fences with prose between them, escaped prose still decodes. ["```a\\n1```\\nmid\\n```b\\n2", "```\\ntail\\nend"], + # An unterminated tilde fence protects a complete backtick pair. + ["intro\\n~~~python\\nouter\\n```json", "\\ninner\\n```\\ntail\\n"], + # An unterminated backtick fence protects a complete tilde pair. + ["intro\\n```python\\nouter\\n~~~json", "\\ninner\\n~~~\\ntail\\n"], + ], + ids=[ + "single-chunk-fence", + "fence-spans-chunks", + "escape-split-across-chunks", + "fence-delimiter-split", + "tilde-delimiter-split", + "two-fences", + "unterminated-tilde-with-backtick-pair", + "unterminated-backtick-with-tilde-pair", ], - ids=["single-chunk-fence", "fence-spans-chunks", "escape-split-across-chunks", "fence-delimiter-split", "two-fences"], ) async def test_no_tools_stream_decode_matches_non_streaming(self, raw_chunks): """The joined native stream equals the one-shot decode of the full reply."""