diff --git a/README.md b/README.md index 47ef412a8a..a5f9c1c539 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,8 @@ For MindIE XML tool calls, see the [argument parsing and newline compatibility guide](backend/docs/CONFIGURATION.md#mindie-xml-tool-arguments). Both synchronous and asynchronous streams retain this compatibility: tool-enabled streams simulate chunks from a non-streaming response, while no-tool streams stay native. +Newline compatibility preserves literal escapes inside backtick and tilde code +fences, including unclosed fences, in complete and streamed replies. 1. **Clone the DeerFlow repository** diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index 61187d0ea3..9cbb42a709 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -453,7 +453,8 @@ backslash. The existing Python-literal fallback also parses the original value. Non-JSON raw-string parameters retain the gateway's multiline compatibility: literal `\n` outside fenced code becomes a real newline, and surrounding -whitespace is trimmed. Escapes inside fenced code remain unchanged. This keeps +whitespace is trimmed. Escapes inside matching triple-backtick or triple-tilde +fences remain unchanged; an unclosed fence preserves the rest of the reply. This keeps multi-line file content and commands working. Raw strings cannot distinguish an intended literal `\n` from a gateway-escaped newline; structured JSON parameters avoid that ambiguity. Numeric conversion failures and unsafe Python-literal diff --git a/backend/packages/harness/deerflow/models/AGENTS.md b/backend/packages/harness/deerflow/models/AGENTS.md index 49bc0a28cd..a6f8902f93 100644 --- a/backend/packages/harness/deerflow/models/AGENTS.md +++ b/backend/packages/harness/deerflow/models/AGENTS.md @@ -202,6 +202,11 @@ the shared simulated chunker. Preserve terminal usage exactly once, XML/native tool calls, invalid calls, and the original messages. Offline SDK boundary tests: `tests/test_mindie_provider.py::test_public_stream_compatibility_through_sdk`. +Whole-response and streaming newline decoding protect both triple-backtick and +triple-tilde fences. The earliest opener wins, only the same delimiter closes it, +and an unclosed fence keeps the remainder verbatim. Boundary and mixed-delimiter +regressions live in `tests/test_mindie_newline_fences.py`. + `_fix_messages` converts tool results to the XML text format expected by MindIE. Only tool-message `type=json` payloads join the text channel; other non-text blocks remain omitted. Keep the `` escaping boundary and the diff --git a/backend/packages/harness/deerflow/models/mindie_provider.py b/backend/packages/harness/deerflow/models/mindie_provider.py index fb72a33383..e1e7488aff 100644 --- a/backend/packages/harness/deerflow/models/mindie_provider.py +++ b/backend/packages/harness/deerflow/models/mindie_provider.py @@ -254,10 +254,10 @@ class _EscapedNewlineStreamDecoder: """Decode literal `\\n` outside fenced code blocks across streamed chunks. `_decode_escaped_newlines_outside_fences` needs the whole reply to pair - ``` delimiters, and a token-sized chunk almost never contains both, so + ``` or ~~~ delimiters, and a token-sized chunk rarely contains both, so decoding chunks independently would also decode escapes inside fences. This decoder carries the fence state between chunks and holds back a - trailing partial ```/`\\n` sequence, so the joined stream equals the + trailing partial delimiter or `\\n` sequence, so the joined stream equals the one-shot decode of the same reply. A fence the stream never closes keeps its escapes: an unterminated fence still renders as code. """ @@ -305,7 +305,7 @@ class _EscapedNewlineStreamDecoder: return "".join(out) def flush(self) -> str: - """Return text held back for a partial ```/`\\n` at the end of the stream.""" + """Return text held back for a partial delimiter/`\\n` at stream end.""" pending, self._pending = self._pending, "" return pending diff --git a/backend/tests/test_mindie_newline_fences.py b/backend/tests/test_mindie_newline_fences.py new file mode 100644 index 0000000000..15674aebb4 --- /dev/null +++ b/backend/tests/test_mindie_newline_fences.py @@ -0,0 +1,53 @@ +"""Whole-response and streaming MindIE decoding must protect the same fences.""" + +import pytest + +from deerflow.models.mindie_provider import ( + _decode_escaped_newlines_outside_fences, + _EscapedNewlineStreamDecoder, +) + + +@pytest.fixture(params=["```", "~~~"], ids=["backticks", "tildes"]) +def fence_delimiters(request): + outer = request.param + inner = "~~~" if outer == "```" else "```" + return outer, inner + + +@pytest.fixture(params=["", r"intro\n"], ids=["at-start", "after-prose"]) +def prefix(request): + return request.param + + +@pytest.fixture(params=[False, True], ids=["unterminated", "closed"]) +def fenced_reply(request, fence_delimiters, prefix): + outer, inner = fence_delimiters + # The inner pair must never close or escape an outer fence of either kind. + code = f"{outer}python\\nouter\\n{inner}json\\ninner\\n{inner}\\ntail\\n" + suffix = "" + if request.param: + code += outer + suffix = r"\noutro\nend" + raw = prefix + code + suffix + expected = prefix.replace("\\n", "\n") + code + suffix.replace("\\n", "\n") + return raw, expected + + +def test_complete_and_unterminated_fences_keep_literal_escapes(fenced_reply): + raw, expected = fenced_reply + + assert _decode_escaped_newlines_outside_fences(raw) == expected + + +def test_stream_matches_whole_reply_at_every_chunk_boundary(fenced_reply): + raw, expected = fenced_reply + chunkings = [[raw[:split], raw[split:]] for split in range(len(raw) + 1)] + chunkings.append(list(raw)) + + for chunks in chunkings: + decoder = _EscapedNewlineStreamDecoder() + streamed = "".join(decoder.push(chunk) for chunk in chunks) + decoder.flush() + + assert streamed == expected, chunks + assert streamed == _decode_escaped_newlines_outside_fences(raw), chunks