From 19618fc30ba666bcf454c1388c6a715245f1131b Mon Sep 17 00:00:00 2001 From: lihongyuan99 <64824864+lihongyuan99@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:57:24 +0800 Subject: [PATCH] fix(frontend): stop treating a fence line with an info string as a closer (#5919) CommonMark forbids an info string on a closing fence, so "```python" inside a plain ``` fence is content, not the end of the block. The closer check that two fence consumers share only compared character and run length, which ends the block early and pushes whatever follows back into prose: on the render path a system marker inside the sample loses its tags and its content becomes visible text, and on the export path stripInternalMarkers deletes real user content from the thread markdown. mermaid.ts already had the correct predicate privately. Move it into the shared leaf module and use it from both consumers, so the three fence scanners in core/streamdown and core/messages agree on what closes a block. --- frontend/src/core/messages/utils.ts | 14 ++++++----- frontend/src/core/streamdown/fences.ts | 25 +++++++++++++++++++ frontend/src/core/streamdown/mermaid.ts | 18 ++----------- frontend/src/core/streamdown/preprocess.ts | 16 ++++++------ .../tests/unit/core/messages/utils.test.ts | 17 +++++++++++++ .../unit/core/streamdown/preprocess.test.ts | 9 +++++++ 6 files changed, 70 insertions(+), 29 deletions(-) diff --git a/frontend/src/core/messages/utils.ts b/frontend/src/core/messages/utils.ts index a96198b82d..506f36d24d 100644 --- a/frontend/src/core/messages/utils.ts +++ b/frontend/src/core/messages/utils.ts @@ -1,6 +1,10 @@ import type { AIMessage, Message } from "@langchain/langgraph-sdk"; -import { FENCE_MARKER_RE, INDENTED_CODE_RE } from "@/core/streamdown/fences"; +import { + FENCE_MARKER_RE, + INDENTED_CODE_RE, + isClosingFence, +} from "@/core/streamdown/fences"; interface GenericMessageGroup { type: T; @@ -1092,7 +1096,8 @@ const INTERNAL_MARKER_RE = new RegExp( /** * Character ranges that must survive marker stripping: fenced code blocks * (marker-aware, so a shorter or different fence inside a block does not - * close it) and 4-space indented code lines — the same protection the render + * close it, and a fence line that carries an info string never closes one) + * and 4-space indented code lines — the same protection the render * path applies in ``stripLeakedSystemTags``. ``project`` and ``documents`` * are generic tag names, so a fenced Maven ``pom.xml`` or pasted XML must not * lose its span on export; a marker whose span STARTS inside a protected @@ -1111,10 +1116,7 @@ function protectedCodeRanges(content: string): Array<[number, number]> { if (fenceMarker === null) { fenceMarker = marker; fenceStart = offset; - } else if ( - marker.startsWith(fenceMarker.charAt(0)) && - marker.length >= fenceMarker.length - ) { + } else if (isClosingFence(line, fenceMarker)) { ranges.push([fenceStart, offset + line.length]); fenceMarker = null; } diff --git a/frontend/src/core/streamdown/fences.ts b/frontend/src/core/streamdown/fences.ts index 06059e4578..d6d2f3b085 100644 --- a/frontend/src/core/streamdown/fences.ts +++ b/frontend/src/core/streamdown/fences.ts @@ -11,3 +11,28 @@ export const INDENTED_CODE_RE = /^(?: {4}|\t)/; // containing a shorter backtick run (or a 4-backtick block containing a // 3-backtick run) does not prematurely close the fence. export const FENCE_MARKER_RE = /^ {0,3}(`{3,}|~{3,})/; + +/** + * Whether `line` closes the fenced block opened by `fence`. + * + * Stricter than a bare marker match: a closing fence may not carry an info + * string, so ``` ```python ``` is content of an outer ``` fence rather than its + * end. Every fence consumer has to apply this, otherwise the block ends early + * and whatever follows — a leaked ```` block on the render path, real + * user content on the export path — is treated as prose. + */ +export function isClosingFence(line: string, fence: string): boolean { + const trimmedLine = line.trimEnd(); + const indentationLength = trimmedLine.length - trimmedLine.trimStart().length; + const fenceMarker = trimmedLine.slice(indentationLength); + const fenceChar = fence.charAt(0); + + if (indentationLength > 3 || !fenceMarker.startsWith(fenceChar)) { + return false; + } + + return ( + fenceMarker.length >= fence.length && + [...fenceMarker].every((char) => char === fenceChar) + ); +} diff --git a/frontend/src/core/streamdown/mermaid.ts b/frontend/src/core/streamdown/mermaid.ts index 98d13163b9..462658a37a 100644 --- a/frontend/src/core/streamdown/mermaid.ts +++ b/frontend/src/core/streamdown/mermaid.ts @@ -1,3 +1,5 @@ +import { isClosingFence } from "@/core/streamdown/fences"; + const MERMAID_OPENING_FENCE_RE = /^[ \t]{0,3}(`{3,}|~{3,})[ \t]*mermaid(?:[ \t].*)?$/i; @@ -24,22 +26,6 @@ function normalizeMermaidCode(code: string): string { .join("\n"); } -function isClosingFence(line: string, fence: string): boolean { - const trimmedLine = line.trimEnd(); - const indentationLength = trimmedLine.length - trimmedLine.trimStart().length; - const fenceMarker = trimmedLine.slice(indentationLength); - const fenceChar = fence.charAt(0); - - if (indentationLength > 3 || !fenceMarker.startsWith(fenceChar)) { - return false; - } - - return ( - fenceMarker.length >= fence.length && - [...fenceMarker].every((char) => char === fenceChar) - ); -} - export function normalizeMermaidMarkdown(markdown: string): string { const lines = markdown.replace(WINDOWS_LINE_ENDING_RE, "\n").split("\n"); const normalizedLines: string[] = []; diff --git a/frontend/src/core/streamdown/preprocess.ts b/frontend/src/core/streamdown/preprocess.ts index a71dd00a25..0b8cb26bbd 100644 --- a/frontend/src/core/streamdown/preprocess.ts +++ b/frontend/src/core/streamdown/preprocess.ts @@ -1,5 +1,9 @@ import { INTERNAL_MARKER_TAGS } from "@/core/messages/utils"; -import { FENCE_MARKER_RE, INDENTED_CODE_RE } from "@/core/streamdown/fences"; +import { + FENCE_MARKER_RE, + INDENTED_CODE_RE, + isClosingFence, +} from "@/core/streamdown/fences"; import { normalizeMermaidMarkdown } from "./mermaid"; @@ -346,7 +350,8 @@ const _INTERNAL_TAG_RE = new RegExp( * Fence tracking is marker-aware (tracking the opening character and run * length) so that a tilde-fenced block containing a shorter backtick run, or a * 4-backtick block containing a 3-backtick run, does not prematurely close the - * fence. + * fence. A fence line that carries an info string never closes one either, so + * a `` ```python `` line inside a plain ``` fence stays sample code. */ export function stripLeakedSystemTags(markdown: string): string { const lines = markdown.split("\n"); @@ -360,11 +365,8 @@ export function stripLeakedSystemTags(markdown: string): string { if (fenceMarker === null) { // Opening a fenced code block fenceMarker = marker; - } else if ( - marker.startsWith(fenceMarker.charAt(0)) && - marker.length >= fenceMarker.length - ) { - // Closing fence: same character and at least as long as opener + } else if (isClosingFence(line, fenceMarker)) { + // Closing fence: same character, at least as long, no info string fenceMarker = null; } // Otherwise: different fence type or shorter run inside a fence diff --git a/frontend/tests/unit/core/messages/utils.test.ts b/frontend/tests/unit/core/messages/utils.test.ts index a9f4135654..25bd5a2a84 100644 --- a/frontend/tests/unit/core/messages/utils.test.ts +++ b/frontend/tests/unit/core/messages/utils.test.ts @@ -1132,6 +1132,23 @@ describe("human message internal context stripping", () => { expect(stripInternalMarkers(indented)).toBe(indented); }); + test("stripInternalMarkers keeps a fence open across a line with an info string", () => { + // CommonMark: a closing fence cannot carry an info string, so "```python" + // is content of the outer fence. Treating it as the closer exposes the + // following block and the export silently deletes real user content. + const content = [ + "Fence tutorial:", + "```", + "```python", + 'keep me', + "```", + "```", + "Export me", + ].join("\n"); + + expect(stripInternalMarkers(content)).toBe(content); + }); + test("stripInternalMarkers still removes an injected block whose content contains a fence", () => { const content = [ "", diff --git a/frontend/tests/unit/core/streamdown/preprocess.test.ts b/frontend/tests/unit/core/streamdown/preprocess.test.ts index 30a895c485..12b6734467 100644 --- a/frontend/tests/unit/core/streamdown/preprocess.test.ts +++ b/frontend/tests/unit/core/streamdown/preprocess.test.ts @@ -462,3 +462,12 @@ test("stripLeakedSystemTags strips tags after real closing fence", () => { ); expect(stripLeakedSystemTags(input)).toBe(expected); }); + +test("stripLeakedSystemTags keeps a fence open across a line with an info string", () => { + // A closing fence cannot carry an info string, so "```python" is content of + // the outer fence and the marker inside it must not become visible text. + const input = ["```", "```python", "inside", "```"].join( + "\n", + ); + expect(stripLeakedSystemTags(input)).toBe(input); +});