diff --git a/frontend/src/core/messages/utils.ts b/frontend/src/core/messages/utils.ts index a96198b82d..506f36d24d 100644 --- a/frontend/src/core/messages/utils.ts +++ b/frontend/src/core/messages/utils.ts @@ -1,6 +1,10 @@ import type { AIMessage, Message } from "@langchain/langgraph-sdk"; -import { FENCE_MARKER_RE, INDENTED_CODE_RE } from "@/core/streamdown/fences"; +import { + FENCE_MARKER_RE, + INDENTED_CODE_RE, + isClosingFence, +} from "@/core/streamdown/fences"; interface GenericMessageGroup { type: T; @@ -1092,7 +1096,8 @@ const INTERNAL_MARKER_RE = new RegExp( /** * Character ranges that must survive marker stripping: fenced code blocks * (marker-aware, so a shorter or different fence inside a block does not - * close it) and 4-space indented code lines — the same protection the render + * close it, and a fence line that carries an info string never closes one) + * and 4-space indented code lines — the same protection the render * path applies in ``stripLeakedSystemTags``. ``project`` and ``documents`` * are generic tag names, so a fenced Maven ``pom.xml`` or pasted XML must not * lose its span on export; a marker whose span STARTS inside a protected @@ -1111,10 +1116,7 @@ function protectedCodeRanges(content: string): Array<[number, number]> { if (fenceMarker === null) { fenceMarker = marker; fenceStart = offset; - } else if ( - marker.startsWith(fenceMarker.charAt(0)) && - marker.length >= fenceMarker.length - ) { + } else if (isClosingFence(line, fenceMarker)) { ranges.push([fenceStart, offset + line.length]); fenceMarker = null; } diff --git a/frontend/src/core/streamdown/fences.ts b/frontend/src/core/streamdown/fences.ts index 06059e4578..d6d2f3b085 100644 --- a/frontend/src/core/streamdown/fences.ts +++ b/frontend/src/core/streamdown/fences.ts @@ -11,3 +11,28 @@ export const INDENTED_CODE_RE = /^(?: {4}|\t)/; // containing a shorter backtick run (or a 4-backtick block containing a // 3-backtick run) does not prematurely close the fence. export const FENCE_MARKER_RE = /^ {0,3}(`{3,}|~{3,})/; + +/** + * Whether `line` closes the fenced block opened by `fence`. + * + * Stricter than a bare marker match: a closing fence may not carry an info + * string, so ``` ```python ``` is content of an outer ``` fence rather than its + * end. Every fence consumer has to apply this, otherwise the block ends early + * and whatever follows — a leaked ```` block on the render path, real + * user content on the export path — is treated as prose. + */ +export function isClosingFence(line: string, fence: string): boolean { + const trimmedLine = line.trimEnd(); + const indentationLength = trimmedLine.length - trimmedLine.trimStart().length; + const fenceMarker = trimmedLine.slice(indentationLength); + const fenceChar = fence.charAt(0); + + if (indentationLength > 3 || !fenceMarker.startsWith(fenceChar)) { + return false; + } + + return ( + fenceMarker.length >= fence.length && + [...fenceMarker].every((char) => char === fenceChar) + ); +} diff --git a/frontend/src/core/streamdown/mermaid.ts b/frontend/src/core/streamdown/mermaid.ts index 98d13163b9..462658a37a 100644 --- a/frontend/src/core/streamdown/mermaid.ts +++ b/frontend/src/core/streamdown/mermaid.ts @@ -1,3 +1,5 @@ +import { isClosingFence } from "@/core/streamdown/fences"; + const MERMAID_OPENING_FENCE_RE = /^[ \t]{0,3}(`{3,}|~{3,})[ \t]*mermaid(?:[ \t].*)?$/i; @@ -24,22 +26,6 @@ function normalizeMermaidCode(code: string): string { .join("\n"); } -function isClosingFence(line: string, fence: string): boolean { - const trimmedLine = line.trimEnd(); - const indentationLength = trimmedLine.length - trimmedLine.trimStart().length; - const fenceMarker = trimmedLine.slice(indentationLength); - const fenceChar = fence.charAt(0); - - if (indentationLength > 3 || !fenceMarker.startsWith(fenceChar)) { - return false; - } - - return ( - fenceMarker.length >= fence.length && - [...fenceMarker].every((char) => char === fenceChar) - ); -} - export function normalizeMermaidMarkdown(markdown: string): string { const lines = markdown.replace(WINDOWS_LINE_ENDING_RE, "\n").split("\n"); const normalizedLines: string[] = []; diff --git a/frontend/src/core/streamdown/preprocess.ts b/frontend/src/core/streamdown/preprocess.ts index a71dd00a25..0b8cb26bbd 100644 --- a/frontend/src/core/streamdown/preprocess.ts +++ b/frontend/src/core/streamdown/preprocess.ts @@ -1,5 +1,9 @@ import { INTERNAL_MARKER_TAGS } from "@/core/messages/utils"; -import { FENCE_MARKER_RE, INDENTED_CODE_RE } from "@/core/streamdown/fences"; +import { + FENCE_MARKER_RE, + INDENTED_CODE_RE, + isClosingFence, +} from "@/core/streamdown/fences"; import { normalizeMermaidMarkdown } from "./mermaid"; @@ -346,7 +350,8 @@ const _INTERNAL_TAG_RE = new RegExp( * Fence tracking is marker-aware (tracking the opening character and run * length) so that a tilde-fenced block containing a shorter backtick run, or a * 4-backtick block containing a 3-backtick run, does not prematurely close the - * fence. + * fence. A fence line that carries an info string never closes one either, so + * a `` ```python `` line inside a plain ``` fence stays sample code. */ export function stripLeakedSystemTags(markdown: string): string { const lines = markdown.split("\n"); @@ -360,11 +365,8 @@ export function stripLeakedSystemTags(markdown: string): string { if (fenceMarker === null) { // Opening a fenced code block fenceMarker = marker; - } else if ( - marker.startsWith(fenceMarker.charAt(0)) && - marker.length >= fenceMarker.length - ) { - // Closing fence: same character and at least as long as opener + } else if (isClosingFence(line, fenceMarker)) { + // Closing fence: same character, at least as long, no info string fenceMarker = null; } // Otherwise: different fence type or shorter run inside a fence diff --git a/frontend/tests/unit/core/messages/utils.test.ts b/frontend/tests/unit/core/messages/utils.test.ts index a9f4135654..25bd5a2a84 100644 --- a/frontend/tests/unit/core/messages/utils.test.ts +++ b/frontend/tests/unit/core/messages/utils.test.ts @@ -1132,6 +1132,23 @@ describe("human message internal context stripping", () => { expect(stripInternalMarkers(indented)).toBe(indented); }); + test("stripInternalMarkers keeps a fence open across a line with an info string", () => { + // CommonMark: a closing fence cannot carry an info string, so "```python" + // is content of the outer fence. Treating it as the closer exposes the + // following block and the export silently deletes real user content. + const content = [ + "Fence tutorial:", + "```", + "```python", + 'keep me', + "```", + "```", + "Export me", + ].join("\n"); + + expect(stripInternalMarkers(content)).toBe(content); + }); + test("stripInternalMarkers still removes an injected block whose content contains a fence", () => { const content = [ "", diff --git a/frontend/tests/unit/core/streamdown/preprocess.test.ts b/frontend/tests/unit/core/streamdown/preprocess.test.ts index 30a895c485..12b6734467 100644 --- a/frontend/tests/unit/core/streamdown/preprocess.test.ts +++ b/frontend/tests/unit/core/streamdown/preprocess.test.ts @@ -462,3 +462,12 @@ test("stripLeakedSystemTags strips tags after real closing fence", () => { ); expect(stripLeakedSystemTags(input)).toBe(expected); }); + +test("stripLeakedSystemTags keeps a fence open across a line with an info string", () => { + // A closing fence cannot carry an info string, so "```python" is content of + // the outer fence and the marker inside it must not become visible text. + const input = ["```", "```python", "inside", "```"].join( + "\n", + ); + expect(stripLeakedSystemTags(input)).toBe(input); +});