mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-08 03:30:34 +00:00
a9bd03ef57
* fix(citations): keep copied markdown references valid formatCitationMarkdownReference pasted the source title and URL into "[title](url)" verbatim. A title with a stray "[" or "]" ends the link text early, and a URL with an unbalanced parenthesis ends the destination, so a copied reference no longer rendered as a link. Escape brackets in the title and wrap a parenthesized URL in the angle-bracket form. * fix(citations): only angle-bracket a parenthesized URL normalizeUrl already percent-encodes "<", ">" and whitespace, so including them in the trigger could wrap such a URL in an angle-bracket destination that CommonMark forbids, which is worse than the bare form. Parentheses are the only reachable delimiter that breaks the bare destination.
1072 lines
37 KiB
TypeScript
1072 lines
37 KiB
TypeScript
import { describe, expect, it } from "@rstest/core";
|
|
|
|
import {
|
|
extractCitationSources,
|
|
formatCitationMarkdownReference,
|
|
} from "@/core/citations/sources";
|
|
|
|
describe("extractCitationSources", () => {
|
|
it("extracts citation markdown links in first-seen order", () => {
|
|
const markdown = [
|
|
"Deep research needs evidence [citation:Paper A](https://example.com/a).",
|
|
"A second claim cites [citation:Report B](https://news.example.org/report?x=1).",
|
|
].join("\n");
|
|
const firstIndex = markdown.indexOf("[citation:Paper A]");
|
|
const secondIndex = markdown.indexOf("[citation:Report B]");
|
|
|
|
expect(extractCitationSources(markdown)).toEqual([
|
|
{
|
|
id: "https://example.com/a",
|
|
title: "Paper A",
|
|
url: "https://example.com/a",
|
|
domain: "example.com",
|
|
count: 1,
|
|
occurrences: [{ index: firstIndex, title: "Paper A" }],
|
|
},
|
|
{
|
|
id: "https://news.example.org/report?x=1",
|
|
title: "Report B",
|
|
url: "https://news.example.org/report?x=1",
|
|
domain: "news.example.org",
|
|
count: 1,
|
|
occurrences: [{ index: secondIndex, title: "Report B" }],
|
|
},
|
|
]);
|
|
});
|
|
|
|
it("deduplicates repeated citation URLs and preserves occurrence titles", () => {
|
|
const markdown = [
|
|
"First [citation:Original Title](https://example.com/research).",
|
|
"Later [citation:Updated Title](https://example.com/research).",
|
|
].join("\n");
|
|
const firstIndex = markdown.indexOf("[citation:Original Title]");
|
|
const secondIndex = markdown.indexOf("[citation:Updated Title]");
|
|
|
|
expect(extractCitationSources(markdown)).toEqual([
|
|
{
|
|
id: "https://example.com/research",
|
|
title: "Original Title",
|
|
url: "https://example.com/research",
|
|
domain: "example.com",
|
|
count: 2,
|
|
occurrences: [
|
|
{ index: firstIndex, title: "Original Title" },
|
|
{ index: secondIndex, title: "Updated Title" },
|
|
],
|
|
},
|
|
]);
|
|
});
|
|
|
|
it("ignores normal links, image links, and citations inside fenced code", () => {
|
|
const markdown = [
|
|
"[Normal](https://example.com/normal)",
|
|
"",
|
|
"```md",
|
|
"[citation:Example](https://example.com/example)",
|
|
"```",
|
|
"Real source [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
const realIndex = markdown.indexOf("[citation:Real]");
|
|
|
|
expect(extractCitationSources(markdown)).toEqual([
|
|
{
|
|
id: "https://example.com/real",
|
|
title: "Real",
|
|
url: "https://example.com/real",
|
|
domain: "example.com",
|
|
count: 1,
|
|
occurrences: [{ index: realIndex, title: "Real" }],
|
|
},
|
|
]);
|
|
});
|
|
|
|
it("keeps every source when citations are directly adjacent", () => {
|
|
const markdown =
|
|
"[citation:A](https://example.com/a)[citation:B](https://example.com/b)[citation:C](https://example.com/c)";
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/a",
|
|
"https://example.com/b",
|
|
"https://example.com/c",
|
|
]);
|
|
});
|
|
|
|
it("keeps URLs that contain multiple balanced parenthetical groups", () => {
|
|
const markdown = "[citation:W](https://en.wikipedia.org/wiki/Foo_(a)_(b))";
|
|
|
|
expect(extractCitationSources(markdown)[0]).toMatchObject({
|
|
url: "https://en.wikipedia.org/wiki/Foo_(a)_(b)",
|
|
domain: "en.wikipedia.org",
|
|
});
|
|
});
|
|
|
|
it("ignores citations inside inline code spans", () => {
|
|
const markdown =
|
|
"Example: `[citation:X](https://x.com/inline)` then real [citation:Real](https://example.com/real).";
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("keeps a citation whose paragraph sits between two unclosed backtick runs", () => {
|
|
// An inline span cannot cross a blank line, so neither stray backtick opens
|
|
// a span here and the citation is a rendered link, not sample code.
|
|
const markdown = [
|
|
"Install it with `npm i",
|
|
"",
|
|
"The upstream guide is [citation:Docs](https://example.com/docs).",
|
|
"",
|
|
"Then run `npm start` to serve it.",
|
|
].join("\n");
|
|
|
|
const sources = extractCitationSources(markdown);
|
|
|
|
expect(sources.map((source) => source.url)).toEqual([
|
|
"https://example.com/docs",
|
|
]);
|
|
expect(sources[0]?.occurrences[0]?.index).toBe(
|
|
markdown.indexOf("[citation:Docs]"),
|
|
);
|
|
});
|
|
|
|
it("still masks a code span that closes inside its own paragraph", () => {
|
|
const markdown = [
|
|
"Install it with `npm i",
|
|
"",
|
|
"Example: `[citation:Fake](https://example.com/fake)` then real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("ends a paragraph on a CRLF blank line, as the inline scanner does", () => {
|
|
const markdown =
|
|
"Install it with `npm i\r\n\r\nThe upstream guide is [citation:Docs](https://example.com/docs).\r\n\r\nThen run `npm start` to serve it.";
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/docs",
|
|
]);
|
|
});
|
|
|
|
it("ignores citations inside an unclosed fenced code block", () => {
|
|
const markdown = [
|
|
"Streaming output:",
|
|
"```md",
|
|
"[citation:Streaming](https://example.com/streaming)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown)).toEqual([]);
|
|
});
|
|
|
|
it("ends an inline span at a heading, thematic break or list marker", () => {
|
|
// CommonMark ends the paragraph at these block boundaries too, so "Use
|
|
// `npm" cannot steal the opener of `npm start` two lines later and the
|
|
// citation between them is a rendered link, not sample code.
|
|
for (const boundary of ["## Title", "---", "- item"]) {
|
|
const markdown = [
|
|
"Use `npm",
|
|
boundary,
|
|
"See [citation:X](https://example.com/x) and `npm start`.",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/x",
|
|
]);
|
|
}
|
|
});
|
|
|
|
it("ends an inline span at a blank line inside a blockquote", () => {
|
|
const markdown = [
|
|
"> Install it with `npm i",
|
|
">",
|
|
"> The guide is [citation:Docs](https://example.com/docs) and `npm start`.",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/docs",
|
|
]);
|
|
});
|
|
|
|
it("masks citations inside a fenced block nested in a list item", () => {
|
|
// The fence is indented past the list marker, so recognising it needs the
|
|
// container prefix; the blank line inside keeps it open across paragraphs.
|
|
const markdown = [
|
|
"- run:",
|
|
" ```md",
|
|
" [citation:Fake1](https://example.com/fake1)",
|
|
"",
|
|
" [citation:Fake2](https://example.com/fake2)",
|
|
" ```",
|
|
"",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("masks a blockquoted fence, and stops at the line that ends the quote", () => {
|
|
// Fake1 is inside the fence. The blank line has no `>` marker, so it closes
|
|
// the blockquote — and the fence with it — which makes Fake2 a rendered link
|
|
// again. Verified against remark-parse rather than by eye.
|
|
const markdown = [
|
|
"> ```md",
|
|
"> [citation:Fake1](https://example.com/fake1)",
|
|
"",
|
|
"> [citation:Fake2](https://example.com/fake2)",
|
|
"> ```",
|
|
"",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/fake2",
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("masks a dedented citation that lands in an indented code block", () => {
|
|
// Losing the `>` closes the block quote and the fence inside it, but the
|
|
// escaping line is not free: four columns of indentation at the top level
|
|
// is an indented code block, so Fake still renders as code. Verified
|
|
// against remark-parse, whose tree is quote>code, code, quote>code, paragraph.
|
|
const markdown = [
|
|
"> ```md",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
"> ```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("counts a tab-indented escaping line as the same four-column run", () => {
|
|
// A tab reaches the next tab stop, so `\t` is four columns and `\t\t` eight;
|
|
// both are indented code, and the marker padding on the opener is irrelevant.
|
|
const markdown = [
|
|
">\t```md",
|
|
"\t[citation:Fake](https://example.com/fake)",
|
|
">\t```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("keeps an indented run across a blank line inside it", () => {
|
|
// The CRLF variant pins the blankness classification: after a split on
|
|
// "\n" the blank line still carries its "\r", and reading it as content
|
|
// cleared the run and let Fake2 through.
|
|
for (const eol of ["\n", "\r\n"]) {
|
|
const markdown = [
|
|
"> ```md",
|
|
" [citation:Fake1](https://example.com/fake1)",
|
|
"",
|
|
" [citation:Fake2](https://example.com/fake2)",
|
|
"> ```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join(eol);
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
}
|
|
});
|
|
|
|
it("masks the indented code block that follows the blank line ending a quoted fence", () => {
|
|
// The blank line ends the blockquote, and the four-space line after it is a
|
|
// top-level indented code block, so Fake stays code instead of surfacing as
|
|
// a phantom source. Verified against remark-parse.
|
|
const markdown = [
|
|
"> ```md",
|
|
"",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
"> ```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("renders the paragraph that follows the blank line ending a quoted fence", () => {
|
|
// Two columns is paragraph territory, so the pending quote exit must not
|
|
// mask the line that classifies it: this pins the boundary against the
|
|
// four-column case above.
|
|
const markdown = [
|
|
"> ```md",
|
|
"",
|
|
" [citation:Para](https://example.com/para)",
|
|
"> ```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/para",
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("measures a quote escape against the list item the quote sat in", () => {
|
|
// The quote is a child of the list item and the item outlives it, so four
|
|
// columns of indentation are only two past the item's content column: the
|
|
// citation is paragraph content, not an indented code block. Verified
|
|
// against remark-parse.
|
|
const markdown = [
|
|
"- item",
|
|
" > ```md",
|
|
" [citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("still masks a quote escape indented past the surviving list item", () => {
|
|
// Six columns are four past the item's content column, so the escaped line
|
|
// does start an indented code block — and the unindented line after it ends
|
|
// the item and the run, rendering again.
|
|
const markdown = [
|
|
"- item",
|
|
" > ```md",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("ends the indented run at a dedent so that citation renders again", () => {
|
|
// Two columns is inside the item-less paragraph the run gives way to, so the
|
|
// second citation is a rendered link while the four-column one above stays
|
|
// code.
|
|
const markdown = [
|
|
"> ```md",
|
|
" [citation:Code](https://example.com/code)",
|
|
" [citation:Para](https://example.com/para)",
|
|
"> ```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/para",
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("resumes normal scanning once an indented run meets a quoted line", () => {
|
|
// The run ends at `> [citation:Quoted]`, which is quote content again, and
|
|
// the `> ``` ` below it opens the fence that the final line escapes.
|
|
const markdown = [
|
|
"> ```md",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
"> [citation:Quoted](https://example.com/quoted)",
|
|
"> ```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/quoted",
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("does not mask an escaping citation three columns in", () => {
|
|
// Three columns is not enough to start an indented code block, so the line
|
|
// is a top-level paragraph and its citation is a real rendered link. This
|
|
// pins the threshold against the four-column case above.
|
|
const markdown = [
|
|
"> ```md",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
"> ```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/fake",
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("does not open a fence from a backtick run in the middle of a line", () => {
|
|
const markdown = [
|
|
"Run ```md please",
|
|
"[citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("keeps a fence open across a shorter backtick run inside it", () => {
|
|
const markdown = [
|
|
"````md",
|
|
"```",
|
|
"[citation:Inside](https://example.com/inside)",
|
|
"```",
|
|
"````",
|
|
"",
|
|
"Outside [citation:Outside](https://example.com/outside).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/outside",
|
|
]);
|
|
});
|
|
|
|
it("keeps an indented top-level fence open for unindented content", () => {
|
|
// The opener's own indentation is not container indentation: content lines
|
|
// may sit further left than the marker, and the closer only has to come back
|
|
// within three columns of the opener.
|
|
const markdown = [
|
|
"Intro text.",
|
|
" ```md",
|
|
"[citation:Fake](https://example.com/fake)",
|
|
" ```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("treats a four-space indented marker inside a column-zero fence as content", () => {
|
|
const markdown = [
|
|
"```md",
|
|
" ```",
|
|
"[citation:Fake](https://example.com/fake)",
|
|
"```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("does not open a fence from a marker indented four columns at the top level", () => {
|
|
// Four columns past the container's content column is an indented code
|
|
// block, not a fence opener, so the citation below it renders as a link.
|
|
const markdown = [
|
|
" ```md",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("still opens a fence from a marker indented three columns at the top level", () => {
|
|
// The limit is three columns, so the next case up pins the boundary rather
|
|
// than overshooting it.
|
|
const markdown = [
|
|
" ```md",
|
|
"[citation:Fake](https://example.com/fake)",
|
|
" ```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("does not open a fence from a list marker with no space after it", () => {
|
|
// `-` has to be followed by indentation to start a list item, so `-```md`
|
|
// is ordinary paragraph text and nothing behind it is code.
|
|
const markdown = [
|
|
"-```md",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("counts a tab after a list marker as columns when it sizes the item", () => {
|
|
// `-<TAB>` advances to the next four-column stop, so this item's content
|
|
// starts at column four and the citation two spaces in has left the fence
|
|
// rather than sitting inside it.
|
|
const markdown = [
|
|
"-\t```md",
|
|
" [citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("ends a fence when a line drops out of the inner blockquote", () => {
|
|
const markdown = [
|
|
">> ```md",
|
|
">> [citation:Fake](https://example.com/fake)",
|
|
"> Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("lets a citation after a list-nested quote escape its fence", () => {
|
|
// `- > ```md` opens the quote behind a list marker, so the line has no
|
|
// leading `>` for a line-start pattern to find even though the fence really
|
|
// is quoted. The dedented citation below sits two columns past the item's
|
|
// content column, which is a paragraph rather than an indented code block.
|
|
const markdown = [
|
|
"- > ```md",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/fake",
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("ends a list-nested quote fence at its first unquoted line", () => {
|
|
const markdown = [
|
|
"- > ```md",
|
|
" [citation:One](https://example.com/one)",
|
|
" [citation:Two](https://example.com/two)",
|
|
"\t[citation:Three](https://example.com/three)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/one",
|
|
"https://example.com/two",
|
|
"https://example.com/three",
|
|
]);
|
|
});
|
|
|
|
it("escapes a quote fence nested behind a star marker", () => {
|
|
const markdown = [
|
|
"* > ```md",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/fake",
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("escapes a quote fence nested behind an ordered marker", () => {
|
|
// `1. ` gives the item a content column of three, so four columns of
|
|
// indentation is one column of content rather than an indented code block.
|
|
const markdown = [
|
|
"1. > ```md",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/fake",
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("keeps a list-nested quote open for a line that carries its marker", () => {
|
|
// The mirror of the cases above: `>` on the second line means the quote did
|
|
// not end, so nothing about the escape path applies here.
|
|
const markdown = [
|
|
"- >> ```md",
|
|
">> [citation:Fake](https://example.com/fake)",
|
|
"> [citation:Mid](https://example.com/mid)",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/fake",
|
|
"https://example.com/mid",
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("ends a list-nested fence at a closer that is itself inside the quote", () => {
|
|
const markdown = [
|
|
"- > ```md",
|
|
"> [citation:Fake](https://example.com/fake)",
|
|
"> ```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/fake",
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("gives a quoted closer its full three-column budget", () => {
|
|
// The one optional space after `>` belongs to the block quote marker, not to
|
|
// the fence, so `>` plus four spaces indents the closer by three columns and
|
|
// it closes the block.
|
|
const markdown = [
|
|
"> ```md",
|
|
"> [citation:Fake](https://example.com/fake)",
|
|
"> ```",
|
|
"> [citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("does not let a marker with an info string close the fence", () => {
|
|
// A closing fence is bare, so ` ```text ` here is literal content and the
|
|
// citation after it stays inside the block instead of becoming a source.
|
|
const markdown = [
|
|
"```md",
|
|
"```text",
|
|
"[citation:Fake](https://example.com/fake)",
|
|
"```",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("does not close a fence from list-looking content inside it", () => {
|
|
// Inside a fenced block `- ``` ` is literal content: a closing fence may
|
|
// carry indentation and a marker, nothing else. Reading it as a closer leaks
|
|
// Fake, and the real closer then masks away Real.
|
|
const markdown = [
|
|
"```md",
|
|
"- ```",
|
|
"[citation:Fake](https://example.com/fake)",
|
|
"```",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("closes a list-nested fence at the item's content column", () => {
|
|
// The same rule one container deeper: `- ```md` opens the fence at the list
|
|
// item's content column, ` - ``` ` stays literal content, and the bare
|
|
// ` ``` ` at that column closes it. Verified against remark-parse, not by
|
|
// eye.
|
|
const markdown = [
|
|
"- ```md",
|
|
" - ```",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
" ```",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("does not open a backtick fence whose info string holds a backtick", () => {
|
|
// ` ```md `x` ` is paragraph text, so the citation below it is a link the
|
|
// reader can reach and must not be masked away.
|
|
const markdown = [
|
|
"```md `x`",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("opens a tilde fence whatever its info string says", () => {
|
|
// Tilde fences take backticks in the info string, so the boundary above is
|
|
// about the marker character rather than a blanket rule.
|
|
const markdown = [
|
|
"~~~md `x`",
|
|
"[citation:Fake](https://example.com/fake)",
|
|
"~~~",
|
|
"Real [citation:Real](https://example.com/real).",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("keeps a literal quote marker inside an open list fence", () => {
|
|
// Once a fence is open its lines are literal, so the `>` here is code rather
|
|
// than a new block quote. Reading it as a container made the line look
|
|
// dedented to column zero, which escaped the column-two list fence and
|
|
// surfaced the sample's citation while reinterpreting the real closer as a
|
|
// top-level opener.
|
|
const markdown = [
|
|
"- ```md",
|
|
" > example",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
" ```",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("keeps a citation on a literal quote line inside a list fence", () => {
|
|
// Same shape with the citation on the quoted line itself: it stays code.
|
|
const markdown = [
|
|
"- ```md",
|
|
" > [citation:Fake](https://example.com/fake)",
|
|
" ```",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("keeps a citation inside a fence opened behind several list markers", () => {
|
|
// `- - - ```md` is a fence in the third list item, whose content column is
|
|
// six. Recording only the first marker put the fence at column two, which
|
|
// rejected the valid opener as four columns past its container.
|
|
const markdown = [
|
|
"- - - ```md",
|
|
" [citation:Fake1](https://example.com/fake1)",
|
|
"",
|
|
" [citation:Fake2](https://example.com/fake2)",
|
|
" ```",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("keeps a literal quote marker inside a quoted list fence", () => {
|
|
// The fence already consumed the quote it was opened in, so the second `>`
|
|
// here is literal code. `linePosition()` stripped both markers, which made
|
|
// the sample line look dedented past the fence's column and ended the fence
|
|
// one line early, surfacing the citation the renderer shows as code.
|
|
const markdown = [
|
|
"> - ```md",
|
|
"> > example",
|
|
"> [citation:Fake](https://example.com/fake)",
|
|
"> ```",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("keeps the list item open across a lazy paragraph continuation", () => {
|
|
// An unindented line of paragraph text continues the item's paragraph rather
|
|
// than leaving it, so the item's content column still measures the fence two
|
|
// columns in. Popping the item there recorded the fence at column zero, which
|
|
// swallowed the unindented citation to end of input.
|
|
const markdown = [
|
|
"- note",
|
|
"lazy continuation",
|
|
"",
|
|
" ~~~md",
|
|
" sample",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("reads five columns after a list marker as indented code, not padding", () => {
|
|
// Only up to four columns of whitespace pad a list marker; the fifth belongs
|
|
// to an indented code block inside an item whose content column is two.
|
|
// Consuming the whole run recorded column six, so the two-space fence looked
|
|
// like a dedent that left the item and the citation after it hid to EOF.
|
|
const markdown = [
|
|
"- code",
|
|
"",
|
|
" ~~~md",
|
|
" sample",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("restores the enclosing list when an ordinary quote ends", () => {
|
|
// The blank line ends `> note`, but the list item outlives the quote, so the
|
|
// two-space fence still belongs to an item whose content column is two. The
|
|
// citation at column zero leaves that item and ends the unclosed fence with
|
|
// it, which makes it a rendered link.
|
|
const markdown = [
|
|
"- item",
|
|
" > note",
|
|
"",
|
|
" ~~~md",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
" sample",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("restores the enclosing list when a quote ends at the fence line", () => {
|
|
// The same exit without a blank line: a fence cannot be lazily continued by
|
|
// a block quote, so `~~~md` already left the quote and the item's column is
|
|
// what the following lines measure against.
|
|
const markdown = [
|
|
"- item",
|
|
" > note",
|
|
" ~~~md",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
" sample",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("restores the enclosing list after a two-deep quote ends", () => {
|
|
const markdown = [
|
|
"- item",
|
|
" >> note",
|
|
"",
|
|
" ~~~md",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("does not open a fence from an ordered marker that cannot interrupt a paragraph", () => {
|
|
// An ordered list that does not start at 1 cannot interrupt a paragraph, so
|
|
// all three lines are one paragraph and its citation renders.
|
|
const markdown = [
|
|
"Intro",
|
|
"2. ~~~md",
|
|
" [citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("does not open a fence from a non-interrupting marker inside an item paragraph", () => {
|
|
const markdown = [
|
|
"- item",
|
|
" 2. ~~~md",
|
|
" [citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("still opens a fence from a `1.` marker that does interrupt the paragraph", () => {
|
|
const markdown = [
|
|
"Intro",
|
|
"1. ~~~md",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown)).toEqual([]);
|
|
});
|
|
|
|
it("still opens a fence from a `2.` marker after a block boundary", () => {
|
|
// The blank line closes the paragraph, so the ordered item starts a real
|
|
// container and its fence hides the citation again.
|
|
const markdown = [
|
|
"Intro",
|
|
"",
|
|
"2. ~~~md",
|
|
" [citation:Fake](https://example.com/fake)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown)).toEqual([]);
|
|
});
|
|
|
|
it("keeps the citation a newly entered quote renders as a paragraph", () => {
|
|
// Pinned against the parser rather than against a rule: remark's tree for
|
|
// these lines is blockquote > [paragraph "2. ```md", paragraph with the
|
|
// Fake link, code], so `2. ```md` never becomes a list item inside the
|
|
// fresh quote and `> ``` ` is the opener of an unclosed fence. Both
|
|
// citations render, and Sources has to surface both.
|
|
const markdown = [
|
|
"Intro",
|
|
"> 2. ```md",
|
|
">",
|
|
"> [citation:Fake](https://example.com/fake)",
|
|
"> ```",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/fake",
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("ends the paragraph for a setext underline over LF", () => {
|
|
const markdown = [
|
|
"Intro",
|
|
"---",
|
|
"2. ```md",
|
|
" [citation:Fake1](https://example.com/fake1)",
|
|
"",
|
|
" [citation:Fake2](https://example.com/fake2)",
|
|
" ```",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("ends the paragraph for a setext underline over CRLF", () => {
|
|
// The trailing "\r" belongs to the line ending, so `---` is still a setext
|
|
// underline and the paragraph is over: `2.` opens a real list, both of its
|
|
// citations are fenced code, and the ` ``` ` line closes that fence rather
|
|
// than starting a new one that would hide the rendered citation to EOF.
|
|
const markdown = [
|
|
"Intro",
|
|
"---",
|
|
"2. ```md",
|
|
" [citation:Fake1](https://example.com/fake1)",
|
|
"",
|
|
" [citation:Fake2](https://example.com/fake2)",
|
|
" ```",
|
|
"[citation:Real](https://example.com/real)",
|
|
].join("\r\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/real",
|
|
]);
|
|
});
|
|
|
|
it("uses the source domain when the citation label is generic", () => {
|
|
const markdown = "See [citation:Source](https://www.example.com/path).";
|
|
|
|
expect(extractCitationSources(markdown)[0]).toMatchObject({
|
|
title: "example.com",
|
|
domain: "example.com",
|
|
url: "https://www.example.com/path",
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("scans a deeply repeated blockquote chain in one pass", () => {
|
|
// tests/e2e/user-message-plain-text.spec.ts renders an AI message whose whole
|
|
// body is one line of thousands of `>` markers, and the citation pass runs on
|
|
// that text before the renderer does. Reading the containers a fence marker
|
|
// may sit behind has to reach each position once: an alternation that can take
|
|
// a space either as a step of its own or as the padding of the next marker
|
|
// describes the same prefix many ways, so a line carrying no fence leaves the
|
|
// engine to enumerate them all. Measured against the pattern this file guards,
|
|
// one check costs about a second at 26 markers and roughly doubles for each
|
|
// marker added, so a three-thousand-marker line never returns. The budget sits
|
|
// two orders of magnitude above what a single pass takes, which pins the growth
|
|
// rather than the machine.
|
|
const shapes: Array<[string, string]> = [
|
|
["> ".repeat(26) + "deep", "26-level quote"],
|
|
["- " + "> ".repeat(26) + "deep-list", "26-level quote behind a list"],
|
|
["> ".repeat(3000) + "deep", "3000-level quote"],
|
|
["- " + "> ".repeat(3000) + "deep-list", "3000-level quote behind a list"],
|
|
];
|
|
|
|
for (const [markdown, label] of shapes) {
|
|
it(`masks nothing in the ${label} shape within one pass of the line`, () => {
|
|
const started = performance.now();
|
|
const sources = extractCitationSources(markdown);
|
|
|
|
expect(sources).toEqual([]);
|
|
expect(performance.now() - started).toBeLessThan(500);
|
|
});
|
|
}
|
|
|
|
it("still reaches a citation that follows a deep quote chain", () => {
|
|
// Skipping the scan would keep every case above green, so the same shape is
|
|
// checked with real work to find after it.
|
|
const markdown = [
|
|
"> ".repeat(26) + "deep",
|
|
"[citation:Deep](https://example.com/deep)",
|
|
].join("\n");
|
|
|
|
expect(extractCitationSources(markdown).map((s) => s.url)).toEqual([
|
|
"https://example.com/deep",
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe("formatCitationMarkdownReference", () => {
|
|
it("formats a source as a reusable markdown reference", () => {
|
|
const [source] = extractCitationSources(
|
|
"Evidence [citation:Paper A](https://example.com/a).",
|
|
);
|
|
|
|
expect(formatCitationMarkdownReference(source!)).toBe(
|
|
"[Paper A](https://example.com/a)",
|
|
);
|
|
});
|
|
|
|
it("escapes brackets in the title so the copied link keeps its label", () => {
|
|
const [source] = extractCitationSources(
|
|
"Evidence [citation:Paper A](https://example.com/a).",
|
|
);
|
|
|
|
expect(
|
|
formatCitationMarkdownReference({
|
|
...source!,
|
|
title: "Why ] matters [draft",
|
|
}),
|
|
).toBe("[Why \\] matters \\[draft](https://example.com/a)");
|
|
});
|
|
|
|
it("wraps a URL with parentheses so the destination stays intact", () => {
|
|
const [source] = extractCitationSources(
|
|
"Evidence [citation:Paper A](https://example.com/a).",
|
|
);
|
|
|
|
expect(
|
|
formatCitationMarkdownReference({
|
|
...source!,
|
|
url: "https://example.com/a)b(c",
|
|
}),
|
|
).toBe("[Paper A](<https://example.com/a)b(c>)");
|
|
});
|
|
});
|