mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-07 19:20:34 +00:00
fix(bookmarks): avoid splitting surrogate pairs in automatic labels (#7966)
* fix(bookmarks): avoid splitting surrogate pairs in automatic labels * fix(bookmarks): sanitize unpaired surrogates before save
This commit is contained in:
parent
d1238c6974
commit
3a85baba9f
@ -73,6 +73,10 @@ file above stores user bookmarks, not deployment settings.
|
||||
most 10 excerpts (1,000 characters each) and a total count. Narrow the query for
|
||||
more matches; the page can load the full saved answer. Duplicate saves are
|
||||
idempotent and do not overwrite an existing label.
|
||||
Automatic labels keep up to 80 UTF-16 code units after whitespace normalization,
|
||||
stopping before a character if the cut would split its surrogate pair.
|
||||
Unpaired UTF-16 surrogates are replaced with U+FFFD in the saved copy and its
|
||||
label.
|
||||
- Storage uses SQLite and blocking I/O is offloaded from the event loop. This is a
|
||||
single-host example, not a multi-node storage contract or tenant administration UI.
|
||||
- Plugin browser/Python code is trusted deployment code. Shadow DOM scopes CSS,
|
||||
|
||||
@ -44,11 +44,18 @@ export default {
|
||||
);
|
||||
return;
|
||||
}
|
||||
// Unicode mode matches lone surrogates without touching intact pairs.
|
||||
const text = answer.text.replace(/[\uD800-\uDFFF]/gu, "\uFFFD");
|
||||
await services.callBackend("save", {
|
||||
thread_id: context.thread.thread_id,
|
||||
message_id: answer.id,
|
||||
label: answer.text.replace(/\s+/g, " ").trim().slice(0, 80),
|
||||
text: answer.text,
|
||||
// Keep the UTF-16 bound without leaving half a surrogate pair.
|
||||
label: text
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.slice(0, 80)
|
||||
.replace(/[\uD800-\uDBFF]$/u, ""),
|
||||
text,
|
||||
});
|
||||
services.showMessage(
|
||||
zh
|
||||
|
||||
40
frontend/tests/unit/core/extensions/bookmarks.test.ts
Normal file
40
frontend/tests/unit/core/extensions/bookmarks.test.ts
Normal file
@ -0,0 +1,40 @@
|
||||
import { expect, rs, test } from "@rstest/core";
|
||||
|
||||
import bookmarks from "../../../../../examples/deerflow-extension-bookmarks/deerflow_extension_bookmarks/static/dist/index.mjs";
|
||||
|
||||
const prefix = "a".repeat(77);
|
||||
|
||||
test.each([
|
||||
["split surrogate pair", `${prefix}\t b😀 remaining answer`, `${prefix} b`],
|
||||
["complete surrogate pair", `${prefix}\t😀`, `${prefix} 😀`],
|
||||
["consecutive emoji", "😀".repeat(80), "😀".repeat(40)],
|
||||
[
|
||||
"unpaired surrogates",
|
||||
"a\uD800b\uDC00c",
|
||||
"a\uFFFDb\uFFFDc",
|
||||
"a\uFFFDb\uFFFDc",
|
||||
],
|
||||
])(
|
||||
"keeps bookmark labels within 80 UTF-16 units: %s",
|
||||
async (_name, answer, label, savedAnswer = answer) => {
|
||||
const text = ` ${answer}\n`;
|
||||
const callBackend = rs.fn();
|
||||
const action = bookmarks.conversationActions(undefined, "en").actions[0]!;
|
||||
|
||||
await action.execute(
|
||||
{ thread: { thread_id: "thread-1" } },
|
||||
{
|
||||
latestVisibleAnswer: async () => ({ id: "answer-1", text }),
|
||||
callBackend,
|
||||
showMessage: rs.fn(),
|
||||
},
|
||||
);
|
||||
|
||||
expect(callBackend).toHaveBeenCalledExactlyOnceWith("save", {
|
||||
thread_id: "thread-1",
|
||||
message_id: "answer-1",
|
||||
label,
|
||||
text: ` ${savedAnswer}\n`,
|
||||
});
|
||||
},
|
||||
);
|
||||
Loading…
x
Reference in New Issue
Block a user