mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-09 13:39:26 +00:00
fix(messages): drop legacy <uploaded_files> tag handling (#4826)
* fix(messages): drop legacy <uploaded_files> tag handling (#4212) PR #4174 unified upload-context injection on <current_uploads> (IM and web both flow through UploadsMiddleware), and #4632 documented the current path. This removes the remaining backward-compat parsing of the pre-#4174 <uploaded_files> tag, the final cleanup item tracked by the issue: - deermem: only <current_uploads> is stripped from human turns before memory persistence, and the upload-sentence scrubber drops the legacy tag alternative. - mem0: the mirrored message filter recognises only <current_uploads>. - InputSanitizationMiddleware: remove the legacy tag from the blocked-tag denylist (it existed only because deermem parsed the old tag). - frontend: stripUploadedFilesTag / stripInternalMarkers / parseUploadedFiles and the message-list fallback parse only <current_uploads>; demo thread fixtures are migrated to the current tag. Scope decision: a <uploaded_files> block in pre-#4174 history is now treated as ordinary user content (pinned by tests in both layers) instead of being silently dropped or stripped. * style: apply prettier formatting to stripUploadedFilesTag * fix(uploads): keep legacy <uploaded_files> stripping for display/export only Addresses review feedback on #4826: removing the legacy tag from the frontend display layer made pre-#4174 threads render raw <uploaded_files> XML (with server-side upload paths) in chat, copy data, and JSON exports. The backend cleanup stands — memory pipelines and the sanitization denylist treat only <current_uploads> as an internal marker. The frontend keeps the legacy spelling in its display/export-only utilities (stripUploadedFilesTag / INTERNAL_MARKER_TAGS / parseUploadedFiles and the message-list fallback) so old history renders cleanly without leaking internal paths, while the memory/sanitization scope-decision tests remain unchanged. Frontend tests now pin both spellings: <current_uploads> and legacy <uploaded_files> are stripped from copy data, markdown leak-stripping, and JSON exports. * docs(ui): record accepted display-spoof tradeoff for legacy upload tag Review note (willem-bd): since <uploaded_files> is off the sanitization denylist, a live user can type the legacy spelling and fabricate file chips / hide their own message text in display. Display-only and self-inflicted with no backend semantics, so it is accepted for now; documented at both the message-list fallback and stripUploadedFilesTag. Age-gating the legacy spelling remains a possible follow-up. --------- Co-authored-by: betterkite <313258397+betterkite@users.noreply.github.com>
This commit is contained in:
betterkite
GitHub
parent
cdc886ae85
commit
b552b5015c
@ -13,7 +13,7 @@ import yaml
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_UPLOAD_BLOCK_RE = re.compile(r"<(?P<tag>uploaded_files|current_uploads)>[\s\S]*?</(?P=tag)>\n*", re.IGNORECASE)
|
||||
_UPLOAD_BLOCK_RE = re.compile(r"<current_uploads>[\s\S]*?</current_uploads>\n*", re.IGNORECASE)
|
||||
|
||||
_PATTERN_CACHE: dict[tuple[str, str | None], list[re.Pattern[str]]] = {}
|
||||
|
||||
@ -186,7 +186,7 @@ def filter_messages_for_memory(messages: list[Any], *, should_keep_hidden_messag
|
||||
if not keep:
|
||||
continue
|
||||
content_str = extract_message_text(msg)
|
||||
if "<uploaded_files>" in content_str.lower() or "<current_uploads>" in content_str.lower():
|
||||
if "<current_uploads>" in content_str.lower():
|
||||
stripped = _UPLOAD_BLOCK_RE.sub("", content_str).strip()
|
||||
if not stripped:
|
||||
skip_next_ai = True
|
||||
|
||||
@ -754,11 +754,11 @@ def format_conversation_for_update(messages: list[Any]) -> str:
|
||||
text_parts.append(text_val)
|
||||
content = " ".join(text_parts) if text_parts else str(content)
|
||||
|
||||
# Strip uploaded_files tags from human messages to avoid persisting
|
||||
# Strip the upload-context tag from human messages to avoid persisting
|
||||
# ephemeral file path info into long-term memory. Skip the turn entirely
|
||||
# when nothing remains after stripping (upload-only message).
|
||||
if role == "human":
|
||||
content = re.sub(r"<(?P<tag>uploaded_files|current_uploads)>[\s\S]*?</(?P=tag)>\n*", "", str(content)).strip()
|
||||
content = re.sub(r"<current_uploads>[\s\S]*?</current_uploads>\n*", "", str(content)).strip()
|
||||
if not content:
|
||||
continue
|
||||
|
||||
|
||||
@ -434,7 +434,7 @@ _UPLOAD_SENTENCE_RE = re.compile(
|
||||
r"upload(?:ed|ing)?(?:\s+\w+){0,3}\s+(?:file|files?|document|documents?|attachment|attachments?)"
|
||||
r"|file\s+upload"
|
||||
r"|/mnt/user-data/uploads/"
|
||||
r"|<(?:uploaded_files|current_uploads)>"
|
||||
r"|<current_uploads>"
|
||||
r")[^.!?]*[.!?]?\s*",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
@ -14,7 +14,7 @@ from collections.abc import Mapping
|
||||
from copy import copy
|
||||
from typing import Any
|
||||
|
||||
_UPLOAD_BLOCK_RE = re.compile(r"<(?P<tag>uploaded_files|current_uploads)>[\s\S]*?</(?P=tag)>\n*", re.IGNORECASE)
|
||||
_UPLOAD_BLOCK_RE = re.compile(r"<current_uploads>[\s\S]*?</current_uploads>\n*", re.IGNORECASE)
|
||||
|
||||
|
||||
def extract_message_text(message: Any) -> str:
|
||||
@ -70,7 +70,7 @@ def filter_messages_for_memory(messages: list[Any]) -> list[Any]:
|
||||
if additional_kwargs.get("hide_from_ui") and not _is_human_clarification_response(additional_kwargs):
|
||||
continue
|
||||
text = extract_message_text(msg)
|
||||
if "<uploaded_files>" in text.lower() or "<current_uploads>" in text.lower():
|
||||
if "<current_uploads>" in text.lower():
|
||||
stripped = _UPLOAD_BLOCK_RE.sub("", text).strip()
|
||||
if not stripped:
|
||||
# Upload-only turn: the following AI ack carries no user content.
|
||||
|
||||
@ -77,7 +77,6 @@ _BLOCKED_TAG_NAMES: frozenset[str] = frozenset(
|
||||
"critical_reminders",
|
||||
"response_style",
|
||||
"citations",
|
||||
"uploaded_files", # old uploads tag — still processed by deermem for backward-compat
|
||||
"current_uploads",
|
||||
"subagent_system",
|
||||
"skill_system",
|
||||
|
||||
@ -255,7 +255,7 @@ class TestMessageFiltering:
|
||||
assert filter_messages_for_memory([hidden, clarification]) == [clarification]
|
||||
|
||||
def test_upload_only_human_drops_it_and_following_ai(self) -> None:
|
||||
upload_only = HumanMessage(content="<uploaded_files>\nfile.pdf\n</uploaded_files>")
|
||||
upload_only = HumanMessage(content="<current_uploads>\nfile.pdf\n</current_uploads>")
|
||||
ack = AIMessage(content="I see your file")
|
||||
followup = HumanMessage(content="what is in it?")
|
||||
assert filter_messages_for_memory([upload_only, ack, followup]) == [followup]
|
||||
|
||||
@ -16,9 +16,13 @@ from deerflow.agents.memory.backends.deermem.deermem.core.updater import _strip_
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_UPLOAD_BLOCK = "<uploaded_files>\nThe following files have been uploaded and are available for use:\n\n- filename: secret.txt\n path: /mnt/user-data/uploads/abc123/secret.txt\n size: 42 bytes\n</uploaded_files>"
|
||||
# ``_UPLOAD_BLOCK`` uses the tag UploadsMiddleware actually emits since #4174.
|
||||
# ``_LEGACY_UPLOAD_BLOCK`` is the pre-#4174 tag: after the #4212 cleanup it is
|
||||
# treated as ordinary user content by the memory pipeline (see the dedicated
|
||||
# scope-decision tests below).
|
||||
_UPLOAD_BLOCK = "<current_uploads>\nThe following files have been uploaded and are available for use:\n\n- filename: secret.txt\n path: /mnt/user-data/uploads/abc123/secret.txt\n size: 42 bytes\n</current_uploads>"
|
||||
|
||||
_CURRENT_UPLOADS_BLOCK = "<current_uploads>\nThe following files have been uploaded in this run:\n\n- filename: report.pdf\n path: /mnt/user-data/uploads/def456/report.pdf\n size: 2048 bytes\n</current_uploads>"
|
||||
_LEGACY_UPLOAD_BLOCK = "<uploaded_files>\nThe following files have been uploaded and are available for use:\n\n- filename: report.pdf\n path: /mnt/user-data/uploads/def456/report.pdf\n size: 2048 bytes\n</uploaded_files>"
|
||||
|
||||
|
||||
def _human(text: str) -> HumanMessage:
|
||||
@ -41,7 +45,7 @@ class TestFilterMessagesForMemory:
|
||||
# --- upload-only turns are excluded ---
|
||||
|
||||
def test_upload_only_turn_is_excluded(self):
|
||||
"""A human turn containing only <uploaded_files> (no real question)
|
||||
"""A human turn containing only the upload-context tag (no real question)
|
||||
and its paired AI response must both be dropped."""
|
||||
msgs = [
|
||||
_human(_UPLOAD_BLOCK),
|
||||
@ -50,16 +54,6 @@ class TestFilterMessagesForMemory:
|
||||
result = filter_messages_for_memory(msgs)
|
||||
assert result == []
|
||||
|
||||
def test_upload_only_turn_is_excluded_current_uploads(self):
|
||||
"""Same as above but with <current_uploads> — the tag actually emitted
|
||||
by UploadsMiddleware in production."""
|
||||
msgs = [
|
||||
_human(_CURRENT_UPLOADS_BLOCK),
|
||||
_ai("I have read the report. It says: Q3 revenue up 12%."),
|
||||
]
|
||||
result = filter_messages_for_memory(msgs)
|
||||
assert result == []
|
||||
|
||||
def test_upload_with_real_question_preserves_question(self):
|
||||
"""When the user asks a question alongside an upload, the question text
|
||||
must reach the memory queue (upload block stripped, AI response kept)."""
|
||||
@ -72,25 +66,21 @@ class TestFilterMessagesForMemory:
|
||||
|
||||
assert len(result) == 2
|
||||
human_result = result[0]
|
||||
assert "<uploaded_files>" not in human_result.content
|
||||
assert "<current_uploads>" not in human_result.content
|
||||
assert "What does this file contain?" in human_result.content
|
||||
assert result[1].content == "The file contains: Hello DeerFlow."
|
||||
|
||||
def test_upload_with_question_preserves_question_current_uploads(self):
|
||||
"""Same as above but with <current_uploads> — the tag actually emitted
|
||||
by UploadsMiddleware in production."""
|
||||
combined = _CURRENT_UPLOADS_BLOCK + "\n\nSummarise this report please."
|
||||
def test_legacy_uploaded_files_block_is_plain_user_content(self):
|
||||
"""Scope decision for #4212: the pre-#4174 ``<uploaded_files>`` tag is no
|
||||
longer special-cased. A turn containing only that tag flows through as
|
||||
ordinary user content instead of being silently dropped."""
|
||||
msgs = [
|
||||
_human(combined),
|
||||
_ai("The report indicates Q3 revenue is up 12%."),
|
||||
_human(_LEGACY_UPLOAD_BLOCK),
|
||||
_ai("I see a legacy upload block."),
|
||||
]
|
||||
result = filter_messages_for_memory(msgs)
|
||||
|
||||
assert len(result) == 2
|
||||
human_result = result[0]
|
||||
assert "<current_uploads>" not in human_result.content
|
||||
assert "Summarise this report please." in human_result.content
|
||||
assert result[1].content == "The report indicates Q3 revenue is up 12%."
|
||||
assert result[0].content == _LEGACY_UPLOAD_BLOCK
|
||||
|
||||
# --- non-upload turns pass through unchanged ---
|
||||
|
||||
@ -159,7 +149,7 @@ class TestFilterMessagesForMemory:
|
||||
result = filter_messages_for_memory(msgs)
|
||||
all_content = " ".join(m.content for m in result if isinstance(m.content, str))
|
||||
assert "/mnt/user-data/uploads/" not in all_content
|
||||
assert "<uploaded_files>" not in all_content
|
||||
assert "<current_uploads>" not in all_content
|
||||
|
||||
# --- hide_from_ui messages are excluded ---
|
||||
|
||||
@ -399,6 +389,13 @@ class TestStripUploadMentionsFromMemory:
|
||||
result = _strip_upload_mentions_from_memory(mem)
|
||||
assert len(result["facts"]) == 2
|
||||
|
||||
def test_legacy_uploaded_files_tag_sentence_preserved(self):
|
||||
"""Scope decision for #4212: only the ``<current_uploads>`` tag is
|
||||
stripped from stored summaries; the pre-#4174 tag is ordinary text."""
|
||||
mem = self._make_memory("User works on reports. <uploaded_files>session.pdf</uploaded_files> is old context.")
|
||||
result = _strip_upload_mentions_from_memory(mem)
|
||||
assert "<uploaded_files>session.pdf</uploaded_files>" in result["user"]["topOfMind"]["summary"]
|
||||
|
||||
def test_empty_memory_handled_gracefully(self):
|
||||
mem = {"user": {}, "history": {}, "facts": []}
|
||||
result = _strip_upload_mentions_from_memory(mem)
|
||||
|
||||
@ -508,7 +508,7 @@ def test_skill_activation_middleware_uses_original_user_content_when_uploads_are
|
||||
|
||||
middleware = SkillActivationMiddleware(slash_source_owner_token=_SLASH_SOURCE_OWNER_TOKEN)
|
||||
original = HumanMessage(
|
||||
content="<uploaded_files>\n- report.pdf\n</uploaded_files>\n\n/data-analysis 分析这个文档",
|
||||
content="<current_uploads>\n- report.pdf\n</current_uploads>\n\n/data-analysis 分析这个文档",
|
||||
id="msg-1",
|
||||
additional_kwargs={ORIGINAL_USER_CONTENT_KEY: "/data-analysis 分析这个文档"},
|
||||
)
|
||||
|
||||
@ -365,7 +365,7 @@ def test_prepare_regenerate_payload_returns_clean_input_and_base_checkpoint():
|
||||
|
||||
human = HumanMessage(
|
||||
id="human-1",
|
||||
content="<uploaded_files>injected</uploaded_files>\n\n/data-analysis analyze data.csv",
|
||||
content="<current_uploads>injected</current_uploads>\n\n/data-analysis analyze data.csv",
|
||||
additional_kwargs={
|
||||
ORIGINAL_USER_CONTENT_KEY: "/data-analysis analyze data.csv",
|
||||
"files": [{"filename": "data.csv", "path": "/mnt/user-data/uploads/data.csv"}],
|
||||
@ -649,7 +649,7 @@ def test_prepare_edit_regenerate_payload_returns_new_human_and_edit_metadata():
|
||||
|
||||
human = HumanMessage(
|
||||
id="human-1",
|
||||
content="<uploaded_files>injected</uploaded_files>\n\noriginal question",
|
||||
content="<current_uploads>injected</current_uploads>\n\noriginal question",
|
||||
name="researcher",
|
||||
additional_kwargs={
|
||||
ORIGINAL_USER_CONTENT_KEY: "original question",
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
"id": "47dcc555-9787-4ce6-88fd-cb4d728243ac"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
@ -108,7 +108,7 @@
|
||||
"id": "800c7d6e-d553-4f30-90e1-5c8d1d71c083"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
@ -147,7 +147,7 @@
|
||||
"id": "e46db0ab-38c5-4f02-bbdd-fdef7c7a708c"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
"id": "ef6ba42d-88c7-4f64-80c0-e3d0dc8fc381"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
@ -103,7 +103,7 @@
|
||||
"id": "1a49946d-9b79-4805-a959-5eb983010982"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
"id": "bd5f52dd-e7c1-4a05-9511-870fb47c6950"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
"id": "c6794328-e667-41fd-8067-b7314bcc4838"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
@ -511,7 +511,7 @@
|
||||
"id": "177797fd-7a9f-480a-8c6e-005dd2db3e59"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
"id": "37aacd32-f56f-4bb3-8184-ebcafde0bd14"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
"id": "82879088-d086-4aa3-aa90-9d1cfaf25a12"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
@ -142,7 +142,7 @@
|
||||
"id": "f97120b2-0071-4454-85b8-8bb636833401"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
@ -375,7 +375,7 @@
|
||||
"id": "9da6213d-a1eb-4170-a61d-8378e6a680a1"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
"id": "ac8f27cd-0f2e-4a82-a432-f4b37d18a846"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nThe following files have been uploaded and are available for use:\n\n- titanic.csv (58.9 KB)\n Path: /mnt/user-data/uploads/titanic.csv\n\nYou can read these files using the `read_file` tool with the paths shown above.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nThe following files have been uploaded and are available for use:\n\n- titanic.csv (58.9 KB)\n Path: /mnt/user-data/uploads/titanic.csv\n\nYou can read these files using the `read_file` tool with the paths shown above.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
"id": "69411c14-efea-4c7f-a322-2dd541b98fda"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
@ -565,7 +565,7 @@
|
||||
"id": "5b5a187a-5b2d-4c9e-b6f7-817fe9c12330"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
"id": "7093f725-fdee-47b0-b135-abeaea804aff"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
"id": "6409a240-5eeb-4df5-8681-4ad80a1daf7e"
|
||||
},
|
||||
{
|
||||
"content": "<uploaded_files>\nNo files have been uploaded yet.\n</uploaded_files>",
|
||||
"content": "<current_uploads>\nNo files have been uploaded yet.\n</current_uploads>",
|
||||
"additional_kwargs": {},
|
||||
"response_metadata": {},
|
||||
"type": "system",
|
||||
|
||||
@ -430,6 +430,11 @@ function MessageContent_({
|
||||
rawContent.includes("<uploaded_files>")
|
||||
) {
|
||||
// If the content contains an upload context tag, we return the parsed files from the content for backward compatibility.
|
||||
// <uploaded_files> is display-only compat for pre-#4174 history (#4212).
|
||||
// Accepted tradeoff (review): a live user typing the legacy spelling can
|
||||
// fabricate chips / hide their own message text — display-only and
|
||||
// self-inflicted, no backend semantics. Age-gating the legacy spelling
|
||||
// is a possible follow-up if this ever matters.
|
||||
return parseUploadedFiles(rawContent);
|
||||
}
|
||||
return null;
|
||||
|
||||
@ -846,6 +846,18 @@ export interface FileInMessage {
|
||||
* Strip backend-injected human context tags from message content.
|
||||
* Kept under its historical name because callers use it for uploaded-file
|
||||
* display cleanup.
|
||||
*
|
||||
* Display-only backward compatibility for #4212: ``<uploaded_files>`` is no
|
||||
* longer emitted by the backend and is treated as plain content by the
|
||||
* memory/sanitization pipelines, but threads persisted before #4174 still
|
||||
* carry legacy blocks in their history. This display/export layer keeps
|
||||
* stripping it so old threads render cleanly instead of showing raw XML
|
||||
* with server-side upload paths.
|
||||
*
|
||||
* Accepted tradeoff (review): a live user typing the legacy spelling can
|
||||
* hide their own message text / fabricate file chips — display-only and
|
||||
* self-inflicted, with no backend semantics. Age-gating the legacy
|
||||
* spelling is a possible follow-up if this ever matters.
|
||||
*/
|
||||
export function stripUploadedFilesTag(content: string): string {
|
||||
return content
|
||||
@ -862,8 +874,9 @@ export function stripUploadedFilesTag(content: string): string {
|
||||
*
|
||||
* These markers are *not* user copy — they come from:
|
||||
*
|
||||
* - ``UploadsMiddleware`` → ``<current_uploads>`` (``<uploaded_files>``
|
||||
* before #4174; still emitted by IM channels and present in history)
|
||||
* - ``UploadsMiddleware`` → ``<current_uploads>`` (``<uploaded_files>`` is
|
||||
* the pre-#4174 spelling, still stripped here for display/export only so
|
||||
* legacy history does not leak raw blocks or server paths — see #4212)
|
||||
* - ``SkillActivationMiddleware`` → ``<slash_skill_activation>``
|
||||
* - ``DynamicContextMiddleware`` → ``<system-reminder>`` (carrying
|
||||
* ``<memory>`` / ``<current_date>`` inside)
|
||||
@ -926,8 +939,9 @@ function parseHumanReadableSize(raw: string): number {
|
||||
}
|
||||
|
||||
export function parseUploadedFiles(content: string): FileInMessage[] {
|
||||
// Match the upload context block; the tag name depends on backend version
|
||||
// (<current_uploads> since #4174, <uploaded_files> before / on IM paths).
|
||||
// Match the upload context block. <current_uploads> is what
|
||||
// UploadsMiddleware emits (#4174); <uploaded_files> is kept for
|
||||
// display-only backward compatibility with pre-#4174 history (#4212).
|
||||
const uploadedFilesRegex =
|
||||
/<(current_uploads|uploaded_files)>([\s\S]*?)<\/\1>/;
|
||||
// eslint-disable-next-line @typescript-eslint/prefer-regexp-exec
|
||||
|
||||
@ -141,7 +141,8 @@ function buildJSONMessage(
|
||||
): JSONExportMessage | null {
|
||||
// Run the same sanitiser the Markdown path uses so the JSON `content`
|
||||
// field never carries inline `<think>...</think>` wrappers, content-array
|
||||
// thinking blocks, `<uploaded_files>` markers, or other internal payloads.
|
||||
// thinking blocks, `<current_uploads>`/`<uploaded_files>` markers, or other
|
||||
// internal payloads.
|
||||
const content = formatMessageContent(msg);
|
||||
const reasoning =
|
||||
options.includeReasoning && msg.type === "ai"
|
||||
|
||||
@ -584,9 +584,12 @@ describe("isHiddenFromUIMessage", () => {
|
||||
});
|
||||
|
||||
describe("human message internal context stripping", () => {
|
||||
test("strips uploaded file context from copy data", () => {
|
||||
test("strips legacy uploaded_files context from copy data", () => {
|
||||
// Display-only backward compatibility (#4212): pre-#4174 history still
|
||||
// carries <uploaded_files> blocks, which copy data must strip rather
|
||||
// than leak as raw XML with server-side paths.
|
||||
const message = {
|
||||
id: "human-with-upload",
|
||||
id: "human-with-legacy-upload",
|
||||
type: "human",
|
||||
content:
|
||||
"<uploaded_files>\nThe following files were uploaded in this message:\n\n- paper.pdf (1.0 MB)\n Path: /mnt/user-data/uploads/paper.pdf\n</uploaded_files>\n\nSummarize this paper",
|
||||
|
||||
@ -321,7 +321,16 @@ test("stripLeakedSystemTags handles no tags present", () => {
|
||||
expect(stripLeakedSystemTags(input)).toBe(input);
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags strips <uploaded_files> tag", () => {
|
||||
test("stripLeakedSystemTags strips <current_uploads> tag", () => {
|
||||
expect(
|
||||
stripLeakedSystemTags("<current_uploads>file.pdf</current_uploads>"),
|
||||
).toBe("file.pdf");
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags strips legacy <uploaded_files> tag", () => {
|
||||
// Display-only backward compatibility (#4212): pre-#4174 history still
|
||||
// carries <uploaded_files> blocks; the leaked-tag stripper keeps handling
|
||||
// the legacy spelling so old threads do not render raw XML.
|
||||
expect(
|
||||
stripLeakedSystemTags("<uploaded_files>file.pdf</uploaded_files>"),
|
||||
).toBe("file.pdf");
|
||||
|
||||
@ -218,10 +218,24 @@ describe("formatThreadAsJSON", () => {
|
||||
expect(raw).toContain("final visible text");
|
||||
});
|
||||
|
||||
it("strips <uploaded_files> markers from content", () => {
|
||||
it("strips <current_uploads> markers from content", () => {
|
||||
const message = human(
|
||||
"real prompt\n<current_uploads>\n/mnt/user-data/uploads/secret.pdf\n</current_uploads>",
|
||||
{ id: "h-clean" } as Partial<Message>,
|
||||
);
|
||||
const raw = formatThreadAsJSON(makeThread(), [message]);
|
||||
expect(raw).not.toContain("<current_uploads>");
|
||||
expect(raw).not.toContain("secret.pdf");
|
||||
expect(raw).toContain("real prompt");
|
||||
});
|
||||
|
||||
it("strips legacy <uploaded_files> markers from content", () => {
|
||||
// Display-only backward compatibility (#4212): pre-#4174 history still
|
||||
// carries <uploaded_files> blocks; exports must keep stripping the
|
||||
// legacy spelling so server-side upload paths never leak.
|
||||
const message = human(
|
||||
"real prompt\n<uploaded_files>\n/mnt/user-data/uploads/secret.pdf\n</uploaded_files>",
|
||||
{ id: "h-clean" } as Partial<Message>,
|
||||
{ id: "h-legacy-clean" } as Partial<Message>,
|
||||
);
|
||||
const raw = formatThreadAsJSON(makeThread(), [message]);
|
||||
expect(raw).not.toContain("<uploaded_files>");
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user