fix(llm): preserve unrelated think-prefixed tags (#6505)

* fix(llm): preserve unrelated think-prefixed tags

* fix(llm): handle self-closing think tags consistently

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>

README.md

@@ -1738,6 +1738,8 @@ Rebuild with `make up` after changing the managed extension set. See
 
 Gateway-generated follow-up suggestions now normalize both plain-string model output and block/list-style rich content before parsing the JSON array response, so provider-specific content wrappers do not silently drop suggestions.
 
+Backend response cleanup preserves unrelated tag names such as `<think-tank>` and `<think:note>` in follow-up suggestions and polished drafts; only the exact `<think>` name (optionally followed by attributes) starts a reasoning block. Self-closing `<think/>` and `<think />` tags are empty reasoning blocks and leave the following answer intact.
+
 The Web UI composer can polish draft input before sending. The rewrite runs as a short Gateway LLM request using the `input_polish` model configuration, keeps slash skill prefixes such as `/data-analysis`, and only replaces the local draft after the user clicks the polish button; it does not create a thread run or persist a message.
 
 When the agent asks for clarification, the Web UI shows the structured response card but keeps the normal composer available. Users can complete the card or send a free-form chat message to bypass it; that message closes the latest pending clarification and becomes the agent's next input. Accompanying answer text stays outside the execution steps panel, and answering the request keeps previously completed text in the conversation while the agent continues.

backend/packages/harness/deerflow/utils/AGENTS.md

@@ -36,6 +36,14 @@ suffix. Keep its case-insensitive tag handling, optional whitespace before
 the closing `>`, and the `truncate_unclosed` behavior. Regression coverage
 lives in `tests/test_utils_llm_text.py`.
 
+Opening reasoning tags require whitespace, `>`, or `/>` after the exact `think`
+name; punctuated names such as `<think-tank>`, `<think:note>`, and `<think/other>`
+are ordinary content. The shared prefix regex owns this boundary rule; do not
+duplicate it in a manual re-check. Self-closing `<think/>` and `<think />` tags
+are empty reasoning blocks: remove the tag without consuming subsequent text.
+Keep exact unfinished `<think` prefixes hidden in leading summaries. Coverage:
+`tests/test_think_tag_boundaries.py`.
+
 Display summaries use `strip_leading_think_blocks` before limiting text;
 `test_run_journal_visible_summary.py` preserves literal tags in the answer.
 

backend/packages/harness/deerflow/utils/llm_text.py

@@ -4,7 +4,10 @@ from __future__ import annotations
 
 import re
 
-_THINK_OPEN_PREFIX_RE = re.compile(r"<think\b", re.IGNORECASE)
+# A word boundary also accepts punctuation in unrelated XML/HTML tag names.
+# Keep the exact unfinished prefix for leading-summary cleanup.
+# Accept the self-closing delimiter without treating <think/other> as reasoning.
+_THINK_OPEN_PREFIX_RE = re.compile(r"<think(?=[\s>]|/>|$)", re.IGNORECASE)
 _THINK_CLOSE_PREFIX_RE = re.compile(r"</think", re.IGNORECASE)
 
 
@@ -19,6 +22,20 @@ def _find_think_open(text: str, start: int) -> tuple[int, int] | None:
     return match.start(), end + 1
 
 
+def _is_self_closing_think_tag(text: str, start: int, end: int) -> bool:
+    """Recognize a terminal slash only when it is outside quoted attributes."""
+    if text[end - 2] != "/":
+        return False
+    quote = ""
+    for char in text[start : end - 1]:
+        if quote:
+            if char == quote:
+                quote = ""
+        elif char in {"'", '"'}:
+            quote = char
+    return not quote
+
+
 def _find_think_close(text: str, start: int) -> tuple[int, int] | None:
     """Find the first closing tag, allowing whitespace before its final ``>``."""
     while match := _THINK_CLOSE_PREFIX_RE.search(text, start):
@@ -34,7 +51,8 @@ def _find_think_close(text: str, start: int) -> tuple[int, int] | None:
 def strip_think_blocks(text: str, *, truncate_unclosed: bool = True) -> str:
     """Remove inline reasoning ``<think>`` blocks from a model response.
 
-    Complete ``<think>...</think>`` blocks are always removed. A dangling,
+    Complete ``<think>...</think>`` blocks are always removed. Self-closing
+    ``<think/>`` tags are empty blocks and leave following text intact. A dangling,
     unclosed ``<think>`` open tag is treated as a model that was truncated
     mid-thought: when ``truncate_unclosed`` is True (the default, used by JSON
     parsers like suggestions/goal where trailing garbage must be dropped) the
@@ -46,6 +64,10 @@ def strip_think_blocks(text: str, *, truncate_unclosed: bool = True) -> str:
     parts: list[str] = []
     start = 0
     while (opening := _find_think_open(text, start)) is not None:
+        if _is_self_closing_think_tag(text, *opening):
+            parts.append(text[start : opening[0]])
+            start = opening[1]
+            continue
         closing = _find_think_close(text, opening[1])
         if closing is None:
             if truncate_unclosed:
@@ -62,7 +84,8 @@ def strip_leading_think_blocks(text: str) -> str:
 
     Unlike structured-response parsing, a displayed answer may explain the
     tag in prose or code. Only a leading XML-style reasoning section is
-    removed; an unfinished section has no answer to summarize.
+    removed; an unfinished section has no answer to summarize. Leading
+    self-closing tags are removed as empty reasoning sections.
     """
     start = 0
     while True:
@@ -74,9 +97,12 @@ def strip_leading_think_blocks(text: str) -> str:
         if (whitespace_start == 0 or newline >= 0) and ("\t" in indent or indent.startswith("    ")):
             break  # Root-level indented Markdown code is a literal example.
         opening = _THINK_OPEN_PREFIX_RE.match(text, start)
-        if opening is None or (opening.end() < len(text) and not (text[opening.end()].isspace() or text[opening.end()] == ">")):
+        if opening is None:
             break
         end = text.find(">", opening.end())
+        if end >= 0 and _is_self_closing_think_tag(text, start, end + 1):
+            start = end + 1
+            continue
         closing = _find_think_close(text, end + 1) if end >= 0 else None
         if closing is None:
             return ""

backend/tests/test_think_tag_boundaries.py

@@ -0,0 +1,103 @@
+"""Reasoning cleanup must not consume unrelated tag names."""
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from app.gateway.routers import input_polish, suggestions
+from deerflow.utils.llm_text import strip_leading_think_blocks, strip_think_blocks
+
+
+@pytest.mark.parametrize("name", ["think-tank", "think:note", "think.note", "think/other", "think=note", "THINK-card"])
+@pytest.mark.parametrize("truncate_unclosed", [True, False])
+def test_unrelated_tag_names_survive_between_real_reasoning_blocks(name: str, truncate_unclosed: bool) -> None:
+    literal = f"<{name}>public content</{name}>"
+    text = f"<think>first reasoning</think>before{literal}<think>second reasoning</think>after"
+    assert strip_think_blocks(text, truncate_unclosed=truncate_unclosed) == f"before{literal}after"
+
+
+@pytest.mark.parametrize("name", ["think-tank", "think:note", "think.note", "THINK-card"])
+def test_unrelated_unclosed_tag_does_not_truncate_response(name: str) -> None:
+    text = f"Explain <{name}> without dropping this answer."
+    assert strip_think_blocks(text) == text
+
+
+@pytest.mark.parametrize("text", ["<think", "<THINK", "<think>", "<think class='reasoning'>"])
+def test_leading_unfinished_reasoning_still_has_no_visible_answer(text: str) -> None:
+    assert strip_leading_think_blocks(text) == ""
+
+
+@pytest.mark.parametrize("tag", ["<think/>", "<think />", "<THINK class='reasoning'/>"])
+@pytest.mark.parametrize("truncate_unclosed", [True, False])
+def test_self_closing_think_tag_preserves_text_before_next_reasoning_block(tag: str, truncate_unclosed: bool) -> None:
+    text = f"before{tag}middle<think>private reasoning</think>after"
+    assert strip_think_blocks(text, truncate_unclosed=truncate_unclosed) == "beforemiddleafter"
+
+
+@pytest.mark.parametrize("tag", ["<think/>", "<think />", "<THINK class='reasoning'/>"])
+def test_leading_self_closing_think_tag_preserves_answer_and_literal_tags(tag: str) -> None:
+    assert strip_leading_think_blocks(f"{tag}answer <think>literal example</think>") == "answer <think>literal example</think>"
+
+
+@pytest.mark.parametrize("cleaner", [strip_think_blocks, strip_leading_think_blocks], ids=["inline", "leading"])
+def test_self_closing_and_complete_reasoning_blocks_can_be_combined(cleaner) -> None:
+    assert cleaner("<think/><think>private reasoning</think><think />answer") == "answer"
+
+
+@pytest.mark.parametrize("cleaner", [strip_think_blocks, strip_leading_think_blocks], ids=["inline", "leading"])
+@pytest.mark.parametrize("quote", ["'", '"'])
+def test_self_closing_delimiter_inside_quoted_attribute_does_not_expose_reasoning(cleaner, quote: str) -> None:
+    text = f"<think note={quote}/>{quote}>private reasoning</think>answer"
+    assert cleaner(text) == "answer"
+
+
+@pytest.mark.parametrize("name", ["think/other", "think//"])
+def test_unrelated_slash_tag_remains_visible_in_leading_answer(name: str) -> None:
+    text = f"<{name}>public content</{name}>"
+    assert strip_leading_think_blocks(text) == text
+
+
+@pytest.mark.parametrize("tag", ["<think/>", "<think />"])
+@pytest.mark.parametrize("indent", ["    ", "\t"])
+def test_self_closing_think_tag_in_indented_code_remains_literal(tag: str, indent: str) -> None:
+    text = f"{indent}{tag}\nanswer"
+    assert strip_leading_think_blocks(text) == text.strip()
+
+
+@pytest.mark.parametrize("tag", ["<think/>", "<think />"])
+def test_self_closing_think_tag_does_not_hide_suggestions(tag: str) -> None:
+    assert suggestions._parse_json_string_list(f'{tag}["What next?"]') == ["What next?"]
+
+
+@pytest.mark.parametrize("tag", ["<think/>", "<think />"])
+def test_input_polish_removes_self_closing_tag_without_losing_draft(tag: str) -> None:
+    assert input_polish._clean_rewritten_text(f"before{tag}after") == "beforeafter"
+
+
+def test_suggestions_parser_preserves_unrelated_tag_inside_json() -> None:
+    text = '<think>choose questions</think>\n["How does <think-tank> work?", "What next?"]'
+    assert suggestions._parse_json_string_list(text) == ["How does <think-tank> work?", "What next?"]
+
+
+def test_generate_suggestions_keeps_questions_with_unrelated_tags(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(suggestions, "get_current_user_from_request", AsyncMock(return_value=SimpleNamespace()))
+    monkeypatch.setattr(suggestions, "authorize_model_use", lambda *_args, **_kwargs: None)
+    invoke = AsyncMock(return_value='<think>choose questions</think>\n["How does <think-tank> work?", "What next?"]')
+    monkeypatch.setattr(suggestions, "run_oneshot_llm", invoke)
+    result = asyncio.run(
+        suggestions.generate_suggestions.__wrapped__(
+            "thread-tags",
+            suggestions.SuggestionsRequest(messages=[suggestions.SuggestionMessage(role="user", content="Explain custom HTML tags")], n=2),
+            request=None,
+            config=SimpleNamespace(suggestions=SimpleNamespace(enabled=True, max_suggestions=2)),
+        )
+    )
+    assert result.suggestions == ["How does <think-tank> work?", "What next?"]
+    invoke.assert_awaited_once()
+
+
+def test_input_polish_keeps_unrelated_tag_before_real_reasoning() -> None:
+    text = "Use <think-tank>public content</think-tank>.<think>private reasoning</think>"
+    assert input_polish._clean_rewritten_text(text) == "Use <think-tank>public content</think-tank>."