fix(parsing): keep literal think tags inside JSON string values (#7907)

* fix(parsing): keep literal think tags inside JSON string values

Structured parsers ran strip_think_blocks over the whole response before
json.loads. A model answer that quotes the tag -- "Explain <think>example
</think>" -- lost the quoted text, and an unclosed literal tag inside a
string cut the JSON document mid-string, so suggestions returned None and
goal evaluation failed to parse.

Strip reasoning only outside JSON string literals: leading blocks are still
removed whole (a JSON example inside reasoning is never taken as the
answer), and blocks after the payload starts are removed only outside
quotes. Suggestions and the goal evaluator both use the shared helper.

* fix(parsing): keep think-tag stripping complete outside the JSON payload

The string-aware walker only covered the payload itself: a reasoning block
after prose and before the payload was copied verbatim, so brackets inside
it became the parsers' find/rfind targets, and an unclosed block in
trailing prose was kept where strip_think_blocks used to truncate.

Find the payload start while stepping over think blocks, strip the head with
strip_think_blocks, keep strip/truncate semantics after the top-level
payload closes, and truncate at an unclosed block outside a string. Adds a
CHANGELOG entry for the user-visible parsing change.

CHANGELOG.md

@@ -28,7 +28,14 @@ This section accumulates work toward the **2.3.0** milestone
   longer create or acquire a sandbox before the first sandbox-backed tool
   call. Existing sandbox bindings and providers that require skill uploads
   retain their eager synchronization behavior. ([#7782])
-  
+
+- **parsing:** Follow-up suggestions and the `/goal` evaluator keep a literal
+  `<think>` tag that appears inside a JSON string value, such as an answer
+  quoting the tag, while reasoning around the JSON is still removed. An
+  unclosed literal tag inside a string no longer truncates the document and
+  fails parsing, and reasoning outside the payload still strips and truncates
+  as before. ([#7907])
+
 - **gateway:** A caller-supplied thread ID that is a Windows reserved device
   name (`NUL`, `CON`, `COM1`, ...) is now rejected at the shared
   `validate_thread_id()` boundary. Such IDs previously passed validation but
@@ -16029,7 +16036,8 @@ with **180 merged pull requests** since the first 2.0 milestone tag.
 [#7762]: https://github.com/bytedance/deer-flow/pull/7762
 [#7772]: https://github.com/bytedance/deer-flow/pull/7772
 [#7781]: https://github.com/bytedance/deer-flow/pull/7781
+[#7782]: https://github.com/bytedance/deer-flow/pull/7782
 [#7792]: https://github.com/bytedance/deer-flow/pull/7792
 [#7831]: https://github.com/bytedance/deer-flow/pull/7831
-[#7782]: https://github.com/bytedance/deer-flow/pull/7782
 [#7892]: https://github.com/bytedance/deer-flow/pull/7892
+[#7907]: https://github.com/bytedance/deer-flow/pull/7907

backend/app/gateway/routers/suggestions.py

@@ -38,11 +38,11 @@ class SuggestionsConfigResponse(BaseModel):
 
 
 _strip_markdown_code_fence = llm_text.strip_markdown_code_fence
-_strip_think_blocks = llm_text.strip_think_blocks
+_strip_think_blocks_outside_json_strings = llm_text.strip_think_blocks_outside_json_strings
 
 
 def _parse_json_string_list(text: str) -> list[str] | None:
-    candidate = _strip_think_blocks(text)
+    candidate = _strip_think_blocks_outside_json_strings(text)
     candidate = _strip_markdown_code_fence(candidate)
     start = candidate.find("[")
     end = candidate.rfind("]")

backend/packages/harness/deerflow/runtime/goal.py

@@ -68,7 +68,7 @@ GOAL_OUTCOME_CHANNEL = "goal_outcome"
 
 _extract_response_text = llm_text.extract_response_text
 _strip_markdown_code_fence = llm_text.strip_markdown_code_fence
-_strip_think_blocks = llm_text.strip_think_blocks
+_strip_think_blocks_outside_json_strings = llm_text.strip_think_blocks_outside_json_strings
 
 _goal_locks = AsyncKeyedLockTable[str]()
 
@@ -157,7 +157,7 @@ def build_goal_outcome(goal: GoalState, evaluation: GoalEvaluation, *, reply_mes
 
 def parse_goal_evaluation_response(text: str, *, require_assumption_attribution: bool = False) -> GoalEvaluation:
     """Parse the evaluator's JSON object response."""
-    candidate = _strip_markdown_code_fence(_strip_think_blocks(text))
+    candidate = _strip_markdown_code_fence(_strip_think_blocks_outside_json_strings(text))
     start = candidate.find("{")
     end = candidate.rfind("}")
     if start == -1 or end == -1 or end <= start:

backend/packages/harness/deerflow/utils/llm_text.py

@@ -153,3 +153,107 @@ def extract_response_text(content: object) -> str:
     if content is None:
         return ""
     return str(content)
+
+
+def _json_payload_start(text: str) -> int:
+    """Index of the first ``{``/``[`` outside a reasoning block, or -1.
+
+    A JSON example inside a reasoning block must not become the payload, so the
+    scan steps over whole think blocks. An unclosed block before the payload is
+    a truncated response: there is no structured answer to find.
+    """
+    index = 0
+    while index < len(text):
+        char = text[index]
+        if char == "<":
+            opening = _THINK_OPEN_PREFIX_RE.match(text, index)
+            if opening is not None:
+                end = _find_tag_end(text, opening.end())
+                if end >= 0:
+                    if _is_self_closing_think_tag(text, index, end + 1):
+                        index = end + 1
+                        continue
+                    closing = _find_think_close(text, end + 1)
+                    if closing is not None:
+                        index = closing[1]
+                        continue
+                    return -1
+        if char in "{[":
+            return index
+        index += 1
+    return -1
+
+
+def strip_think_blocks_outside_json_strings(text: str) -> str:
+    """Strip reasoning around a JSON response but keep tags inside string values.
+
+    Structured parsers used to run :func:`strip_think_blocks` over the whole
+    response before ``json.loads``. That deleted a literal ``<think>`` inside a
+    JSON string value; an unclosed one cut the document mid-string and failed
+    parsing. Tags inside string values are data and are preserved verbatim.
+
+    Everything outside the payload keeps :func:`strip_think_blocks` semantics:
+    the prose before it is stripped (so a JSON example inside a reasoning block
+    is never mistaken for the answer), a block between the payload's own tokens
+    is removed, and an unclosed block truncates the remainder as a truncated
+    model response. After the top-level payload closes, trailing prose goes
+    through the same strip/truncate rules.
+    """
+    start = _json_payload_start(text)
+    if start < 0:
+        return strip_think_blocks(text)
+
+    out = [strip_think_blocks(text[:start])]
+    quote = ""
+    depth = 0
+    index = start
+    while index < len(text):
+        char = text[index]
+        if quote:
+            out.append(char)
+            if char == "\\" and index + 1 < len(text):
+                out.append(text[index + 1])
+                index += 2
+                continue
+            if char == quote:
+                quote = ""
+            index += 1
+            continue
+        if char == '"':
+            quote = char
+            out.append(char)
+            index += 1
+            continue
+        if char in "{[":
+            depth += 1
+            out.append(char)
+            index += 1
+            continue
+        if char in "}]":
+            depth -= 1
+            out.append(char)
+            index += 1
+            if depth <= 0:
+                # The top-level payload is closed: trailing prose is not JSON,
+                # so it keeps the old strip (and truncate) behavior.
+                out.append(strip_think_blocks(text[index:]))
+                break
+            continue
+        if char == "<":
+            opening = _THINK_OPEN_PREFIX_RE.match(text, index)
+            if opening is not None:
+                end = _find_tag_end(text, opening.end())
+                if end >= 0:
+                    if _is_self_closing_think_tag(text, index, end + 1):
+                        index = end + 1
+                        continue
+                    closing = _find_think_close(text, end + 1)
+                    if closing is not None:
+                        index = closing[1]
+                        continue
+                    # Unclosed outside a string: truncated mid-thought, so the
+                    # remainder cannot hold a payload either.
+                    return "".join(out).strip()
+        out.append(char)
+        index += 1
+    return "".join(out).strip()

backend/tests/test_json_think_tag_stripping.py

@@ -0,0 +1,79 @@
+"""Literal think tags inside JSON values must survive structured parsing."""
+
+from app.gateway.routers.suggestions import _parse_json_string_list
+from deerflow.runtime.goal import parse_goal_evaluation_response
+from deerflow.utils.llm_text import strip_think_blocks_outside_json_strings
+
+
+def test_suggestions_keep_a_complete_literal_tag_inside_a_string():
+    assert _parse_json_string_list('["Explain <think>example</think>"]') == ["Explain <think>example</think>"]
+
+
+def test_suggestions_keep_an_unclosed_literal_tag_inside_a_string():
+    assert _parse_json_string_list('["Explain <think>"]') == ["Explain <think>"]
+
+
+def test_suggestions_strip_reasoning_before_the_list():
+    raw = '<think>maybe ["wrong"]</think>\n```json\n["right"]\n```'
+
+    assert _parse_json_string_list(raw) == ["right"]
+
+
+def test_suggestions_do_not_take_a_json_example_inside_reasoning():
+    raw = '<think>["wrong"]</think>["right"]'
+
+    assert _parse_json_string_list(raw) == ["right"]
+
+
+def test_goal_keeps_a_complete_literal_tag_in_the_reason():
+    evaluation = parse_goal_evaluation_response('{"satisfied": true, "reason": "Found <think>example</think>"}')
+
+    assert evaluation["reason"] == "Found <think>example</think>"
+
+
+def test_goal_keeps_an_unclosed_literal_tag_in_the_reason():
+    evaluation = parse_goal_evaluation_response('{"satisfied": false, "reason": "Found <think>"}')
+
+    assert evaluation["reason"] == "Found <think>"
+
+
+def test_goal_strips_reasoning_before_the_object():
+    evaluation = parse_goal_evaluation_response('<think>maybe {"satisfied": false}</think>{"satisfied": true, "reason": "ok"}')
+
+    assert evaluation["satisfied"] is True
+    assert evaluation["reason"] == "ok"
+
+
+def test_helper_keeps_tags_inside_strings_and_strips_them_outside():
+    payload = '{"a": "<think>x</think>", "b": "<think>"}'
+
+    assert strip_think_blocks_outside_json_strings(payload) == payload
+    assert strip_think_blocks_outside_json_strings('<think>reasoning</think>{"a": 1}') == '{"a": 1}'
+
+
+def test_goal_strips_a_reasoning_block_that_follows_prose():
+    evaluation = parse_goal_evaluation_response('Hmm. <think>{"satisfied": false}</think>{"satisfied": true, "reason": "ok"}')
+
+    assert evaluation["satisfied"] is True
+    assert evaluation["reason"] == "ok"
+
+
+def test_suggestions_strip_a_reasoning_block_that_follows_prose():
+    assert _parse_json_string_list('Okay. <think>["wrong"]</think>["right"]') == ["right"]
+
+
+def test_suggestions_drop_trailing_prose_after_an_unclosed_block():
+    # The old whole-text stripper truncated here; keep that so a trailing ']'
+    # cannot become the parsers' rfind target.
+    assert _parse_json_string_list('["Q1"] <think>tail mentioning ]') == ["Q1"]
+
+
+def test_goal_drops_trailing_prose_after_an_unclosed_block():
+    evaluation = parse_goal_evaluation_response('{"satisfied": true, "reason": "ok"} <think>tail mentioning }')
+
+    assert evaluation["satisfied"] is True
+    assert evaluation["reason"] == "ok"
+
+
+def test_trailing_stray_quote_cannot_mask_a_later_reasoning_block():
+    assert _parse_json_string_list('["Q1"] stray " quote <think>tail ]') == ["Q1"]

backend/tests/test_suggestions_router.py

@@ -46,23 +46,23 @@ def test_parse_json_string_list_rejects_non_list():
 
 def test_strip_think_blocks_removes_complete_block():
     text = "<think>\nreasoning here\n</think>\nanswer"
-    assert suggestions._strip_think_blocks(text) == "answer"
+    assert suggestions._strip_think_blocks_outside_json_strings(text) == "answer"
 
 
 def test_strip_think_blocks_is_case_insensitive():
     text = "<Think>reasoning</THINK>\nanswer"
-    assert suggestions._strip_think_blocks(text) == "answer"
+    assert suggestions._strip_think_blocks_outside_json_strings(text) == "answer"
 
 
 def test_strip_think_blocks_drops_unclosed_block():
     # Reasoning models truncated at max_tokens emit an unclosed <think>.
     text = "<think>\nreasoning that never finished because tokens ran out"
-    assert suggestions._strip_think_blocks(text) == ""
+    assert suggestions._strip_think_blocks_outside_json_strings(text) == ""
 
 
 def test_strip_think_blocks_keeps_text_without_think():
     text = '["a", "b"]'
-    assert suggestions._strip_think_blocks(text) == '["a", "b"]'
+    assert suggestions._strip_think_blocks_outside_json_strings(text) == '["a", "b"]'
 
 
 def test_parse_json_string_list_ignores_brackets_inside_think_block():