From a41d0fcec1219390bae31283d38ef0e1155910a8 Mon Sep 17 00:00:00 2001 From: yetuge <182123929+yetuge@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:19:52 +0800 Subject: [PATCH] fix(parsing): keep literal think tags inside JSON string values (#7907) * fix(parsing): keep literal think tags inside JSON string values Structured parsers ran strip_think_blocks over the whole response before json.loads. A model answer that quotes the tag -- "Explain example " -- lost the quoted text, and an unclosed literal tag inside a string cut the JSON document mid-string, so suggestions returned None and goal evaluation failed to parse. Strip reasoning only outside JSON string literals: leading blocks are still removed whole (a JSON example inside reasoning is never taken as the answer), and blocks after the payload starts are removed only outside quotes. Suggestions and the goal evaluator both use the shared helper. * fix(parsing): keep think-tag stripping complete outside the JSON payload The string-aware walker only covered the payload itself: a reasoning block after prose and before the payload was copied verbatim, so brackets inside it became the parsers' find/rfind targets, and an unclosed block in trailing prose was kept where strip_think_blocks used to truncate. Find the payload start while stepping over think blocks, strip the head with strip_think_blocks, keep strip/truncate semantics after the top-level payload closes, and truncate at an unclosed block outside a string. Adds a CHANGELOG entry for the user-visible parsing change. --- CHANGELOG.md | 12 +- backend/app/gateway/routers/suggestions.py | 4 +- .../packages/harness/deerflow/runtime/goal.py | 4 +- .../harness/deerflow/utils/llm_text.py | 104 ++++++++++++++++++ .../tests/test_json_think_tag_stripping.py | 79 +++++++++++++ backend/tests/test_suggestions_router.py | 8 +- 6 files changed, 201 insertions(+), 10 deletions(-) create mode 100644 backend/tests/test_json_think_tag_stripping.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3548c1a6f9..d5d343774a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,14 @@ This section accumulates work toward the **2.3.0** milestone longer create or acquire a sandbox before the first sandbox-backed tool call. Existing sandbox bindings and providers that require skill uploads retain their eager synchronization behavior. ([#7782]) - + +- **parsing:** Follow-up suggestions and the `/goal` evaluator keep a literal + `` tag that appears inside a JSON string value, such as an answer + quoting the tag, while reasoning around the JSON is still removed. An + unclosed literal tag inside a string no longer truncates the document and + fails parsing, and reasoning outside the payload still strips and truncates + as before. ([#7907]) + - **gateway:** A caller-supplied thread ID that is a Windows reserved device name (`NUL`, `CON`, `COM1`, ...) is now rejected at the shared `validate_thread_id()` boundary. Such IDs previously passed validation but @@ -16029,7 +16036,8 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7762]: https://github.com/bytedance/deer-flow/pull/7762 [#7772]: https://github.com/bytedance/deer-flow/pull/7772 [#7781]: https://github.com/bytedance/deer-flow/pull/7781 +[#7782]: https://github.com/bytedance/deer-flow/pull/7782 [#7792]: https://github.com/bytedance/deer-flow/pull/7792 [#7831]: https://github.com/bytedance/deer-flow/pull/7831 -[#7782]: https://github.com/bytedance/deer-flow/pull/7782 [#7892]: https://github.com/bytedance/deer-flow/pull/7892 +[#7907]: https://github.com/bytedance/deer-flow/pull/7907 diff --git a/backend/app/gateway/routers/suggestions.py b/backend/app/gateway/routers/suggestions.py index 9a3fd582cc..6607babc5d 100644 --- a/backend/app/gateway/routers/suggestions.py +++ b/backend/app/gateway/routers/suggestions.py @@ -38,11 +38,11 @@ class SuggestionsConfigResponse(BaseModel): _strip_markdown_code_fence = llm_text.strip_markdown_code_fence -_strip_think_blocks = llm_text.strip_think_blocks +_strip_think_blocks_outside_json_strings = llm_text.strip_think_blocks_outside_json_strings def _parse_json_string_list(text: str) -> list[str] | None: - candidate = _strip_think_blocks(text) + candidate = _strip_think_blocks_outside_json_strings(text) candidate = _strip_markdown_code_fence(candidate) start = candidate.find("[") end = candidate.rfind("]") diff --git a/backend/packages/harness/deerflow/runtime/goal.py b/backend/packages/harness/deerflow/runtime/goal.py index e20e28163b..17ad7c27a3 100644 --- a/backend/packages/harness/deerflow/runtime/goal.py +++ b/backend/packages/harness/deerflow/runtime/goal.py @@ -68,7 +68,7 @@ GOAL_OUTCOME_CHANNEL = "goal_outcome" _extract_response_text = llm_text.extract_response_text _strip_markdown_code_fence = llm_text.strip_markdown_code_fence -_strip_think_blocks = llm_text.strip_think_blocks +_strip_think_blocks_outside_json_strings = llm_text.strip_think_blocks_outside_json_strings _goal_locks = AsyncKeyedLockTable[str]() @@ -157,7 +157,7 @@ def build_goal_outcome(goal: GoalState, evaluation: GoalEvaluation, *, reply_mes def parse_goal_evaluation_response(text: str, *, require_assumption_attribution: bool = False) -> GoalEvaluation: """Parse the evaluator's JSON object response.""" - candidate = _strip_markdown_code_fence(_strip_think_blocks(text)) + candidate = _strip_markdown_code_fence(_strip_think_blocks_outside_json_strings(text)) start = candidate.find("{") end = candidate.rfind("}") if start == -1 or end == -1 or end <= start: diff --git a/backend/packages/harness/deerflow/utils/llm_text.py b/backend/packages/harness/deerflow/utils/llm_text.py index c267ffe6cd..cafc0c74a9 100644 --- a/backend/packages/harness/deerflow/utils/llm_text.py +++ b/backend/packages/harness/deerflow/utils/llm_text.py @@ -153,3 +153,107 @@ def extract_response_text(content: object) -> str: if content is None: return "" return str(content) + + +def _json_payload_start(text: str) -> int: + """Index of the first ``{``/``[`` outside a reasoning block, or -1. + + A JSON example inside a reasoning block must not become the payload, so the + scan steps over whole think blocks. An unclosed block before the payload is + a truncated response: there is no structured answer to find. + """ + index = 0 + while index < len(text): + char = text[index] + if char == "<": + opening = _THINK_OPEN_PREFIX_RE.match(text, index) + if opening is not None: + end = _find_tag_end(text, opening.end()) + if end >= 0: + if _is_self_closing_think_tag(text, index, end + 1): + index = end + 1 + continue + closing = _find_think_close(text, end + 1) + if closing is not None: + index = closing[1] + continue + return -1 + if char in "{[": + return index + index += 1 + return -1 + + +def strip_think_blocks_outside_json_strings(text: str) -> str: + """Strip reasoning around a JSON response but keep tags inside string values. + + Structured parsers used to run :func:`strip_think_blocks` over the whole + response before ``json.loads``. That deleted a literal ```` inside a + JSON string value; an unclosed one cut the document mid-string and failed + parsing. Tags inside string values are data and are preserved verbatim. + + Everything outside the payload keeps :func:`strip_think_blocks` semantics: + the prose before it is stripped (so a JSON example inside a reasoning block + is never mistaken for the answer), a block between the payload's own tokens + is removed, and an unclosed block truncates the remainder as a truncated + model response. After the top-level payload closes, trailing prose goes + through the same strip/truncate rules. + """ + start = _json_payload_start(text) + if start < 0: + return strip_think_blocks(text) + + out = [strip_think_blocks(text[:start])] + quote = "" + depth = 0 + index = start + while index < len(text): + char = text[index] + if quote: + out.append(char) + if char == "\\" and index + 1 < len(text): + out.append(text[index + 1]) + index += 2 + continue + if char == quote: + quote = "" + index += 1 + continue + if char == '"': + quote = char + out.append(char) + index += 1 + continue + if char in "{[": + depth += 1 + out.append(char) + index += 1 + continue + if char in "}]": + depth -= 1 + out.append(char) + index += 1 + if depth <= 0: + # The top-level payload is closed: trailing prose is not JSON, + # so it keeps the old strip (and truncate) behavior. + out.append(strip_think_blocks(text[index:])) + break + continue + if char == "<": + opening = _THINK_OPEN_PREFIX_RE.match(text, index) + if opening is not None: + end = _find_tag_end(text, opening.end()) + if end >= 0: + if _is_self_closing_think_tag(text, index, end + 1): + index = end + 1 + continue + closing = _find_think_close(text, end + 1) + if closing is not None: + index = closing[1] + continue + # Unclosed outside a string: truncated mid-thought, so the + # remainder cannot hold a payload either. + return "".join(out).strip() + out.append(char) + index += 1 + return "".join(out).strip() diff --git a/backend/tests/test_json_think_tag_stripping.py b/backend/tests/test_json_think_tag_stripping.py new file mode 100644 index 0000000000..ed90841050 --- /dev/null +++ b/backend/tests/test_json_think_tag_stripping.py @@ -0,0 +1,79 @@ +"""Literal think tags inside JSON values must survive structured parsing.""" + +from app.gateway.routers.suggestions import _parse_json_string_list +from deerflow.runtime.goal import parse_goal_evaluation_response +from deerflow.utils.llm_text import strip_think_blocks_outside_json_strings + + +def test_suggestions_keep_a_complete_literal_tag_inside_a_string(): + assert _parse_json_string_list('["Explain example"]') == ["Explain example"] + + +def test_suggestions_keep_an_unclosed_literal_tag_inside_a_string(): + assert _parse_json_string_list('["Explain "]') == ["Explain "] + + +def test_suggestions_strip_reasoning_before_the_list(): + raw = 'maybe ["wrong"]\n```json\n["right"]\n```' + + assert _parse_json_string_list(raw) == ["right"] + + +def test_suggestions_do_not_take_a_json_example_inside_reasoning(): + raw = '["wrong"]["right"]' + + assert _parse_json_string_list(raw) == ["right"] + + +def test_goal_keeps_a_complete_literal_tag_in_the_reason(): + evaluation = parse_goal_evaluation_response('{"satisfied": true, "reason": "Found example"}') + + assert evaluation["reason"] == "Found example" + + +def test_goal_keeps_an_unclosed_literal_tag_in_the_reason(): + evaluation = parse_goal_evaluation_response('{"satisfied": false, "reason": "Found "}') + + assert evaluation["reason"] == "Found " + + +def test_goal_strips_reasoning_before_the_object(): + evaluation = parse_goal_evaluation_response('maybe {"satisfied": false}{"satisfied": true, "reason": "ok"}') + + assert evaluation["satisfied"] is True + assert evaluation["reason"] == "ok" + + +def test_helper_keeps_tags_inside_strings_and_strips_them_outside(): + payload = '{"a": "x", "b": ""}' + + assert strip_think_blocks_outside_json_strings(payload) == payload + assert strip_think_blocks_outside_json_strings('reasoning{"a": 1}') == '{"a": 1}' + + +def test_goal_strips_a_reasoning_block_that_follows_prose(): + evaluation = parse_goal_evaluation_response('Hmm. {"satisfied": false}{"satisfied": true, "reason": "ok"}') + + assert evaluation["satisfied"] is True + assert evaluation["reason"] == "ok" + + +def test_suggestions_strip_a_reasoning_block_that_follows_prose(): + assert _parse_json_string_list('Okay. ["wrong"]["right"]') == ["right"] + + +def test_suggestions_drop_trailing_prose_after_an_unclosed_block(): + # The old whole-text stripper truncated here; keep that so a trailing ']' + # cannot become the parsers' rfind target. + assert _parse_json_string_list('["Q1"] tail mentioning ]') == ["Q1"] + + +def test_goal_drops_trailing_prose_after_an_unclosed_block(): + evaluation = parse_goal_evaluation_response('{"satisfied": true, "reason": "ok"} tail mentioning }') + + assert evaluation["satisfied"] is True + assert evaluation["reason"] == "ok" + + +def test_trailing_stray_quote_cannot_mask_a_later_reasoning_block(): + assert _parse_json_string_list('["Q1"] stray " quote tail ]') == ["Q1"] diff --git a/backend/tests/test_suggestions_router.py b/backend/tests/test_suggestions_router.py index ef2df09875..6da74629bb 100644 --- a/backend/tests/test_suggestions_router.py +++ b/backend/tests/test_suggestions_router.py @@ -46,23 +46,23 @@ def test_parse_json_string_list_rejects_non_list(): def test_strip_think_blocks_removes_complete_block(): text = "\nreasoning here\n\nanswer" - assert suggestions._strip_think_blocks(text) == "answer" + assert suggestions._strip_think_blocks_outside_json_strings(text) == "answer" def test_strip_think_blocks_is_case_insensitive(): text = "reasoning\nanswer" - assert suggestions._strip_think_blocks(text) == "answer" + assert suggestions._strip_think_blocks_outside_json_strings(text) == "answer" def test_strip_think_blocks_drops_unclosed_block(): # Reasoning models truncated at max_tokens emit an unclosed . text = "\nreasoning that never finished because tokens ran out" - assert suggestions._strip_think_blocks(text) == "" + assert suggestions._strip_think_blocks_outside_json_strings(text) == "" def test_strip_think_blocks_keeps_text_without_think(): text = '["a", "b"]' - assert suggestions._strip_think_blocks(text) == '["a", "b"]' + assert suggestions._strip_think_blocks_outside_json_strings(text) == '["a", "b"]' def test_parse_json_string_list_ignores_brackets_inside_think_block():