diff --git a/CHANGELOG.md b/CHANGELOG.md index 3548c1a6f9..d5d343774a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,14 @@ This section accumulates work toward the **2.3.0** milestone longer create or acquire a sandbox before the first sandbox-backed tool call. Existing sandbox bindings and providers that require skill uploads retain their eager synchronization behavior. ([#7782]) - + +- **parsing:** Follow-up suggestions and the `/goal` evaluator keep a literal + `` tag that appears inside a JSON string value, such as an answer + quoting the tag, while reasoning around the JSON is still removed. An + unclosed literal tag inside a string no longer truncates the document and + fails parsing, and reasoning outside the payload still strips and truncates + as before. ([#7907]) + - **gateway:** A caller-supplied thread ID that is a Windows reserved device name (`NUL`, `CON`, `COM1`, ...) is now rejected at the shared `validate_thread_id()` boundary. Such IDs previously passed validation but @@ -16029,7 +16036,8 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7762]: https://github.com/bytedance/deer-flow/pull/7762 [#7772]: https://github.com/bytedance/deer-flow/pull/7772 [#7781]: https://github.com/bytedance/deer-flow/pull/7781 +[#7782]: https://github.com/bytedance/deer-flow/pull/7782 [#7792]: https://github.com/bytedance/deer-flow/pull/7792 [#7831]: https://github.com/bytedance/deer-flow/pull/7831 -[#7782]: https://github.com/bytedance/deer-flow/pull/7782 [#7892]: https://github.com/bytedance/deer-flow/pull/7892 +[#7907]: https://github.com/bytedance/deer-flow/pull/7907 diff --git a/backend/app/gateway/routers/suggestions.py b/backend/app/gateway/routers/suggestions.py index 9a3fd582cc..6607babc5d 100644 --- a/backend/app/gateway/routers/suggestions.py +++ b/backend/app/gateway/routers/suggestions.py @@ -38,11 +38,11 @@ class SuggestionsConfigResponse(BaseModel): _strip_markdown_code_fence = llm_text.strip_markdown_code_fence -_strip_think_blocks = llm_text.strip_think_blocks +_strip_think_blocks_outside_json_strings = llm_text.strip_think_blocks_outside_json_strings def _parse_json_string_list(text: str) -> list[str] | None: - candidate = _strip_think_blocks(text) + candidate = _strip_think_blocks_outside_json_strings(text) candidate = _strip_markdown_code_fence(candidate) start = candidate.find("[") end = candidate.rfind("]") diff --git a/backend/packages/harness/deerflow/runtime/goal.py b/backend/packages/harness/deerflow/runtime/goal.py index e20e28163b..17ad7c27a3 100644 --- a/backend/packages/harness/deerflow/runtime/goal.py +++ b/backend/packages/harness/deerflow/runtime/goal.py @@ -68,7 +68,7 @@ GOAL_OUTCOME_CHANNEL = "goal_outcome" _extract_response_text = llm_text.extract_response_text _strip_markdown_code_fence = llm_text.strip_markdown_code_fence -_strip_think_blocks = llm_text.strip_think_blocks +_strip_think_blocks_outside_json_strings = llm_text.strip_think_blocks_outside_json_strings _goal_locks = AsyncKeyedLockTable[str]() @@ -157,7 +157,7 @@ def build_goal_outcome(goal: GoalState, evaluation: GoalEvaluation, *, reply_mes def parse_goal_evaluation_response(text: str, *, require_assumption_attribution: bool = False) -> GoalEvaluation: """Parse the evaluator's JSON object response.""" - candidate = _strip_markdown_code_fence(_strip_think_blocks(text)) + candidate = _strip_markdown_code_fence(_strip_think_blocks_outside_json_strings(text)) start = candidate.find("{") end = candidate.rfind("}") if start == -1 or end == -1 or end <= start: diff --git a/backend/packages/harness/deerflow/utils/llm_text.py b/backend/packages/harness/deerflow/utils/llm_text.py index c267ffe6cd..cafc0c74a9 100644 --- a/backend/packages/harness/deerflow/utils/llm_text.py +++ b/backend/packages/harness/deerflow/utils/llm_text.py @@ -153,3 +153,107 @@ def extract_response_text(content: object) -> str: if content is None: return "" return str(content) + + +def _json_payload_start(text: str) -> int: + """Index of the first ``{``/``[`` outside a reasoning block, or -1. + + A JSON example inside a reasoning block must not become the payload, so the + scan steps over whole think blocks. An unclosed block before the payload is + a truncated response: there is no structured answer to find. + """ + index = 0 + while index < len(text): + char = text[index] + if char == "<": + opening = _THINK_OPEN_PREFIX_RE.match(text, index) + if opening is not None: + end = _find_tag_end(text, opening.end()) + if end >= 0: + if _is_self_closing_think_tag(text, index, end + 1): + index = end + 1 + continue + closing = _find_think_close(text, end + 1) + if closing is not None: + index = closing[1] + continue + return -1 + if char in "{[": + return index + index += 1 + return -1 + + +def strip_think_blocks_outside_json_strings(text: str) -> str: + """Strip reasoning around a JSON response but keep tags inside string values. + + Structured parsers used to run :func:`strip_think_blocks` over the whole + response before ``json.loads``. That deleted a literal ```` inside a + JSON string value; an unclosed one cut the document mid-string and failed + parsing. Tags inside string values are data and are preserved verbatim. + + Everything outside the payload keeps :func:`strip_think_blocks` semantics: + the prose before it is stripped (so a JSON example inside a reasoning block + is never mistaken for the answer), a block between the payload's own tokens + is removed, and an unclosed block truncates the remainder as a truncated + model response. After the top-level payload closes, trailing prose goes + through the same strip/truncate rules. + """ + start = _json_payload_start(text) + if start < 0: + return strip_think_blocks(text) + + out = [strip_think_blocks(text[:start])] + quote = "" + depth = 0 + index = start + while index < len(text): + char = text[index] + if quote: + out.append(char) + if char == "\\" and index + 1 < len(text): + out.append(text[index + 1]) + index += 2 + continue + if char == quote: + quote = "" + index += 1 + continue + if char == '"': + quote = char + out.append(char) + index += 1 + continue + if char in "{[": + depth += 1 + out.append(char) + index += 1 + continue + if char in "}]": + depth -= 1 + out.append(char) + index += 1 + if depth <= 0: + # The top-level payload is closed: trailing prose is not JSON, + # so it keeps the old strip (and truncate) behavior. + out.append(strip_think_blocks(text[index:])) + break + continue + if char == "<": + opening = _THINK_OPEN_PREFIX_RE.match(text, index) + if opening is not None: + end = _find_tag_end(text, opening.end()) + if end >= 0: + if _is_self_closing_think_tag(text, index, end + 1): + index = end + 1 + continue + closing = _find_think_close(text, end + 1) + if closing is not None: + index = closing[1] + continue + # Unclosed outside a string: truncated mid-thought, so the + # remainder cannot hold a payload either. + return "".join(out).strip() + out.append(char) + index += 1 + return "".join(out).strip() diff --git a/backend/tests/test_json_think_tag_stripping.py b/backend/tests/test_json_think_tag_stripping.py new file mode 100644 index 0000000000..ed90841050 --- /dev/null +++ b/backend/tests/test_json_think_tag_stripping.py @@ -0,0 +1,79 @@ +"""Literal think tags inside JSON values must survive structured parsing.""" + +from app.gateway.routers.suggestions import _parse_json_string_list +from deerflow.runtime.goal import parse_goal_evaluation_response +from deerflow.utils.llm_text import strip_think_blocks_outside_json_strings + + +def test_suggestions_keep_a_complete_literal_tag_inside_a_string(): + assert _parse_json_string_list('["Explain example"]') == ["Explain example"] + + +def test_suggestions_keep_an_unclosed_literal_tag_inside_a_string(): + assert _parse_json_string_list('["Explain "]') == ["Explain "] + + +def test_suggestions_strip_reasoning_before_the_list(): + raw = 'maybe ["wrong"]\n```json\n["right"]\n```' + + assert _parse_json_string_list(raw) == ["right"] + + +def test_suggestions_do_not_take_a_json_example_inside_reasoning(): + raw = '["wrong"]["right"]' + + assert _parse_json_string_list(raw) == ["right"] + + +def test_goal_keeps_a_complete_literal_tag_in_the_reason(): + evaluation = parse_goal_evaluation_response('{"satisfied": true, "reason": "Found example"}') + + assert evaluation["reason"] == "Found example" + + +def test_goal_keeps_an_unclosed_literal_tag_in_the_reason(): + evaluation = parse_goal_evaluation_response('{"satisfied": false, "reason": "Found "}') + + assert evaluation["reason"] == "Found " + + +def test_goal_strips_reasoning_before_the_object(): + evaluation = parse_goal_evaluation_response('maybe {"satisfied": false}{"satisfied": true, "reason": "ok"}') + + assert evaluation["satisfied"] is True + assert evaluation["reason"] == "ok" + + +def test_helper_keeps_tags_inside_strings_and_strips_them_outside(): + payload = '{"a": "x", "b": ""}' + + assert strip_think_blocks_outside_json_strings(payload) == payload + assert strip_think_blocks_outside_json_strings('reasoning{"a": 1}') == '{"a": 1}' + + +def test_goal_strips_a_reasoning_block_that_follows_prose(): + evaluation = parse_goal_evaluation_response('Hmm. {"satisfied": false}{"satisfied": true, "reason": "ok"}') + + assert evaluation["satisfied"] is True + assert evaluation["reason"] == "ok" + + +def test_suggestions_strip_a_reasoning_block_that_follows_prose(): + assert _parse_json_string_list('Okay. ["wrong"]["right"]') == ["right"] + + +def test_suggestions_drop_trailing_prose_after_an_unclosed_block(): + # The old whole-text stripper truncated here; keep that so a trailing ']' + # cannot become the parsers' rfind target. + assert _parse_json_string_list('["Q1"] tail mentioning ]') == ["Q1"] + + +def test_goal_drops_trailing_prose_after_an_unclosed_block(): + evaluation = parse_goal_evaluation_response('{"satisfied": true, "reason": "ok"} tail mentioning }') + + assert evaluation["satisfied"] is True + assert evaluation["reason"] == "ok" + + +def test_trailing_stray_quote_cannot_mask_a_later_reasoning_block(): + assert _parse_json_string_list('["Q1"] stray " quote tail ]') == ["Q1"] diff --git a/backend/tests/test_suggestions_router.py b/backend/tests/test_suggestions_router.py index ef2df09875..6da74629bb 100644 --- a/backend/tests/test_suggestions_router.py +++ b/backend/tests/test_suggestions_router.py @@ -46,23 +46,23 @@ def test_parse_json_string_list_rejects_non_list(): def test_strip_think_blocks_removes_complete_block(): text = "\nreasoning here\n\nanswer" - assert suggestions._strip_think_blocks(text) == "answer" + assert suggestions._strip_think_blocks_outside_json_strings(text) == "answer" def test_strip_think_blocks_is_case_insensitive(): text = "reasoning\nanswer" - assert suggestions._strip_think_blocks(text) == "answer" + assert suggestions._strip_think_blocks_outside_json_strings(text) == "answer" def test_strip_think_blocks_drops_unclosed_block(): # Reasoning models truncated at max_tokens emit an unclosed . text = "\nreasoning that never finished because tokens ran out" - assert suggestions._strip_think_blocks(text) == "" + assert suggestions._strip_think_blocks_outside_json_strings(text) == "" def test_strip_think_blocks_keeps_text_without_think(): text = '["a", "b"]' - assert suggestions._strip_think_blocks(text) == '["a", "b"]' + assert suggestions._strip_think_blocks_outside_json_strings(text) == '["a", "b"]' def test_parse_json_string_list_ignores_brackets_inside_think_block():