mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-08 03:30:34 +00:00
a41d0fcec1
* fix(parsing): keep literal think tags inside JSON string values Structured parsers ran strip_think_blocks over the whole response before json.loads. A model answer that quotes the tag -- "Explain <think>example </think>" -- lost the quoted text, and an unclosed literal tag inside a string cut the JSON document mid-string, so suggestions returned None and goal evaluation failed to parse. Strip reasoning only outside JSON string literals: leading blocks are still removed whole (a JSON example inside reasoning is never taken as the answer), and blocks after the payload starts are removed only outside quotes. Suggestions and the goal evaluator both use the shared helper. * fix(parsing): keep think-tag stripping complete outside the JSON payload The string-aware walker only covered the payload itself: a reasoning block after prose and before the payload was copied verbatim, so brackets inside it became the parsers' find/rfind targets, and an unclosed block in trailing prose was kept where strip_think_blocks used to truncate. Find the payload start while stepping over think blocks, strip the head with strip_think_blocks, keep strip/truncate semantics after the top-level payload closes, and truncate at an unclosed block outside a string. Adds a CHANGELOG entry for the user-visible parsing change.
287 lines
12 KiB
Python
287 lines
12 KiB
Python
import asyncio
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
from app.gateway.routers import suggestions
|
|
from deerflow.trace_context import request_trace_context
|
|
from deerflow.utils import oneshot_llm
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clear_langfuse_env(monkeypatch):
|
|
from deerflow.config.tracing_config import reset_tracing_config
|
|
|
|
for name in ("LANGFUSE_TRACING", "LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL", "DEER_FLOW_ENV", "ENVIRONMENT"):
|
|
monkeypatch.delenv(name, raising=False)
|
|
reset_tracing_config()
|
|
# These tests bypass route auth to exercise parsing and tracing. Model
|
|
# admission is covered through HTTP in test_suggestions_model_authorization.
|
|
monkeypatch.setattr(suggestions, "get_current_user_from_request", AsyncMock(return_value=SimpleNamespace()))
|
|
monkeypatch.setattr(suggestions, "authorize_model_use", lambda *_args, **_kwargs: None)
|
|
yield
|
|
reset_tracing_config()
|
|
|
|
|
|
def test_strip_markdown_code_fence_removes_wrapping():
|
|
text = '```json\n["a"]\n```'
|
|
assert suggestions._strip_markdown_code_fence(text) == '["a"]'
|
|
|
|
|
|
def test_strip_markdown_code_fence_no_fence_keeps_content():
|
|
text = ' ["a"] '
|
|
assert suggestions._strip_markdown_code_fence(text) == '["a"]'
|
|
|
|
|
|
def test_parse_json_string_list_filters_invalid_items():
|
|
text = '```json\n["a", " ", 1, "b"]\n```'
|
|
assert suggestions._parse_json_string_list(text) == ["a", "b"]
|
|
|
|
|
|
def test_parse_json_string_list_rejects_non_list():
|
|
text = '{"a": 1}'
|
|
assert suggestions._parse_json_string_list(text) is None
|
|
|
|
|
|
def test_strip_think_blocks_removes_complete_block():
|
|
text = "<think>\nreasoning here\n</think>\nanswer"
|
|
assert suggestions._strip_think_blocks_outside_json_strings(text) == "answer"
|
|
|
|
|
|
def test_strip_think_blocks_is_case_insensitive():
|
|
text = "<Think>reasoning</THINK>\nanswer"
|
|
assert suggestions._strip_think_blocks_outside_json_strings(text) == "answer"
|
|
|
|
|
|
def test_strip_think_blocks_drops_unclosed_block():
|
|
# Reasoning models truncated at max_tokens emit an unclosed <think>.
|
|
text = "<think>\nreasoning that never finished because tokens ran out"
|
|
assert suggestions._strip_think_blocks_outside_json_strings(text) == ""
|
|
|
|
|
|
def test_strip_think_blocks_keeps_text_without_think():
|
|
text = '["a", "b"]'
|
|
assert suggestions._strip_think_blocks_outside_json_strings(text) == '["a", "b"]'
|
|
|
|
|
|
def test_parse_json_string_list_ignores_brackets_inside_think_block():
|
|
# MiniMax-M3 inlines its chain-of-thought as <think>...</think> in content
|
|
# (reasoning_split=false). When that reasoning contains '[' / ']', the old
|
|
# find('[')/rfind(']') logic grabbed the wrong span and parsing failed.
|
|
text = '<think>\nMaybe a list like ["x", "y"] could work. Let me craft 3.\n</think>\n["Q1", "Q2", "Q3"]'
|
|
assert suggestions._parse_json_string_list(text) == ["Q1", "Q2", "Q3"]
|
|
|
|
|
|
def test_parse_json_string_list_strips_think_then_code_fence():
|
|
text = '<think>reasoning</think>\n```json\n["Q1", "Q2"]\n```'
|
|
assert suggestions._parse_json_string_list(text) == ["Q1", "Q2"]
|
|
|
|
|
|
def test_generate_suggestions_strips_inline_think_block(monkeypatch):
|
|
# End-to-end: model returns thinking inline followed by the JSON array.
|
|
req = suggestions.SuggestionsRequest(
|
|
messages=[
|
|
suggestions.SuggestionMessage(role="user", content="介绍深度学习"),
|
|
suggestions.SuggestionMessage(role="assistant", content="深度学习是机器学习的分支。"),
|
|
],
|
|
n=3,
|
|
model_name=None,
|
|
)
|
|
content = '<think>\nThe user asked about deep learning. Options: maybe [1] frameworks, [2] math basics.\n</think>\n["深度学习和机器学习的区别？", "常用框架有哪些？", "需要什么数学基础？"]'
|
|
fake_model = MagicMock()
|
|
fake_model.ainvoke = AsyncMock(return_value=MagicMock(content=content))
|
|
monkeypatch.setattr(oneshot_llm, "create_chat_model", lambda **kwargs: fake_model)
|
|
|
|
result = asyncio.run(suggestions.generate_suggestions.__wrapped__("t1", req, request=None, config=SimpleNamespace(suggestions=SimpleNamespace(enabled=True))))
|
|
|
|
assert result.suggestions == ["深度学习和机器学习的区别？", "常用框架有哪些？", "需要什么数学基础？"]
|
|
|
|
|
|
def test_format_conversation_formats_roles():
|
|
messages = [
|
|
suggestions.SuggestionMessage(role="User", content="Hi"),
|
|
suggestions.SuggestionMessage(role="assistant", content="Hello"),
|
|
suggestions.SuggestionMessage(role="system", content="note"),
|
|
]
|
|
assert suggestions._format_conversation(messages) == "User: Hi\nAssistant: Hello\nsystem: note"
|
|
|
|
|
|
def test_generate_suggestions_parses_and_limits(monkeypatch):
|
|
req = suggestions.SuggestionsRequest(
|
|
messages=[
|
|
suggestions.SuggestionMessage(role="user", content="Hi"),
|
|
suggestions.SuggestionMessage(role="assistant", content="Hello"),
|
|
],
|
|
n=3,
|
|
model_name=None,
|
|
)
|
|
fake_model = MagicMock()
|
|
fake_model.ainvoke = AsyncMock(return_value=MagicMock(content='```json\n["Q1", "Q2", "Q3", "Q4"]\n```'))
|
|
monkeypatch.setattr(oneshot_llm, "create_chat_model", lambda **kwargs: fake_model)
|
|
|
|
# Bypass the require_permission decorator (which needs request +
|
|
# thread_store) — these tests cover the parsing logic.
|
|
result = asyncio.run(suggestions.generate_suggestions.__wrapped__("t1", req, request=None, config=SimpleNamespace(suggestions=SimpleNamespace(enabled=True))))
|
|
|
|
assert result.suggestions == ["Q1", "Q2", "Q3"]
|
|
fake_model.ainvoke.assert_awaited_once()
|
|
assert fake_model.ainvoke.await_args.kwargs["config"] == {"run_name": "suggest_agent"}
|
|
|
|
|
|
def test_generate_suggestions_respects_configured_max(monkeypatch):
|
|
req = suggestions.SuggestionsRequest(
|
|
messages=[
|
|
suggestions.SuggestionMessage(role="user", content="Hi"),
|
|
suggestions.SuggestionMessage(role="assistant", content="Hello"),
|
|
],
|
|
n=4,
|
|
model_name=None,
|
|
)
|
|
fake_model = MagicMock()
|
|
fake_model.ainvoke = AsyncMock(return_value=MagicMock(content='["Q1", "Q2", "Q3", "Q4"]'))
|
|
monkeypatch.setattr(oneshot_llm, "create_chat_model", lambda **kwargs: fake_model)
|
|
|
|
result = asyncio.run(
|
|
suggestions.generate_suggestions.__wrapped__(
|
|
"t1",
|
|
req,
|
|
request=None,
|
|
config=SimpleNamespace(suggestions=SimpleNamespace(enabled=True, max_suggestions=2)),
|
|
)
|
|
)
|
|
|
|
assert result.suggestions == ["Q1", "Q2"]
|
|
|
|
|
|
def test_generate_suggestions_injects_deerflow_trace_metadata_when_langfuse_enabled(monkeypatch):
|
|
monkeypatch.setenv("LANGFUSE_TRACING", "true")
|
|
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test")
|
|
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test")
|
|
from deerflow.config.tracing_config import reset_tracing_config
|
|
|
|
reset_tracing_config()
|
|
req = suggestions.SuggestionsRequest(
|
|
messages=[
|
|
suggestions.SuggestionMessage(role="user", content="Hi"),
|
|
suggestions.SuggestionMessage(role="assistant", content="Hello"),
|
|
],
|
|
n=1,
|
|
model_name="suggest-model",
|
|
)
|
|
fake_model = MagicMock()
|
|
fake_model.ainvoke = AsyncMock(return_value=MagicMock(content='["Q1"]'))
|
|
monkeypatch.setattr(oneshot_llm, "create_chat_model", lambda **kwargs: fake_model)
|
|
|
|
try:
|
|
with request_trace_context("suggest-trace-1"):
|
|
result = asyncio.run(suggestions.generate_suggestions.__wrapped__("thread-suggest", req, request=None, config=SimpleNamespace(suggestions=SimpleNamespace(enabled=True))))
|
|
finally:
|
|
reset_tracing_config()
|
|
|
|
assert result.suggestions == ["Q1"]
|
|
metadata = fake_model.ainvoke.await_args.kwargs["config"]["metadata"]
|
|
assert metadata["deerflow_trace_id"] == "suggest-trace-1"
|
|
assert metadata["langfuse_session_id"] == "thread-suggest"
|
|
assert metadata["langfuse_trace_name"] == "suggest_agent"
|
|
|
|
|
|
def test_generate_suggestions_parses_list_block_content(monkeypatch):
|
|
req = suggestions.SuggestionsRequest(
|
|
messages=[
|
|
suggestions.SuggestionMessage(role="user", content="Hi"),
|
|
suggestions.SuggestionMessage(role="assistant", content="Hello"),
|
|
],
|
|
n=2,
|
|
model_name=None,
|
|
)
|
|
fake_model = MagicMock()
|
|
fake_model.ainvoke = AsyncMock(return_value=MagicMock(content=[{"type": "text", "text": '```json\n["Q1", "Q2"]\n```'}]))
|
|
monkeypatch.setattr(oneshot_llm, "create_chat_model", lambda **kwargs: fake_model)
|
|
|
|
# Bypass the require_permission decorator (which needs request +
|
|
# thread_store) — these tests cover the parsing logic.
|
|
result = asyncio.run(suggestions.generate_suggestions.__wrapped__("t1", req, request=None, config=SimpleNamespace(suggestions=SimpleNamespace(enabled=True))))
|
|
|
|
assert result.suggestions == ["Q1", "Q2"]
|
|
fake_model.ainvoke.assert_awaited_once()
|
|
assert fake_model.ainvoke.await_args.kwargs["config"] == {"run_name": "suggest_agent"}
|
|
|
|
|
|
def test_generate_suggestions_parses_output_text_block_content(monkeypatch):
|
|
req = suggestions.SuggestionsRequest(
|
|
messages=[
|
|
suggestions.SuggestionMessage(role="user", content="Hi"),
|
|
suggestions.SuggestionMessage(role="assistant", content="Hello"),
|
|
],
|
|
n=2,
|
|
model_name=None,
|
|
)
|
|
fake_model = MagicMock()
|
|
fake_model.ainvoke = AsyncMock(return_value=MagicMock(content=[{"type": "output_text", "text": '```json\n["Q1", "Q2"]\n```'}]))
|
|
monkeypatch.setattr(oneshot_llm, "create_chat_model", lambda **kwargs: fake_model)
|
|
|
|
# Bypass the require_permission decorator (which needs request +
|
|
# thread_store) — these tests cover the parsing logic.
|
|
result = asyncio.run(suggestions.generate_suggestions.__wrapped__("t1", req, request=None, config=SimpleNamespace(suggestions=SimpleNamespace(enabled=True))))
|
|
|
|
assert result.suggestions == ["Q1", "Q2"]
|
|
fake_model.ainvoke.assert_awaited_once()
|
|
assert fake_model.ainvoke.await_args.kwargs["config"] == {"run_name": "suggest_agent"}
|
|
|
|
|
|
def test_generate_suggestions_returns_empty_on_model_error(monkeypatch):
|
|
req = suggestions.SuggestionsRequest(
|
|
messages=[suggestions.SuggestionMessage(role="user", content="Hi")],
|
|
n=2,
|
|
model_name=None,
|
|
)
|
|
fake_model = MagicMock()
|
|
fake_model.ainvoke = AsyncMock(side_effect=RuntimeError("boom"))
|
|
monkeypatch.setattr(oneshot_llm, "create_chat_model", lambda **kwargs: fake_model)
|
|
|
|
# Bypass the require_permission decorator (which needs request +
|
|
# thread_store) — these tests cover the parsing logic.
|
|
result = asyncio.run(suggestions.generate_suggestions.__wrapped__("t1", req, request=None, config=SimpleNamespace(suggestions=SimpleNamespace(enabled=True))))
|
|
|
|
assert result.suggestions == []
|
|
|
|
|
|
def test_generate_suggestions_returns_empty_when_disabled(monkeypatch):
|
|
"""Ensure suggestions are bypassed and returned an empty list when disabled in config."""
|
|
req = suggestions.SuggestionsRequest(
|
|
messages=[
|
|
suggestions.SuggestionMessage(role="user", content="Hi"),
|
|
suggestions.SuggestionMessage(role="assistant", content="Hello"),
|
|
],
|
|
n=3,
|
|
model_name=None,
|
|
)
|
|
|
|
mock_config = SimpleNamespace(suggestions=SimpleNamespace(enabled=False))
|
|
|
|
fake_model = MagicMock()
|
|
fake_model.ainvoke = AsyncMock(side_effect=RuntimeError("Model should not be called."))
|
|
monkeypatch.setattr(oneshot_llm, "create_chat_model", lambda **kwargs: fake_model)
|
|
|
|
result = asyncio.run(suggestions.generate_suggestions.__wrapped__("t1", req, request=None, config=mock_config))
|
|
|
|
assert result.suggestions == []
|
|
fake_model.ainvoke.assert_not_called()
|
|
|
|
|
|
def test_get_suggestions_config():
|
|
"""Ensure the GET /config endpoint correctly returns the boolean state."""
|
|
|
|
# Test when enabled
|
|
mock_config_true = SimpleNamespace(suggestions=SimpleNamespace(enabled=True))
|
|
result_true = asyncio.run(suggestions.get_suggestions_config(config=mock_config_true))
|
|
assert result_true.enabled is True
|
|
assert result_true.max_suggestions == 3
|
|
|
|
# Test when disabled
|
|
mock_config_false = SimpleNamespace(suggestions=SimpleNamespace(enabled=False, max_suggestions=4))
|
|
result_false = asyncio.run(suggestions.get_suggestions_config(config=mock_config_false))
|
|
assert result_false.enabled is False
|
|
assert result_false.max_suggestions == 4
|