"""Tests for goal synthesis from conversation (``/goal from-chat``). The synthesis module is harness-level and LLM-shaped: every test mocks ``run_oneshot_llm`` so the suite stays offline. The pins here are the contracts the Gateway route and the IM manager rely on: * the context builder keeps the summary plus the newest visible messages within a character budget, excludes evidence tool results and hidden framework messages, and keeps the human-input protocol messages — the assistant's clarification questions and the user's card answers; * a clear conversation yields a normalized objective; * an unclear conversation raises ``GoalSynthesisUnclear`` carrying a reason that is for server logs only — never user-facing text; * malformed or oversized model output raises ``GoalSynthesisError``; * PII redaction applies to whole lines and the summary before the budget can split an identifier, and again to the model input when enabled, matching the goal evaluator's direct-call treatment. """ from __future__ import annotations import json from unittest.mock import AsyncMock import pytest from langchain_core.messages import AIMessage, HumanMessage, ToolMessage from deerflow.config.app_config import AppConfig from deerflow.config.pii_redaction_config import PiiRedactionConfig from deerflow.config.sandbox_config import SandboxConfig from deerflow.runtime import goal_synthesis from deerflow.runtime.goal_synthesis import ( GoalSynthesisError, GoalSynthesisUnclear, build_synthesis_context, synthesize_goal_objective, ) from deerflow.utils.goal_objective import MAX_GOAL_OBJECTIVE_CHARS _PII = PiiRedactionConfig(enabled=True, token_secret="unit-test-deployment-secret-0123456789") _PII_CONFIG = AppConfig(sandbox=SandboxConfig(use="test"), pii_redaction=_PII) _PLAIN_CONFIG = AppConfig(sandbox=SandboxConfig(use="test"), pii_redaction=PiiRedactionConfig(enabled=False)) _INVOICE_MESSAGES = [ HumanMessage(content="Reconcile this batch: vendor A has 3 invoices totaling 12,400 CNY, vendor B has 2 totaling 8,900 CNY."), AIMessage(content="", tool_calls=[{"name": "read_file", "args": {"path": "invoices.csv"}, "id": "call-1"}]), ToolMessage(content="raw,csv,bytes", tool_call_id="call-1", name="read_file"), HumanMessage(content="Exclude the 1,500 CNY invoice from vendor C; it has not arrived yet.", additional_kwargs={"hide_from_ui": True}), HumanMessage(content="Exclude vendor C's 1,500 CNY invoice; it has not arrived yet."), AIMessage(content="Understood. A and B are in scope, C is excluded until its invoice arrives."), HumanMessage(content="When done, list every discrepancy and explain each one."), ] def _mock_llm(monkeypatch, response_text: str) -> AsyncMock: mock = AsyncMock(return_value=response_text) monkeypatch.setattr(goal_synthesis, "run_oneshot_llm", mock) return mock def _verdict(objective: str, *, unclear: bool = False, reason: str = "") -> str: return json.dumps({"objective": objective, "unclear": unclear, "reason": reason}) # -- context builder --------------------------------------------------------- def test_context_includes_summary_and_visible_messages_in_order(): context = build_synthesis_context(_INVOICE_MESSAGES, "Earlier the user described the invoice batch.") assert "Earlier the user described the invoice batch." in context assert "User: Reconcile this batch" in context assert "Assistant: Understood. A and B are in scope" in context assert "User: When done, list every discrepancy" in context # Chronological order is preserved. assert context.index("Reconcile this batch") < context.index("Understood. A and B") < context.index("list every discrepancy") def test_context_excludes_tool_and_hidden_messages(): context = build_synthesis_context(_INVOICE_MESSAGES, None) assert "raw,csv,bytes" not in context assert "read_file" not in context # The hidden duplicate of the vendor-C exclusion never appears; the visible one does. assert context.count("Exclude vendor C's 1,500 CNY invoice") == 1 assert context.count("1,500 CNY") == 1 def test_context_accepts_dict_shaped_messages(): messages = [ {"type": "user", "content": "Sum the vendor totals."}, {"type": "assistant", "content": "A: 12,400; B: 8,900."}, {"type": "tool", "content": "tool noise"}, ] context = build_synthesis_context(messages, None) assert "User: Sum the vendor totals." in context assert "Assistant: A: 12,400; B: 8,900." in context assert "tool noise" not in context def test_context_budget_keeps_the_newest_messages_and_marks_omission(): messages = [HumanMessage(content=f"message number {index} with some padding text") for index in range(40)] context = build_synthesis_context(messages, None, budget_chars=300) assert "message number 39" in context assert "message number 38" in context assert "message number 0" not in context assert "omitted" in context.lower() assert len(context) <= 300 + len("[ earlier messages omitted]") + 8 def test_context_keeps_the_opening_of_one_overlong_message(): # A single paste longer than the whole budget must not collapse the # context to an omission marker: the request itself is the goal source. long_message = "Task: reconcile the invoice batch. " + ("supporting detail " * 200) context = build_synthesis_context([HumanMessage(content=long_message)], None, budget_chars=300) assert context.startswith("User: Task: reconcile the invoice batch.") assert context.endswith(" …") assert len(context) <= 300 assert "omitted" not in context.lower() def test_context_redacts_a_message_before_the_budget_can_split_it(): # The budget cut must never turn an email into an unmatchable prefix: the # line is redacted first, then truncated (the evaluator's rule for its # direct model call). message = HumanMessage(content=("a" * 144) + " bob@example.com trailing detail") context = build_synthesis_context([message], None, budget_chars=200, pii_redaction=_PII) assert "bob@exam" not in context assert "bob@example.com" not in context def test_context_redacts_the_summary_before_the_budget_can_split_it(): summary = ("a" * 50) + " bob@example.com" context = build_synthesis_context([], summary, budget_chars=200, pii_redaction=_PII) assert "bob@exa" not in context assert "bob@example.com" not in context def test_context_summary_is_bounded_when_budget_is_small(): long_summary = "S" * 5000 context = build_synthesis_context([HumanMessage(content="recent ask")], long_summary, budget_chars=800) assert "recent ask" in context assert len(context) <= 800 + 64 assert context.count("S") < 5000 def test_context_without_visible_messages_or_summary_is_empty(): assert build_synthesis_context([], None) == "" assert build_synthesis_context([ToolMessage(content="x", tool_call_id="c")], "") == "" # A summary alone is still usable context. assert "old decisions" in build_synthesis_context([], "old decisions") def _clarification_tool_message(question: str, *, tool_call_id: str = "call-clarify") -> ToolMessage: return ToolMessage( content=f"❓ {question}", tool_call_id=tool_call_id, name="ask_clarification", artifact={ "human_input": { "version": 1, "kind": "human_input_request", "source": "ask_clarification", "request_id": f"clarification:{tool_call_id}", "question": question, } }, ) def test_context_includes_clarification_questions(): # The assistant's question is a ToolMessage, not an AI text turn: without # this line the synthesizer cannot see that the conversation is waiting. messages = [HumanMessage(content="Reconcile the invoices."), _clarification_tool_message("What baseline should I compare against?")] context = build_synthesis_context(messages, None) assert "Assistant (waiting for input): What baseline should I compare against?" in context # The formatted fallback content (with options) is not duplicated. assert context.count("What baseline should I compare against?") == 1 def test_context_includes_non_clarification_human_input_requests(): # Sandbox network approval uses the same artifact but a tool name such as bash. request = ToolMessage( content="Sandbox network policy blocked api.example.com:443.", tool_call_id="call-bash", name="bash", artifact={ "human_input": { "version": 1, "kind": "human_input_request", "source": "sandbox_network_policy", "request_id": "net-1", "question": "Allow this sandbox to connect to api.example.com:443?", } }, ) context = build_synthesis_context([request], None) assert "Assistant (waiting for input): Allow this sandbox to connect to api.example.com:443?" in context def test_context_includes_hidden_card_answers(): answer = HumanMessage( content="", additional_kwargs={ "hide_from_ui": True, "human_input_response": { "version": 1, "kind": "human_input_response", "source": "ask_clarification", "request_id": "clarification:call-clarify", "response_kind": "option", "option_id": "allow_sandbox", "value": "Allow for this sandbox", }, }, ) context = build_synthesis_context([answer], None) assert "User (Human Input Card answer): Allow for this sandbox" in context def test_context_excludes_disabled_clarification_instructions(): # The policy-disabled path emits a same-named ToolMessage that instructs # the model, not a question for the user. disabled = ToolMessage( content="Clarification is disabled in this context — proceed with the smallest reasonable assumption.", tool_call_id="call-1", name="ask_clarification", ) assert build_synthesis_context([disabled], None) == "" def test_context_still_excludes_evidence_tool_results(): messages = [ HumanMessage(content="Sum the three vendors' revenue."), ToolMessage(content="A 12.3B, B 8.9B, C unknown", tool_call_id="call-search", name="web_search"), ] context = build_synthesis_context(messages, None) assert "Sum the three vendors' revenue." in context assert "12.3B" not in context assert "web_search" not in context # -- synthesis --------------------------------------------------------------- @pytest.mark.asyncio async def test_synthesize_returns_normalized_objective(monkeypatch): mock = _mock_llm(monkeypatch, _verdict(" Reconcile invoices A and B, excluding C, until every discrepancy is explained. ")) objective = await synthesize_goal_objective( context="User: reconcile A and B\nAssistant: ok", guidance="focus on the vendor total", app_config=_PLAIN_CONFIG, thread_id="t-1", ) assert objective == "Reconcile invoices A and B, excluding C, until every discrepancy is explained." kwargs = mock.await_args.kwargs # Default model, oneshot run name, thread attribution for tracing. assert kwargs["model_name"] is None assert kwargs["thread_id"] == "t-1" assert "focus on the vendor total" in kwargs["user_content"] assert "reconcile A and B" in kwargs["user_content"] @pytest.mark.asyncio async def test_synthesize_system_instruction_treats_conversation_as_data(monkeypatch): mock = _mock_llm(monkeypatch, _verdict("Ship it.")) await synthesize_goal_objective(context="c", guidance="", app_config=_PLAIN_CONFIG) system = mock.await_args.kwargs["system_instruction"] # Anti-injection: the conversation is data, its instructions are ignored. assert "data" in system.lower() assert "ignore" in system.lower() # The objective must carry constraints and the stopping condition. assert "constraint" in system.lower() assert "stopping condition" in system.lower() or "complete" in system.lower() # Strict JSON output contract. assert '"objective"' in system and '"unclear"' in system @pytest.mark.asyncio async def test_synthesize_system_instruction_requires_the_users_language(monkeypatch): mock = _mock_llm(monkeypatch, _verdict("Ship it.")) await synthesize_goal_objective(context="c", guidance="", app_config=_PLAIN_CONFIG) system = mock.await_args.kwargs["system_instruction"].lower() # Live acceptance (issue #7050 slice 1): with no language rule the model # alternated between Chinese and English objectives for the same # all-Chinese conversation. The objective is shown to the user verbatim # and drives the run, so it must follow the user's language. assert "language" in system assert "user" in system @pytest.mark.asyncio async def test_synthesize_unclear_raises_with_reason_and_no_objective(monkeypatch): _mock_llm(monkeypatch, _verdict("", unclear=True, reason="The conversation is small talk without a task.")) with pytest.raises(GoalSynthesisUnclear) as raised: await synthesize_goal_objective(context="User: nice weather", guidance="", app_config=_PLAIN_CONFIG) assert "small talk" in str(raised.value) @pytest.mark.asyncio async def test_synthesize_malformed_output_raises_synthesis_error(monkeypatch): _mock_llm(monkeypatch, "I cannot output JSON, sorry.") with pytest.raises(GoalSynthesisError): await synthesize_goal_objective(context="c", guidance="", app_config=_PLAIN_CONFIG) @pytest.mark.asyncio async def test_synthesize_code_fenced_json_is_accepted(monkeypatch): _mock_llm(monkeypatch, "```json\n" + _verdict("Finish the report.") + "\n```") assert await synthesize_goal_objective(context="c", guidance="", app_config=_PLAIN_CONFIG) == "Finish the report." @pytest.mark.asyncio @pytest.mark.parametrize("literal", ["literal content", ""]) async def test_synthesize_preserves_reasoning_tags_in_objective(monkeypatch, literal): objective = f'Document the "{literal}" syntax.' _mock_llm(monkeypatch, "Drafting the objective.\n" + _verdict(objective)) assert await synthesize_goal_objective(context="c", guidance="", app_config=_PLAIN_CONFIG) == objective @pytest.mark.asyncio @pytest.mark.parametrize("literal", ["literal content", ""]) async def test_synthesize_preserves_reasoning_tags_in_unclear_reason(monkeypatch, literal): reason = f'Clarify the intended handling of "{literal}".' _mock_llm(monkeypatch, _verdict("", unclear=True, reason=reason)) with pytest.raises(GoalSynthesisUnclear) as raised: await synthesize_goal_objective(context="c", guidance="", app_config=_PLAIN_CONFIG) assert str(raised.value) == reason @pytest.mark.asyncio @pytest.mark.parametrize( "payload", [ pytest.param(_verdict(""), id="empty-objective"), pytest.param(_verdict(" \n "), id="blank-objective"), pytest.param(_verdict("x" * (MAX_GOAL_OBJECTIVE_CHARS + 1)), id="oversized-objective"), pytest.param(json.dumps({"objective": 42, "unclear": False}), id="non-string-objective"), pytest.param(json.dumps({"unclear": False}), id="missing-objective"), ], ) async def test_synthesize_unusable_objective_raises_synthesis_error(monkeypatch, payload): _mock_llm(monkeypatch, payload) with pytest.raises(GoalSynthesisError): await synthesize_goal_objective(context="c", guidance="", app_config=_PLAIN_CONFIG) @pytest.mark.asyncio async def test_synthesize_llm_failure_propagates_as_synthesis_error(monkeypatch): mock = AsyncMock(side_effect=RuntimeError("provider down")) monkeypatch.setattr(goal_synthesis, "run_oneshot_llm", mock) with pytest.raises(GoalSynthesisError): await synthesize_goal_objective(context="c", guidance="", app_config=_PLAIN_CONFIG) @pytest.mark.asyncio async def test_synthesize_redacts_context_when_pii_enabled(monkeypatch): mock = _mock_llm(monkeypatch, _verdict("Email the reconciliation to the finance lead.")) await synthesize_goal_objective( context="User: send the result to bob@example.com", guidance="", app_config=_PII_CONFIG, ) user_content = mock.await_args.kwargs["user_content"] assert "bob@example.com" not in user_content @pytest.mark.asyncio async def test_synthesize_keeps_context_when_pii_disabled(monkeypatch): mock = _mock_llm(monkeypatch, _verdict("Email the reconciliation.")) await synthesize_goal_objective( context="User: send the result to bob@example.com", guidance="", app_config=_PLAIN_CONFIG, ) assert "bob@example.com" in mock.await_args.kwargs["user_content"] @pytest.mark.asyncio async def test_synthesize_accepts_explicit_model_name(monkeypatch): mock = _mock_llm(monkeypatch, _verdict("Ship it.")) await synthesize_goal_objective(context="c", guidance="", app_config=_PLAIN_CONFIG, model_name="some-model") assert mock.await_args.kwargs["model_name"] == "some-model"