From 4ced9dddf50a5ddd90bec57aaae874bc1fd649fc Mon Sep 17 00:00:00 2001 From: inchang-ing <291394088@qq.com> Date: Thu, 1 Oct 2026 08:12:32 +0800 Subject: [PATCH] fix(vllm): fall back to legacy reasoning_content field (#6048) * fix(vllm): fall back to legacy reasoning_content field * fix(vllm): document the reasoning_content fallback and pin precedence The previous change set left the docs describing the provider as preserving only `reasoning`: - the module docstring now mentions the legacy `reasoning_content` wire field, and the vLLM Provider bullet in models/AGENTS.md is extended to match the repo's documentation-update policy; - _create_chat_result evaluated choice["message"] twice, once per key; hoist it into a single local and read both keys from it; - two regression tests pin that a payload carrying both fields keeps `reasoning`, so a future refactor cannot silently invert the precedence (the existing tests only covered the absent-`reasoning` case). Reported-by: willem-bd * refactor(vllm): share reasoning fallback and cover follow-up payloads --------- Co-authored-by: inchang-ing <197932532+inchang-ing@users.noreply.github.com> Co-authored-by: Willem Jiang --- .../harness/deerflow/models/AGENTS.md | 2 +- .../harness/deerflow/models/vllm_provider.py | 19 ++- backend/tests/test_vllm_provider.py | 146 +++++++++++++++++- 3 files changed, 160 insertions(+), 7 deletions(-) diff --git a/backend/packages/harness/deerflow/models/AGENTS.md b/backend/packages/harness/deerflow/models/AGENTS.md index 92d3c5f7e6..08ce52c32b 100644 --- a/backend/packages/harness/deerflow/models/AGENTS.md +++ b/backend/packages/harness/deerflow/models/AGENTS.md @@ -72,7 +72,7 @@ the same policy first so run metadata reports the effective values. Design note: ### vLLM Provider (`packages/harness/deerflow/models/vllm_provider.py`) - `VllmChatModel` subclasses `langchain_openai:ChatOpenAI` for vLLM 0.19.0 OpenAI-compatible endpoints -- Preserves vLLM's non-standard assistant `reasoning` field on full responses, streaming deltas, and follow-up tool-call turns +- Preserves vLLM's non-standard assistant `reasoning` field on full responses, streaming deltas, and follow-up tool-call turns, falling back to the legacy `reasoning_content` wire field when `reasoning` is absent or null (a payload carrying both keeps `reasoning`); `_pick_reasoning` owns this precedence across all three paths and preserves empty-string `reasoning` rather than falling back - Designed for configs that enable thinking through `extra_body.chat_template_kwargs.enable_thinking` on vLLM 0.19.0 Qwen reasoning models, while accepting the older `thinking` alias - `cumulative_stream_usage` is an opt-in model setting (default `false`) for endpoints that repeat cumulative token totals on each streaming chunk. The provider converts snapshots to deltas only when a stable completion id is present, isolates interleaved streams by id, and leaves the original usage untouched otherwise. Per-model tracking is lock-protected and cleared on the trailing empty-`choices` frame whether or not that frame carries usage. A soft cap of 1024 ids evicts only entries idle for at least one hour; active streams may temporarily exceed the cap so eviction cannot corrupt their deltas. Regression coverage lives in `tests/test_vllm_provider.py`. diff --git a/backend/packages/harness/deerflow/models/vllm_provider.py b/backend/packages/harness/deerflow/models/vllm_provider.py index ba75d8191a..7eca9c012d 100644 --- a/backend/packages/harness/deerflow/models/vllm_provider.py +++ b/backend/packages/harness/deerflow/models/vllm_provider.py @@ -10,6 +10,11 @@ This provider preserves ``reasoning`` on: - non-streaming responses - streaming deltas - multi-turn request payloads + +Deployments that still emit the older ``reasoning_content`` wire field are +covered too: when ``reasoning`` is absent or null the provider falls back to +``reasoning_content``, so either spelling is echoed back under the single +``reasoning`` key. A payload carrying both keeps ``reasoning``. """ from __future__ import annotations @@ -99,6 +104,12 @@ def _reasoning_to_text(reasoning: Any) -> str: return str(reasoning) +def _pick_reasoning(fields: Mapping[str, Any]) -> Any: + """Prefer non-null ``reasoning`` (including empty strings) over the legacy field.""" + reasoning = fields.get("reasoning") + return fields.get("reasoning_content") if reasoning is None else reasoning + + def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], default_class: type[BaseMessageChunk]) -> BaseMessageChunk: """Convert a streaming delta to a LangChain message chunk while preserving reasoning.""" id_ = _dict.get("id") @@ -112,7 +123,7 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def function_call["name"] = "" additional_kwargs["function_call"] = function_call - reasoning = _dict.get("reasoning") + reasoning = _pick_reasoning(_dict) if reasoning is not None: additional_kwargs["reasoning"] = reasoning reasoning_text = _reasoning_to_text(reasoning) @@ -157,9 +168,7 @@ def _convert_delta_to_message_chunk_with_reasoning(_dict: Mapping[str, Any], def def _restore_reasoning_field(payload_msg: dict[str, Any], orig_msg: AIMessage) -> None: """Re-inject vLLM reasoning onto outgoing assistant messages.""" - reasoning = orig_msg.additional_kwargs.get("reasoning") - if reasoning is None: - reasoning = orig_msg.additional_kwargs.get("reasoning_content") + reasoning = _pick_reasoning(orig_msg.additional_kwargs) if reasoning is not None: payload_msg["reasoning"] = reasoning @@ -253,7 +262,7 @@ class VllmChatModel(ChatOpenAI): message = generation.message if not isinstance(message, AIMessage): continue - reasoning = choice.get("message", {}).get("reasoning") + reasoning = _pick_reasoning(choice.get("message") or {}) if reasoning is None: continue message.additional_kwargs["reasoning"] = reasoning diff --git a/backend/tests/test_vllm_provider.py b/backend/tests/test_vllm_provider.py index fe64a7002a..3f98b6ca85 100644 --- a/backend/tests/test_vllm_provider.py +++ b/backend/tests/test_vllm_provider.py @@ -1,6 +1,7 @@ from __future__ import annotations -from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage +import pytest +from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage, ToolMessage from deerflow.models.vllm_provider import VllmChatModel @@ -76,6 +77,40 @@ def test_vllm_provider_restores_reasoning_in_request_payload(): assert assistant_message["tool_calls"][0]["function"]["name"] == "bash" +@pytest.mark.parametrize( + ("fields", "expected"), + [ + pytest.param({"reasoning_content": "legacy"}, "legacy", id="legacy-only"), + pytest.param({"reasoning": None, "reasoning_content": "legacy"}, "legacy", id="null-canonical"), + pytest.param({"reasoning": "new", "reasoning_content": "legacy"}, "new", id="both-fields"), + pytest.param({"reasoning": "", "reasoning_content": "legacy"}, "", id="empty-canonical"), + pytest.param({"reasoning_content": ""}, "", id="empty-legacy"), + pytest.param({}, None, id="no-reasoning"), + ], +) +def test_vllm_provider_restores_reasoning_field_precedence_in_request_payload(fields, expected): + model = _make_model() + payload = model._get_request_payload( + [ + HumanMessage(content="Inspect the workspace."), + AIMessage( + content="", + tool_calls=[{"name": "bash", "args": {"cmd": "pwd"}, "id": "tool-1", "type": "tool_call"}], + additional_kwargs=fields, + ), + ToolMessage(content="/workspace", tool_call_id="tool-1"), + ] + ) + + assistant_message = payload["messages"][1] + if expected is None: + assert "reasoning" not in assistant_message + else: + assert assistant_message["reasoning"] == expected + assert assistant_message["tool_calls"][0]["function"]["name"] == "bash" + assert payload["messages"][2]["tool_call_id"] == "tool-1" + + def test_vllm_provider_normalizes_legacy_thinking_kwarg_to_enable_thinking(): model = VllmChatModel( model="qwen3", @@ -507,3 +542,112 @@ def test_vllm_provider_evicts_only_idle_streams_above_soft_capacity(monkeypatch) "chatcmpl-2", "chatcmpl-3", ] + + +def test_vllm_provider_falls_back_to_reasoning_content_in_chat_result(): + model = _make_model() + result = model._create_chat_result( + { + "model": "Qwen/QwQ-32B", + "choices": [ + { + "message": { + "role": "assistant", + "content": "42", + "reasoning_content": "I compared the two numbers directly.", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) + + message = result.generations[0].message + assert message.additional_kwargs["reasoning"] == "I compared the two numbers directly." + assert message.additional_kwargs["reasoning_content"] == "I compared the two numbers directly." + + payload = model._get_request_payload([HumanMessage(content="Compare the numbers."), message, HumanMessage(content="Continue.")]) + assert payload["messages"][1]["reasoning"] == "I compared the two numbers directly." + + +def test_vllm_provider_falls_back_to_reasoning_content_in_streaming_chunks(): + model = _make_model() + chunk = model._convert_chunk_to_generation_chunk( + { + "model": "Qwen/QwQ-32B", + "choices": [ + { + "delta": { + "role": "assistant", + "reasoning_content": "First, call the weather tool.", + "content": "Calling tool...", + }, + "finish_reason": None, + } + ], + }, + AIMessageChunk, + {}, + ) + + assert chunk is not None + assert chunk.message.additional_kwargs["reasoning"] == "First, call the weather tool." + assert chunk.message.additional_kwargs["reasoning_content"] == "First, call the weather tool." + assert chunk.message.content == "Calling tool..." + + payload = model._get_request_payload([HumanMessage(content="Check the weather."), chunk.message, HumanMessage(content="Continue.")]) + assert payload["messages"][1]["reasoning"] == "First, call the weather tool." + + +def test_vllm_provider_prefers_reasoning_over_reasoning_content_in_chat_result(): + """A payload carrying both fields keeps `reasoning` (#6047).""" + model = _make_model() + result = model._create_chat_result( + { + "model": "Qwen/QwQ-32B", + "choices": [ + { + "message": { + "role": "assistant", + "content": "42", + "reasoning": "new", + "reasoning_content": "legacy", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) + + message = result.generations[0].message + assert message.additional_kwargs["reasoning"] == "new" + assert message.additional_kwargs["reasoning_content"] == "new" + + +def test_vllm_provider_prefers_reasoning_over_reasoning_content_in_streaming_chunks(): + """Streaming deltas resolve both fields the same way (#6047).""" + model = _make_model() + chunk = model._convert_chunk_to_generation_chunk( + { + "model": "Qwen/QwQ-32B", + "choices": [ + { + "delta": { + "role": "assistant", + "reasoning": "new", + "reasoning_content": "legacy", + "content": "Calling tool...", + }, + "finish_reason": None, + } + ], + }, + AIMessageChunk, + {}, + ) + + assert chunk is not None + assert chunk.message.additional_kwargs["reasoning"] == "new" + assert chunk.message.additional_kwargs["reasoning_content"] == "new"