diff --git a/README.md b/README.md index 0c44fd1e1a..1632dcbc6c 100644 --- a/README.md +++ b/README.md @@ -1079,6 +1079,7 @@ In plan mode, malformed TODO statuses return normal tool-validation errors witho Tool-produced paths and URLs can be retained as short artifact handles across context compaction (`tool_artifacts` in `config.yaml`). Handles distinguish separate tool-result occurrences, even when a provider reuses call IDs. Standard MCP image URLs are captured alongside file references, including results from stdio and HTTP servers. Detected file URLs preserve their query strings and fragments. Free-text sandbox paths retain spaces inside matching ASCII quotes, backticks, or curly quote pairs, including backslash-escaped wrappers. Unterminated spans and delimited values containing a line break or tab are skipped. Free-text artifact references retain balanced ASCII and CJK brackets in literal paths and URLs while trimming surrounding prose punctuation. When PII redaction is enabled, model-visible artifact labels follow that policy; internal references stay intact for tool argument resolution. The configured registry limit retains the newest artifacts, while checkpointed processing identities prevent evicted results from being recaptured after restart. Resolution runs before authorization and write-safety checks; unknown or expired handles return an error without executing the tool. Small unknown structured results may be retained as complete JSON up to 4096 UTF-8 bytes; empty or oversized payloads are skipped. Handles are agent-local: task arguments resolve parent handles to concrete references, and delegated reports must return concrete references rather than child-local handles. A truncated model projection reports how many handles are omitted. DeerFlow supports configurable MCP servers and skills to extend its capabilities. +Inline MCP audio results are not supported. Stdio tools report an explicit error, matching HTTP/SSE tools, rather than treating base64 audio as a successful text result. When durable MCP background tasks are enabled, agents can use `list_background_tasks(status="failed")` or `status="input_required"` to find failed tasks or tasks awaiting input in the current chat. Other supported statuses are `submitted`, `working`, `completed`, and `cancelled`; omitting the status preserves existing behavior. The database applies the filter before limiting results to the 20 most recent matching tasks. When combined with `active_only=true`, both filters apply: active statuses are `submitted`, `working`, and `input_required`, so terminal statuses return an empty list. `GET /api/threads/{thread_id}/mcp-tasks` supports the same `status` and `active_only` filters before its `limit` (default 50, range 1–100); for example, `?status=failed&limit=20`. Unknown statuses return 422, and omitting the filters preserves the existing response. For HTTP/SSE MCP servers, OAuth token flows are supported (`client_credentials`, `refresh_token`). Missing, malformed, or out-of-range token response `expires_in` values use a one-hour default lifetime. This includes lifetimes that cannot be added to the current time without overflowing the expiry timestamp. diff --git a/backend/packages/harness/deerflow/mcp/AGENTS.md b/backend/packages/harness/deerflow/mcp/AGENTS.md index b3fcc9aee2..735aa9d986 100644 --- a/backend/packages/harness/deerflow/mcp/AGENTS.md +++ b/backend/packages/harness/deerflow/mcp/AGENTS.md @@ -1,5 +1,7 @@ ### MCP System (`packages/harness/deerflow/mcp/`) +Inline `AudioContent` raises `NotImplementedError`, matching the adapter's unsupported-audio behavior. Never stringify its base64 payload into text; `ToolErrorHandlingMiddleware` reports a recoverable tool error. Coverage: `tests/test_mcp_audio_content.py`. + Poll and cancel snapshot writes rejected by the ownership fence log the discard; neither path changes retry semantics when a peer owns the task. The shared `McpTaskService._apply_cancel_snapshot` checks the control lease and logs rejected diff --git a/backend/packages/harness/deerflow/mcp/tools.py b/backend/packages/harness/deerflow/mcp/tools.py index 819d9efa45..05444ff7a8 100644 --- a/backend/packages/harness/deerflow/mcp/tools.py +++ b/backend/packages/harness/deerflow/mcp/tools.py @@ -510,7 +510,7 @@ def _convert_call_tool_result( from langchain_core.messages import ToolMessage from langchain_core.messages.content import create_file_block, create_image_block, create_text_block from langchain_core.tools import ToolException - from mcp.types import EmbeddedResource, ImageContent, ResourceLink, TextContent, TextResourceContents + from mcp.types import AudioContent, EmbeddedResource, ImageContent, ResourceLink, TextContent, TextResourceContents # Pass ToolMessage through directly (interceptor short-circuit). if isinstance(call_tool_result, ToolMessage): @@ -554,6 +554,8 @@ def _convert_call_tool_result( lc_content.append(create_text_block(text=_resolve_text(item.text))) elif isinstance(item, ImageContent): lc_content.append(create_image_block(base64=item.data, mime_type=item.mimeType)) + elif isinstance(item, AudioContent): + raise NotImplementedError(f"AudioContent conversion to LangChain content blocks is not yet supported. Received audio with mime type: {item.mimeType}") elif isinstance(item, ResourceLink): mime = item.mimeType or None url = _resolve_link_url(str(item.uri)) diff --git a/backend/tests/test_mcp_audio_content.py b/backend/tests/test_mcp_audio_content.py new file mode 100644 index 0000000000..70ebb74060 --- /dev/null +++ b/backend/tests/test_mcp_audio_content.py @@ -0,0 +1,118 @@ +"""Unsupported MCP audio must not become model-visible base64 text.""" + +import base64 +import io +import wave +from contextlib import asynccontextmanager + +import pytest +from _agent_e2e_helpers import build_single_tool_call_model +from langchain.agents import create_agent +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.messages import ToolMessage +from langchain_mcp_adapters.tools import load_mcp_tools +from mcp.types import AudioContent, CallToolResult, ListToolsResult, TextContent, Tool + +from deerflow.agents.middlewares.tool_error_handling_middleware import ToolErrorHandlingMiddleware +from deerflow.config.extensions_config import ExtensionsConfig +from deerflow.config.paths import Paths +from deerflow.mcp.session_pool import MCPSessionPool +from deerflow.mcp.tools import _convert_call_tool_result, _RawServerDiscovery, _wrap_discovered_server_tools + + +@pytest.fixture +def audio_content(): + """A valid, synthetic WAV; no provider or user audio is involved.""" + buffer = io.BytesIO() + with wave.open(buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(b"\x00\x00" * 1600) + return AudioContent(type="audio", data=base64.b64encode(buffer.getvalue()).decode("ascii"), mimeType="audio/wav") + + +@pytest.mark.parametrize("is_error", [False, True]) +@pytest.mark.parametrize("with_text", [False, True]) +def test_audio_conversion_fails_without_stringifying_payload(audio_content, is_error, with_text): + content = [TextContent(type="text", text="Generated audio")] if with_text else [] + content.append(audio_content) + + with pytest.raises(NotImplementedError, match="AudioContent.*not.*supported") as exc: + _convert_call_tool_result(CallToolResult(content=content, isError=is_error)) + + assert f"Received audio with mime type: {audio_content.mimeType}" in str(exc.value) + assert audio_content.data not in str(exc.value) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transport", ["stdio", "streamable_http"]) +async def test_audio_tool_reports_error_without_payload_in_agent_messages(transport, audio_content, monkeypatch, tmp_path): + calls = [] + model_inputs = [] + + class CaptureModelInputs(BaseCallbackHandler): + def on_chat_model_start(self, serialized, messages, **kwargs): + model_inputs.extend(messages) + + class Session: + async def initialize(self): + pass + + async def list_tools(self, **kwargs): + return ListToolsResult(tools=[Tool(name="speak", description="Generate speech", inputSchema={"type": "object", "properties": {}})]) + + async def call_tool(self, name, arguments, **kwargs): + calls.append((name, arguments)) + return CallToolResult(content=[TextContent(type="text", text="Generated audio"), audio_content]) + + @asynccontextmanager + async def fake_session(*args, **kwargs): + yield Session() + + # Stub only the external session and filesystem root. Discovery, wrapping, + # pooling, graph execution, result conversion, and error handling stay real. + monkeypatch.setattr("langchain_mcp_adapters.tools.create_session", fake_session) + monkeypatch.setattr("langchain_mcp_adapters.sessions.create_session", fake_session) + monkeypatch.setattr("deerflow.mcp.tools.get_paths", lambda: Paths(tmp_path)) + connection = {"transport": "stdio", "command": "offline-audio-fixture"} if transport == "stdio" else {"transport": transport, "url": "https://example.com/mcp"} + discovered = await load_mcp_tools(None, connection=connection, server_name="speech", tool_name_prefix=True) + pool = MCPSessionPool() + try: + wrapped = _wrap_discovered_server_tools( + ExtensionsConfig(), + "speech", + connection, + _RawServerDiscovery(discovered, []), + tool_name_prefix=True, + session_init_timeout=None, + pool=pool, + binding=None, + ownership_domain="deployment", + connection_scope="deployment", + ) + graph = create_agent( + model=build_single_tool_call_model(tool_name="speech_speak", tool_args={}), + tools=list(wrapped.tools), + middleware=[ToolErrorHandlingMiddleware()], + context_schema=dict, + ) + state = await graph.ainvoke( + {"messages": [("user", "Generate speech")]}, + config={"callbacks": [CaptureModelInputs()]}, + context={"thread_id": "audio-thread", "thread_incarnation": "audio-fixture"}, + ) + finally: + await pool.close_all() + + assert calls == [("speak", {})] + (result,) = [message for message in state["messages"] if isinstance(message, ToolMessage)] + assert result.status == "error" + assert "AudioContent" in result.content + assert "not yet supported" in result.content + assert f"Received audio with mime type: {audio_content.mimeType}" in result.content + assert len(model_inputs) == 2 + assert any(isinstance(message, ToolMessage) and message.status == "error" for message in model_inputs[-1]) + for message in [*state["messages"], *model_inputs[-1]]: + assert audio_content.data not in message.model_dump_json() + assert state["messages"][-1].content == "done"