diff --git a/README.md b/README.md index b8d11c4f88..06cec7c0e5 100644 --- a/README.md +++ b/README.md @@ -835,7 +835,7 @@ already received by the browser, without an additional secret-redaction layer. In plan mode, malformed TODO statuses return normal tool-validation errors without aborting token attribution, so the agent can correct the call. -Tool-produced paths and URLs can be retained as short artifact handles across context compaction (`tool_artifacts` in `config.yaml`). Handles distinguish separate tool-result occurrences, even when a provider reuses call IDs. Detected file URLs preserve their query strings and fragments. When PII redaction is enabled, model-visible artifact labels follow that policy; internal references stay intact for tool argument resolution. The configured registry limit retains the newest artifacts, while checkpointed processing identities prevent evicted results from being recaptured after restart. Resolution runs before authorization and write-safety checks; unknown or expired handles return an error without executing the tool. Small unknown structured results may be retained as complete JSON up to 4096 UTF-8 bytes; empty or oversized payloads are skipped. Handles are agent-local: task arguments resolve parent handles to concrete references, and delegated reports must return concrete references rather than child-local handles. A truncated model projection reports how many handles are omitted. +Tool-produced paths and URLs can be retained as short artifact handles across context compaction (`tool_artifacts` in `config.yaml`). Handles distinguish separate tool-result occurrences, even when a provider reuses call IDs. Standard MCP image URLs are captured alongside file references, including results from stdio and HTTP servers. Detected file URLs preserve their query strings and fragments. When PII redaction is enabled, model-visible artifact labels follow that policy; internal references stay intact for tool argument resolution. The configured registry limit retains the newest artifacts, while checkpointed processing identities prevent evicted results from being recaptured after restart. Resolution runs before authorization and write-safety checks; unknown or expired handles return an error without executing the tool. Small unknown structured results may be retained as complete JSON up to 4096 UTF-8 bytes; empty or oversized payloads are skipped. Handles are agent-local: task arguments resolve parent handles to concrete references, and delegated reports must return concrete references rather than child-local handles. A truncated model projection reports how many handles are omitted. DeerFlow supports configurable MCP servers and skills to extend its capabilities. When durable MCP background tasks are enabled, agents can use `list_background_tasks(status="failed")` or `status="input_required"` to find failed tasks or tasks awaiting input in the current chat. Other supported statuses are `submitted`, `working`, `completed`, and `cancelled`; omitting the status preserves existing behavior. The database applies the filter before limiting results to the 20 most recent matching tasks. When combined with `active_only=true`, both filters apply: active statuses are `submitted`, `working`, and `input_required`, so terminal statuses return an empty list. `GET /api/threads/{thread_id}/mcp-tasks` supports the same `status` and `active_only` filters before its `limit` (default 50, range 1–100); for example, `?status=failed&limit=20`. Unknown statuses return 422, and omitting the filters preserves the existing response. diff --git a/backend/packages/harness/deerflow/tools/AGENTS.md b/backend/packages/harness/deerflow/tools/AGENTS.md index f6b1105b1c..457f626b10 100644 --- a/backend/packages/harness/deerflow/tools/AGENTS.md +++ b/backend/packages/harness/deerflow/tools/AGENTS.md @@ -1,5 +1,10 @@ ### Tool System (`packages/harness/deerflow/tools/`) +Artifact capture accepts standard top-level `url`/`mime_type` content blocks and +legacy `source.url`/`source.mime_type`. A present top-level `url` selects that +shape, even when invalid; never mix its MIME metadata with a conflicting legacy +source. Both shapes use the existing referenceability gate and per-result deduplication. + `artifact_registry.py` checks remote file suffixes against the complete parsed URL path without case sensitivity. Free-text URL and sandbox-path extraction strips common trailing ASCII/CJK prose punctuation while preserving CJK diff --git a/backend/packages/harness/deerflow/tools/artifact_registry.py b/backend/packages/harness/deerflow/tools/artifact_registry.py index 0648e02004..489c19d372 100644 --- a/backend/packages/harness/deerflow/tools/artifact_registry.py +++ b/backend/packages/harness/deerflow/tools/artifact_registry.py @@ -303,8 +303,9 @@ def extract_artifacts_from_result( results — become ``file`` entries, subject to the same referenceability gate as every other ref source (``data:``/``blob:`` and other unresolvable URIs are skipped). - 3. ``content`` blocks of type ``file`` / ``image`` with a URL source become - ``file`` / ``image`` entries. + 3. ``content`` blocks of type ``file`` / ``image`` with a top-level ``url`` + or legacy ``source.url`` become ``file`` / ``image`` entries. A present + top-level URL takes precedence, with MIME metadata from the same shape. 4. ``content`` text blocks and plain-string results are scanned conservatively for sandbox paths and remote file URLs (gated by ``detect_refs_in_text``). @@ -383,7 +384,10 @@ def extract_artifacts_from_result( block_type = block.get("type", "") if block_type in {"file", "image"}: - source = block.get("source") or {} + # Standard LangChain blocks keep URL/MIME at the top level. Select + # one shape before validation so conflicting legacy fields cannot + # replace an invalid standard URL or supply unrelated MIME metadata. + source = block if "url" in block else block.get("source") or {} if not isinstance(source, dict): continue url = source.get("url") diff --git a/backend/tests/test_artifact_registry.py b/backend/tests/test_artifact_registry.py index 4fde91f290..3e60c4c769 100644 --- a/backend/tests/test_artifact_registry.py +++ b/backend/tests/test_artifact_registry.py @@ -4,6 +4,7 @@ import json import pytest from langchain_core.messages import ToolMessage +from langchain_core.messages.content import create_file_block, create_image_block from deerflow.agents.thread_state import merge_tool_artifacts from deerflow.tools.artifact_registry import ( @@ -192,7 +193,55 @@ def test_extract_from_image_block(): assert entries[0]["real_ref"] == "/mnt/user-data/outputs/chart.png" -def test_file_block_rejects_data_and_blob_urls(): +@pytest.mark.parametrize("make_block,artifact_type,mime", [(create_file_block, "file", "application/pdf"), (create_image_block, "image", "image/png")]) +def test_extract_from_standard_url_block(make_block, artifact_type, mime): + url = "https://example.com/report?download=1#page=2" + result = ToolMessage(content=[make_block(url=url, mime_type=mime)], tool_call_id="call_1") + + (entry,) = extract_artifacts_from_result(result, thread_id="thread-1") + + assert (entry["real_ref"], entry["artifact_type"], entry["mime_type"]) == (url, artifact_type, mime) + + +@pytest.mark.parametrize("source", [{"url": "https://example.com/legacy.png", "mime_type": "image/jpeg"}, "malformed"]) +@pytest.mark.parametrize("mime", ["image/png", None]) +def test_standard_url_and_mime_take_precedence_over_legacy_source(source, mime): + block = create_image_block(url="https://example.com/chart.png", mime_type=mime) + block["source"] = source + result = ToolMessage(content=[block], tool_call_id="call_1") + + (entry,) = extract_artifacts_from_result(result, thread_id="thread-1") + + assert entry["real_ref"] == "https://example.com/chart.png" + assert entry.get("mime_type") == mime + + +@pytest.mark.parametrize("url", [None, "", "relative.png", "data:image/png;base64,AAAA", "blob:https://example.com/id"]) +def test_invalid_standard_url_does_not_fall_back_to_conflicting_legacy_source(url): + block = {"type": "image", "url": url, "source": {"url": "https://example.com/legacy.png"}} + result = ToolMessage(content=[block], tool_call_id="call_1") + + assert extract_artifacts_from_result(result, thread_id="thread-1") == [] + + +def test_standard_and_legacy_blocks_deduplicate_the_same_reference(): + url = "https://example.com/chart.png" + result = ToolMessage( + content=[ + create_image_block(url=url, mime_type="image/png"), + {"type": "file", "source": {"url": url, "mime_type": "application/octet-stream"}}, + {"type": "text", "text": f"Chart: {url}"}, + ], + tool_call_id="call_1", + ) + + (entry,) = extract_artifacts_from_result(result, thread_id="thread-1") + + assert (entry["real_ref"], entry["artifact_type"], entry["mime_type"]) == (url, "image", "image/png") + + +@pytest.mark.parametrize("standard", [False, True]) +def test_file_block_rejects_data_and_blob_urls(standard): """Embedded-resource URIs must never enter the registry or tool args.""" result = ToolMessage( content=[ @@ -204,6 +253,8 @@ def test_file_block_rejects_data_and_blob_urls(): tool_call_id="call-2b", name="mcp_embedded", ) + if standard: + result.content = [{"type": block["type"], "url": block["source"]["url"]} for block in result.content] entries = extract_artifacts_from_result(result, thread_id="thread-1") refs = {entry["real_ref"] for entry in entries} assert refs == {"/mnt/user-data/outputs/real.txt"} diff --git a/backend/tests/test_mcp_artifact_capture.py b/backend/tests/test_mcp_artifact_capture.py new file mode 100644 index 0000000000..6968313a6d --- /dev/null +++ b/backend/tests/test_mcp_artifact_capture.py @@ -0,0 +1,114 @@ +"""MCP resource links must reach the persistent artifact registry through the graph.""" + +from contextlib import asynccontextmanager + +import pytest +from _agent_e2e_helpers import build_single_tool_call_model +from langchain.agents import create_agent +from langchain_core.messages import ToolMessage +from langchain_mcp_adapters.tools import load_mcp_tools +from mcp.types import CallToolResult, ListToolsResult, ResourceLink, Tool + +from deerflow.agents.middlewares.artifact_capture_middleware import ArtifactCaptureMiddleware +from deerflow.agents.thread_state import ThreadState +from deerflow.config.extensions_config import ExtensionsConfig +from deerflow.config.paths import Paths +from deerflow.mcp.tools import _RawServerDiscovery, _wrap_discovered_server_tools + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "transport,mime,suffix", + [ + ("stdio", "application/pdf", "pdf"), + ("stdio", "image/png", "png"), + ("streamable_http", "image/png", "png"), + ], +) +async def test_resource_link_captured_by_agent_graph(transport, mime, suffix, monkeypatch, tmp_path): + url = f"https://example.com/report.{suffix}" + calls = [] + + class Session: + async def initialize(self): + pass + + async def list_tools(self, **kwargs): + return ListToolsResult( + tools=[ + Tool( + name="export", + description="Export a report", + inputSchema={"type": "object", "properties": {}}, + ) + ] + ) + + async def call_tool(self, name, arguments, **kwargs): + calls.append((name, arguments)) + return CallToolResult( + content=[ + ResourceLink( + type="resource_link", + name="report", + uri=url, + mimeType=mime, + ) + ] + ) + + @asynccontextmanager + async def fake_session(*args, **kwargs): + yield Session() + + class Pool: + async def get_session(self, *args, **kwargs): + return Session() + + # Replace only the external MCP session and the filesystem base directory. + # Discovery conversion, transport dispatch, tool execution, and capture are real. + monkeypatch.setattr("langchain_mcp_adapters.tools.create_session", fake_session) + monkeypatch.setattr("deerflow.mcp.tools.get_paths", lambda: Paths(tmp_path)) + connection = {"transport": "stdio", "command": "offline-mcp-fixture"} if transport == "stdio" else {"transport": "streamable_http", "url": "https://example.com/mcp"} + discovered = await load_mcp_tools( + None, + connection=connection, + server_name="reports", + tool_name_prefix=True, + ) + wrapped = _wrap_discovered_server_tools( + ExtensionsConfig(), + "reports", + connection, + _RawServerDiscovery(discovered, []), + tool_name_prefix=True, + session_init_timeout=None, + pool=Pool(), + binding=None, + ownership_domain="deployment", + connection_scope="deployment", + ) + model = build_single_tool_call_model(tool_name="reports_export", tool_args={}, tool_call_id="export-1") + graph = create_agent( + model=model, + tools=list(wrapped.tools), + middleware=[ArtifactCaptureMiddleware()], + state_schema=ThreadState, + context_schema=dict, + ) + state = await graph.ainvoke( + {"messages": [("user", "Export a report")]}, + context={"thread_id": "report-thread", "thread_incarnation": "fixture-incarnation"}, + ) + (message,) = [m for m in state["messages"] if isinstance(m, ToolMessage)] + assert calls == [("export", {})] + assert message.status == "success" + if mime.startswith("image/"): + assert message.content[0]["type"] == "image" + assert message.content[0]["url"] == url + entries = state.get("tool_artifacts", []) + (entry,) = entries + assert entry["real_ref"] == url + assert entry["artifact_type"] == ("image" if mime.startswith("image/") else "file") + assert entry["mime_type"] == mime + assert entry["tool_name"] == "reports_export"