diff --git a/backend/packages/harness/deerflow/agents/lead_agent/prompt.py b/backend/packages/harness/deerflow/agents/lead_agent/prompt.py index 2bb15cfc6..e4ece90d0 100644 --- a/backend/packages/harness/deerflow/agents/lead_agent/prompt.py +++ b/backend/packages/harness/deerflow/agents/lead_agent/prompt.py @@ -647,7 +647,11 @@ combined with a FastAPI gateway for REST API access [citation:FastAPI](https://f keeps each tool call small and avoids mid-stream chunk-gap timeouts on oversized single-shot writes. (See issue #3189.) - Clarity: Be direct and helpful, avoid unnecessary meta-commentary -- Including Images and Mermaid: Images and Mermaid diagrams are always welcomed in the Markdown format, and you're encouraged to use `![Image Description](image_path)\n\n` or "```mermaid" to display images in response or Markdown files +- Including Images and Mermaid: Images and Mermaid diagrams are welcomed in Markdown. + - To render an output image in a final response, use its complete virtual artifact path, for example `![Chart](/mnt/user-data/outputs/chart.png)`. + - Never use a bare or workspace-relative filename. + - Call `present_files` for the image before referencing it. + - Use "```mermaid" for Mermaid diagrams. - Multi-task: Better utilize parallel tool calling to call multiple tools at one time for better performance - Language Consistency: Keep using the same language as user's - Always Respond: Your thinking is internal. You MUST always provide a visible response to the user after thinking. diff --git a/backend/tests/test_lead_agent_prompt.py b/backend/tests/test_lead_agent_prompt.py index 65db7cf0f..60137f353 100644 --- a/backend/tests/test_lead_agent_prompt.py +++ b/backend/tests/test_lead_agent_prompt.py @@ -433,6 +433,14 @@ def test_system_prompt_template_contains_file_editing_workflow_rule(): assert "append=True" in template +def test_system_prompt_template_requires_virtual_paths_for_output_images(): + template = prompt_module.SYSTEM_PROMPT_TEMPLATE + + assert "![Chart](/mnt/user-data/outputs/chart.png)" in template + assert "Never use a bare or workspace-relative filename" in template + assert "Call `present_files` for the image before referencing it" in template + + def test_system_prompt_template_preserves_placeholders(): """Ensure the chunking-rule edit didn't drop any f-string placeholder consumed by apply_prompt_template(). A missing placeholder would diff --git a/frontend/src/components/workspace/messages/message-list-item.tsx b/frontend/src/components/workspace/messages/message-list-item.tsx index d97645ea9..a42680b64 100644 --- a/frontend/src/components/workspace/messages/message-list-item.tsx +++ b/frontend/src/components/workspace/messages/message-list-item.tsx @@ -31,7 +31,10 @@ import { upsertFeedback, type FeedbackData, } from "@/core/api/feedback"; -import { resolveArtifactURL } from "@/core/artifacts/utils"; +import { + resolveArtifactURL, + resolveMessageImageURL, +} from "@/core/artifacts/utils"; import { extractCitationSources } from "@/core/citations/sources"; import { useI18n } from "@/core/i18n/hooks"; import { @@ -135,6 +138,7 @@ export function MessageListItem({ feedback, runId, threadId, + artifactPaths = [], showCopyButton = true, turnStartTime, }: { @@ -142,6 +146,7 @@ export function MessageListItem({ message: Message; isLoading?: boolean; threadId: string; + artifactPaths?: readonly string[]; feedback?: FeedbackData | null; runId?: string; showCopyButton?: boolean; @@ -158,6 +163,7 @@ export function MessageListItem({ message={message} isLoading={isLoading} threadId={threadId} + artifactPaths={artifactPaths} runId={runId} turnStartTime={turnStartTime} /> @@ -193,10 +199,12 @@ function MessageImage({ src, alt, threadId, + artifactPaths, maxWidth = "90%", ...props }: React.ImgHTMLAttributes & { threadId: string; + artifactPaths: readonly string[]; maxWidth?: string; }) { if (!src) return null; @@ -207,7 +215,7 @@ function MessageImage({ return {alt}; } - const url = src.startsWith("/mnt/") ? resolveArtifactURL(src, threadId) : src; + const url = resolveMessageImageURL(src, threadId, artifactPaths); return ( @@ -259,6 +267,7 @@ function MessageContent_({ message, isLoading = false, threadId, + artifactPaths, runId, turnStartTime, }: { @@ -266,6 +275,7 @@ function MessageContent_({ message: Message; isLoading?: boolean; threadId: string; + artifactPaths: readonly string[]; runId?: string; turnStartTime?: number | null; }) { @@ -317,11 +327,16 @@ function MessageContent_({ const components = useMemo( () => ({ img: (props: ImgHTMLAttributes) => ( - + ), a: createMarkdownLinkComponent(threadId), }), - [threadId], + [artifactPaths, threadId], ); const rawContent = extractContentFromMessage(message); diff --git a/frontend/src/components/workspace/messages/message-list.tsx b/frontend/src/components/workspace/messages/message-list.tsx index 21fc2abb4..83e783cbb 100644 --- a/frontend/src/components/workspace/messages/message-list.tsx +++ b/frontend/src/components/workspace/messages/message-list.tsx @@ -28,6 +28,7 @@ import { ReasoningTrigger, } from "@/components/ai-elements/reasoning"; import { Button } from "@/components/ui/button"; +import { extractArtifactsFromThread } from "@/core/artifacts/utils"; import { useI18n } from "@/core/i18n/hooks"; import { deriveHumanInputThreadState, @@ -730,6 +731,7 @@ export function MessageList({ groupIndex === groupedMessages.length - 1 } threadId={threadId} + artifactPaths={extractArtifactsFromThread(thread)} runId={ group.type === "assistant" ? (msg as { run_id?: string }).run_id diff --git a/frontend/src/core/artifacts/utils.ts b/frontend/src/core/artifacts/utils.ts index e205b739a..e674a053d 100644 --- a/frontend/src/core/artifacts/utils.ts +++ b/frontend/src/core/artifacts/utils.ts @@ -1,6 +1,6 @@ import { getBackendBaseURL } from "../config"; import { isStaticWebsiteOnly } from "../static-mode"; -import type { AgentThread } from "../threads"; +import type { AgentThreadState } from "../threads"; export function urlOfArtifact({ filepath, @@ -22,7 +22,9 @@ export function urlOfArtifact({ return `${getBackendBaseURL()}/api/threads/${threadId}/artifacts${filepath}${download ? "?download=true" : ""}`; } -export function extractArtifactsFromThread(thread: AgentThread) { +export function extractArtifactsFromThread(thread: { + values: Pick; +}) { return thread.values.artifacts ?? []; } @@ -33,6 +35,37 @@ export function resolveArtifactURL(absolutePath: string, threadId: string) { return `${getBackendBaseURL()}/api/threads/${threadId}/artifacts${absolutePath}`; } +export function resolveMessageImageURL( + src: string, + threadId: string, + artifactPaths: readonly string[], +) { + if (src.startsWith("/mnt/")) { + return resolveArtifactURL(src, threadId); + } + + const [relativePath = ""] = src.split(/[?#]/, 1); + const normalizedPath = relativePath.replace(/^(?:\.\/)+/, ""); + if ( + !normalizedPath || + normalizedPath.startsWith("/") || + /^[a-z][a-z\d+.-]*:/i.test(normalizedPath) || + normalizedPath.startsWith("//") || + normalizedPath.split("/").includes("..") + ) { + return src; + } + + const matches = artifactPaths.filter((path) => + path.endsWith(`/${normalizedPath}`), + ); + if (matches.length !== 1) { + return src; + } + + return `${resolveArtifactURL(matches[0]!, threadId)}${src.slice(relativePath.length)}`; +} + function staticDemoArtifactURL({ filepath, threadId, diff --git a/frontend/tests/unit/core/artifacts/utils.test.ts b/frontend/tests/unit/core/artifacts/utils.test.ts index 23a45d70d..091f652f2 100644 --- a/frontend/tests/unit/core/artifacts/utils.test.ts +++ b/frontend/tests/unit/core/artifacts/utils.test.ts @@ -73,4 +73,51 @@ describe("artifact URL helpers", () => { resolveArtifactURL("/mnt/user-data/outputs/style.css", "thread-1"), ).toBe("/demo/threads/thread-1/user-data/outputs/style.css"); }); + + test("resolves a relative message image when it uniquely matches an artifact", async () => { + const { resolveMessageImageURL } = await loadFreshArtifactUtils(); + const artifacts = [ + "/mnt/user-data/outputs/aws-agent-overview.png", + "/mnt/user-data/outputs/aws-agent-console-config.png", + ]; + + expect( + resolveMessageImageURL("aws-agent-overview.png", "thread-1", artifacts), + ).toBe( + "/api/threads/thread-1/artifacts/mnt/user-data/outputs/aws-agent-overview.png", + ); + expect( + resolveMessageImageURL( + "./aws-agent-overview.png#detail", + "thread-1", + artifacts, + ), + ).toBe( + "/api/threads/thread-1/artifacts/mnt/user-data/outputs/aws-agent-overview.png#detail", + ); + }); + + test("does not rewrite unregistered, ambiguous, or external message images", async () => { + const { resolveMessageImageURL } = await loadFreshArtifactUtils(); + + expect(resolveMessageImageURL("missing.png", "thread-1", [])).toBe( + "missing.png", + ); + expect( + resolveMessageImageURL("shared.png", "thread-1", [ + "/mnt/user-data/outputs/first/shared.png", + "/mnt/user-data/outputs/second/shared.png", + ]), + ).toBe("shared.png"); + expect( + resolveMessageImageURL("../etc/secret.png", "thread-1", [ + "/mnt/user-data/outputs/secret.png", + ]), + ).toBe("../etc/secret.png"); + expect( + resolveMessageImageURL("https://example.com/image.png", "thread-1", [ + "/mnt/user-data/outputs/image.png", + ]), + ).toBe("https://example.com/image.png"); + }); });