From b5d2a504a99b692842fac53e27e3536993a337be Mon Sep 17 00:00:00 2001 From: Wu JiaCheng <134792033+patrick-andstar@users.noreply.github.com> Date: Sun, 26 Jul 2026 08:32:04 +0800 Subject: [PATCH] perf(messages): index tool call results (#4411) --- frontend/AGENTS.md | 2 + .../workspace/messages/message-group.tsx | 38 +++-- .../workspace/messages/message-group.test.ts | 154 ++++++++++++++++++ 3 files changed, 182 insertions(+), 12 deletions(-) diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md index feaf46c79..988abfcb8 100644 --- a/frontend/AGENTS.md +++ b/frontend/AGENTS.md @@ -85,6 +85,8 @@ Human input requests are a structured message protocol layered on normal chat hi Tool-calling AI messages can contain user-visible text as well as `tool_calls`. `core/messages/utils.ts` keeps these turns in an `assistant:processing` group, and `components/workspace/messages/message-group.tsx` must render the visible text as a processing step instead of treating the message as only tool metadata. This preserves provider text such as error explanations or "trying another approach" notes during tool-heavy runs. +`MessageGroup` builds its tool-result and browser-preview lookups once per processing group before converting messages to steps. The lookup preserves the first non-empty result and first screenshot-bearing browser view for each tool-call ID, matching the streamed-message display semantics without repeatedly scanning the full group for every tool call. + ### Key Patterns - **Server Components by default**, `"use client"` only for interactive components diff --git a/frontend/src/components/workspace/messages/message-group.tsx b/frontend/src/components/workspace/messages/message-group.tsx index a12096c00..82e647534 100644 --- a/frontend/src/components/workspace/messages/message-group.tsx +++ b/frontend/src/components/workspace/messages/message-group.tsx @@ -36,7 +36,7 @@ import type { TokenDebugStep } from "@/core/messages/usage-model"; import { extractContentFromMessage, extractReasoningContentFromMessage, - findToolCallResult, + extractTextFromMessage, } from "@/core/messages/utils"; import { extractTitleFromMarkdown } from "@/core/utils/markdown"; import { env } from "@/env"; @@ -895,27 +895,41 @@ interface BrowserViewMeta { title?: string; } -function findBrowserViewMeta( - toolCallId: string, - messages: Message[], -): BrowserViewMeta | undefined { +function indexToolCallData(messages: Message[]) { + const toolCallResults = new Map(); + const browserViews = new Map(); + for (const message of messages) { - if (message.type === "tool" && message.tool_call_id === toolCallId) { - const meta = ( + if (message.type !== "tool" || !message.tool_call_id) { + continue; + } + + const toolCallId = message.tool_call_id; + if (!toolCallResults.has(toolCallId)) { + const result = extractTextFromMessage(message); + if (result) { + toolCallResults.set(toolCallId, result); + } + } + + if (!browserViews.has(toolCallId)) { + const browserView = ( message.additional_kwargs as | { browser_view?: BrowserViewMeta } | undefined )?.browser_view; - if (meta && typeof meta.screenshot === "string") { - return meta; + if (browserView && typeof browserView.screenshot === "string") { + browserViews.set(toolCallId, browserView); } } } - return undefined; + + return { browserViews, toolCallResults }; } function convertToSteps(messages: Message[]): CoTStep[] { const steps: CoTStep[] = []; + const { browserViews, toolCallResults } = indexToolCallData(messages); for (const [messageIndex, message] of messages.entries()) { if (message.type === "ai") { const content = extractContentFromMessage(message); @@ -950,7 +964,7 @@ function convertToSteps(messages: Message[]): CoTStep[] { }; const toolCallId = tool_call.id; if (toolCallId) { - const toolCallResult = findToolCallResult(toolCallId, messages); + const toolCallResult = toolCallResults.get(toolCallId); if (toolCallResult) { try { const json = JSON.parse(toolCallResult); @@ -959,7 +973,7 @@ function convertToSteps(messages: Message[]): CoTStep[] { step.result = toolCallResult; } } - step.browserView = findBrowserViewMeta(toolCallId, messages); + step.browserView = browserViews.get(toolCallId); } steps.push(step); } diff --git a/frontend/tests/unit/components/workspace/messages/message-group.test.ts b/frontend/tests/unit/components/workspace/messages/message-group.test.ts index ad2eec510..3971f2d82 100644 --- a/frontend/tests/unit/components/workspace/messages/message-group.test.ts +++ b/frontend/tests/unit/components/workspace/messages/message-group.test.ts @@ -204,6 +204,160 @@ describe("MessageGroup", () => { expect(visibleHtml).toContain('decoding="async"'); expect(deferredHtml).not.toContain(" { + const html = renderGroup([ + { + id: "ai-1", + type: "ai", + content: "", + tool_calls: [ + { + id: "call-fetch", + name: "web_fetch", + args: { url: "https://example.com" }, + }, + { + id: "call-task", + name: "task", + args: { description: "Do not render this subagent call" }, + }, + ], + } as Message, + { + id: "tool-empty", + type: "tool", + name: "web_fetch", + tool_call_id: "call-fetch", + content: "", + } as Message, + { + id: "tool-first", + type: "tool", + name: "web_fetch", + tool_call_id: "call-fetch", + content: "# First fetched title\n\nFirst result.", + } as Message, + { + id: "tool-later", + type: "tool", + name: "web_fetch", + tool_call_id: "call-fetch", + content: "# Later fetched title\n\nLater result.", + } as Message, + ]); + + expect(html).toContain("First fetched title"); + expect(html).not.toContain("Later fetched title"); + expect(html).not.toContain("Do not render this subagent call"); + }); + + it("keeps the first browser view that includes a screenshot", () => { + const html = renderGroup( + [ + { + id: "ai-1", + type: "ai", + content: "", + tool_calls: [ + { + id: "call-browser", + name: "browser_navigate", + args: { url: "https://example.com" }, + }, + ], + } as Message, + { + id: "tool-without-shot", + type: "tool", + name: "browser_navigate", + tool_call_id: "call-browser", + content: "Opened without a preview.", + additional_kwargs: { + browser_view: { url: "https://example.com" }, + }, + } as Message, + { + id: "tool-first-shot", + type: "tool", + name: "browser_navigate", + tool_call_id: "call-browser", + content: "Opened with the first preview.", + additional_kwargs: { + browser_view: { + screenshot: "/mnt/user-data/outputs/first-browser.png", + url: "https://example.com/first", + }, + }, + } as Message, + { + id: "tool-later-shot", + type: "tool", + name: "browser_navigate", + tool_call_id: "call-browser", + content: "Opened with a later preview.", + additional_kwargs: { + browser_view: { + screenshot: "/mnt/user-data/outputs/later-browser.png", + url: "https://example.com/later", + }, + }, + } as Message, + ], + { threadId: "thread-1" }, + ); + + expect(html).toContain("first-browser.png"); + expect(html).not.toContain("later-browser.png"); + expect(html).toContain("https://example.com/first"); + }); + + it("renders the earliest JSON tool result after an empty streamed update", () => { + const html = renderGroup([ + { + id: "ai-1", + type: "ai", + content: "", + tool_calls: [ + { + id: "call-search", + name: "web_search", + args: { query: "DeerFlow" }, + }, + ], + } as Message, + { + id: "tool-empty", + type: "tool", + name: "web_search", + tool_call_id: "call-search", + content: "", + } as Message, + { + id: "tool-first", + type: "tool", + name: "web_search", + tool_call_id: "call-search", + content: JSON.stringify([ + { title: "First source", url: "https://first.example" }, + ]), + } as Message, + { + id: "tool-later", + type: "tool", + name: "web_search", + tool_call_id: "call-search", + content: JSON.stringify([ + { title: "Later source", url: "https://later.example" }, + ]), + } as Message, + ]); + + expect(html).toContain("First source"); + expect(html).toContain("https://first.example"); + expect(html).not.toContain("Later source"); + expect(html).not.toContain("https://later.example"); + }); }); function renderGroup(