From e5c62cab5acd9b8bf9e33533e493f17a7180db76 Mon Sep 17 00:00:00 2001 From: Aari Date: Fri, 7 Aug 2026 21:38:27 +0800 Subject: [PATCH] feat(frontend): add Browser Live to Custom Agent chats (#4719) * feat(frontend): add Browser Live to Custom Agent chats * test(frontend): cover mock Custom Agent Browser Live --- README.md | 2 + frontend/AGENTS.md | 2 +- .../[agent_name]/chats/[thread_id]/layout.tsx | 12 +- .../[agent_name]/chats/[thread_id]/page.tsx | 11 +- frontend/tests/e2e/agent-chat.spec.ts | 117 ++++++++++++++++++ frontend/tests/e2e/utils/mock-api.ts | 1 + 6 files changed, 133 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 75ef0b8e6..0c1fc8dac 100644 --- a/README.md +++ b/README.md @@ -1009,6 +1009,8 @@ uv run playwright install chromium Then uncomment the `group: browser` tool entries in `config.yaml` (`browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_get_text`, `browser_back`, `browser_screenshot`, `browser_close`). `make dev` / Docker startup detects an enabled `browser_navigate` tool and preserves the `browser` extra on dependency syncs. The Gateway fails startup if browser control is configured but Playwright is missing, and `/api/features` hides the Browser UI unless the backend can actually serve it. Keep `headless: true` and `allow_private_addresses: false` for anything but local, trusted debugging. Attaching to an existing Chrome with `cdp_url` cannot enforce DeerFlow's subresource/redirect SSRF guard and therefore fails closed unless `allow_unguarded_cdp: true` explicitly acknowledges that risk; use it only with a trusted local browser. Browser sessions are process-local; keep `GATEWAY_WORKERS=1` while this tool group is enabled because ordinary uvicorn worker dispatch does not provide thread affinity. +Existing, non-mock Custom Agent chats expose the same Browser Live controls when browser control is available and the agent either leaves `tool_groups` unrestricted or includes the `browser` group. An explicit tool-group allowlist without `browser` keeps those controls hidden. + The workspace Browser Live client negotiates binary JPEG WebSocket frames, keeps only the newest pending frame per display refresh, and revokes replaced object URLs. Gateway control messages remain JSON, and clients that do not diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md index c71ad7430..31caf907e 100644 --- a/frontend/AGENTS.md +++ b/frontend/AGENTS.md @@ -135,7 +135,7 @@ Edit-and-rerun is deliberately latest-turn-only. `core/messages/utils.ts::getLat - `src/app/workspace/chats/[thread_id]/page.tsx` owns composer busy-state wiring. - `src/app/workspace/chats/[thread_id]/page.tsx` owns branch-from-turn submission and navigation; sidecar `MessageList` instances do not receive the branch action. - `src/app/workspace/chats/[thread_id]/page.tsx` and `src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx` own edit-and-rerun submission wiring because the page must preserve normal/custom-agent run context; `MessageList` only detects the latest editable user turn and renders the inline editor. -- `src/app/workspace/chats/[thread_id]/page.tsx` gates the Workspace Browser trigger and browser right panel on `/api/features -> browser_control.enabled`; default/failed feature discovery hides the browser control so optional backend installs do not show a dead Live socket. +- `src/app/workspace/chats/[thread_id]/page.tsx` gates the Workspace Browser trigger and browser right panel on `/api/features -> browser_control.enabled`; `src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx` applies the same capability gate and additionally requires the Custom Agent's tool groups to be unrestricted or include `browser`. Default/failed feature discovery hides the browser control so optional backend installs do not show a dead Live socket. - `src/app/workspace/chats/[thread_id]/page.tsx` and `src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx` own active-goal display state for their composer overlays. - `src/components/workspace/messages/message-list.tsx` owns human-input card answered/latest/pending gating; entry pages only translate a submitted card response into `sendMessage` calls. - `src/components/workspace/browser-view/browser-view-panel.tsx` forwards each physical pointer click as one `click` input; do not also emit `down`/`up` for the same gesture because the remote Playwright click would run twice. diff --git a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/layout.tsx b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/layout.tsx index 68f51b607..811be80f0 100644 --- a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/layout.tsx +++ b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/layout.tsx @@ -1,19 +1,11 @@ "use client"; -import { PromptInputProvider } from "@/components/ai-elements/prompt-input"; -import { ArtifactsProvider } from "@/components/workspace/artifacts"; -import { SubtasksProvider } from "@/core/tasks/context"; +import { ChatProviders } from "@/components/workspace/chats/chat-providers"; export default function AgentChatLayout({ children, }: { children: React.ReactNode; }) { - return ( - - - {children} - - - ); + return {children}; } diff --git a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx index e3f6c80ba..4070d40d1 100644 --- a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx +++ b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx @@ -9,6 +9,7 @@ import { Button } from "@/components/ui/button"; import { SidebarTrigger } from "@/components/ui/sidebar"; import { AgentWelcome } from "@/components/workspace/agent-welcome"; import { ArtifactTrigger } from "@/components/workspace/artifacts"; +import { BrowserTrigger } from "@/components/workspace/browser-view"; import { ChatBox, useThreadChat } from "@/components/workspace/chats"; import { ContextUsageBadge } from "@/components/workspace/context-usage-badge"; import { ExportTrigger } from "@/components/workspace/export-trigger"; @@ -32,6 +33,7 @@ import { TokenUsageIndicator } from "@/components/workspace/token-usage-indicato import { Tooltip } from "@/components/workspace/tooltip"; import { useActiveGoal } from "@/components/workspace/use-active-goal"; import { useAgent } from "@/core/agents"; +import { useBrowserControlEnabled } from "@/core/features"; import { useI18n } from "@/core/i18n/hooks"; import { buildHumanInputResponseText, @@ -74,6 +76,7 @@ export default function AgentChatPage() { const [isWelcomeMode, setIsWelcomeMode] = useState(isNewThread); const [settings, setSettings] = useThreadSettings(threadId); const [localSettings, setLocalSettings] = useLocalSettings(); + const { enabled: browserControlEnabled } = useBrowserControlEnabled(); const { tokenUsageEnabled } = useModels(); const threadTokenUsage = useThreadTokenUsage( isNewThread || isMock ? undefined : threadId, @@ -225,6 +228,11 @@ export default function AgentChatPage() { ? localSettings.tokenUsage.inlineMode : "off"; const hasTodos = (thread.values.todos?.length ?? 0) > 0; + const agentBrowserEnabled = + agent !== null && + (agent.tool_groups == null || agent.tool_groups.includes("browser")); + const browserEnabled = + !isNewThread && !isMock && browserControlEnabled && agentBrowserEnabled; const { activeGoal, hasGoal, setLocalGoal } = useActiveGoal( threadId, thread.values.goal, @@ -245,7 +253,7 @@ export default function AgentChatPage() { context={{ ...settings.context, agent_name }} isMock={isMock} > - +
)} + {browserEnabled && }
diff --git a/frontend/tests/e2e/agent-chat.spec.ts b/frontend/tests/e2e/agent-chat.spec.ts index 4b3ffc811..5f680e05a 100644 --- a/frontend/tests/e2e/agent-chat.spec.ts +++ b/frontend/tests/e2e/agent-chat.spec.ts @@ -78,6 +78,123 @@ test.describe("Agent chat", () => { ).toBeVisible({ timeout: 15_000 }); }); + for (const { + name, + toolGroups, + browserControlEnabled, + expectedVisible, + mock, + } of [ + { + name: "shows Browser Live for an explicit browser tool group", + toolGroups: ["browser"], + browserControlEnabled: true, + expectedVisible: true, + }, + { + name: "shows Browser Live when tool groups are unrestricted", + toolGroups: null, + browserControlEnabled: true, + expectedVisible: true, + }, + { + name: "hides Browser Live without the browser tool group", + toolGroups: ["web"], + browserControlEnabled: true, + expectedVisible: false, + }, + { + name: "hides Browser Live when browser control is unavailable", + toolGroups: ["browser"], + browserControlEnabled: false, + expectedVisible: false, + }, + { + name: "hides Browser Live in mock custom-agent chats", + toolGroups: ["browser"], + browserControlEnabled: true, + expectedVisible: false, + mock: true, + }, + ]) { + test(name, async ({ page }) => { + const agent = { + name: "browser-agent", + description: "A custom agent for Browser Live tests", + tool_groups: toolGroups, + }; + mockLangGraphAPI(page, { + agents: [agent], + features: { browserControlEnabled }, + threads: [ + { + thread_id: MOCK_THREAD_ID, + title: "Browser agent conversation", + agent_name: agent.name, + messages: [ + { + type: "ai", + id: "msg-ai-browser-agent", + content: "Ready to browse", + }, + ], + }, + ], + }); + + const featuresLoaded = page.waitForResponse( + (response) => + new URL(response.url()).pathname === "/api/features" && + response.status() === 200, + ); + await page.goto( + `/workspace/agents/${agent.name}/chats/${MOCK_THREAD_ID}${mock ? "?mock=true" : ""}`, + ); + await featuresLoaded; + if (mock) { + await expect( + page.locator("header span", { hasText: agent.name }), + ).toBeVisible({ timeout: 15_000 }); + } else { + await expect(page.getByText("Ready to browse")).toBeVisible({ + timeout: 15_000, + }); + } + + const browserTrigger = page.getByTestId("browser-trigger"); + if (expectedVisible) { + await expect(browserTrigger).toBeVisible(); + await browserTrigger.click(); + await expect( + page.getByPlaceholder("Enter a URL and press Enter"), + ).toBeVisible(); + } else { + await expect(browserTrigger).toHaveCount(0); + } + }); + } + + test("hides Browser Live before a custom-agent thread is created", async ({ + page, + }) => { + mockLangGraphAPI(page, { + agents: [ + { + name: "browser-agent", + description: "A custom agent for Browser Live tests", + tool_groups: ["browser"], + }, + ], + features: { browserControlEnabled: true }, + }); + + await page.goto("/workspace/agents/browser-agent/chats/new"); + await expect(page.getByPlaceholder(/how can i assist you/i)).toBeVisible({ + timeout: 15_000, + }); + await expect(page.getByTestId("browser-trigger")).toHaveCount(0); + }); + test("agent chat can regenerate its latest response", async ({ page }) => { const humanMessage = { type: "human", diff --git a/frontend/tests/e2e/utils/mock-api.ts b/frontend/tests/e2e/utils/mock-api.ts index a2966b191..ad636487b 100644 --- a/frontend/tests/e2e/utils/mock-api.ts +++ b/frontend/tests/e2e/utils/mock-api.ts @@ -46,6 +46,7 @@ export type MockAgent = { name: string; description?: string; system_prompt?: string; + tool_groups?: string[] | null; }; export type MockSkill = {