Ryker_Feng fa496c0c8d
feat(browser): add agentic browser control (#4187)
* feat(browser): add agentic browser control

* fix(frontend): format browser view changes

* fix(browser): keep browser optional and isolate sidecar layout

* fix(browser): address PR review security and IME findings

- Nginx: add a browser-stream WebSocket location before the generic
  /api/threads regex so Live upgrades instead of downgrading to HTTP
  (both nginx.conf and nginx.local.conf).
- Ownership: require an existing owned thread for the WS stream and REST
  navigate, and tear down the browser session on thread deletion so a
  later caller cannot reuse a retained page/cookies by guessing the id.
- SSRF: enforce the URL policy at the browser request boundary via a
  context-level route guard covering redirects, popups, iframes, and
  subresources (skipped for CDP-attached Chrome).
- IME: skip key forwarding while a composition is active so confirming a
  CJK candidate with Enter no longer submits the remote page form.

Adds regression tests for the request guard, session teardown on delete,
and the composing-Enter key decision.

* fix(frontend): smooth streaming in long tool threads

* Revert "fix(frontend): smooth streaming in long tool threads"

This reverts commit f0462516eabe77f138d4027ea1c714fb226683cf.

* fix(browser): address review security and lifecycle findings

- Reject cross-origin WebSocket upgrades on the live browser stream
  (Origin allow-list reuse of CORS/same-origin helpers) to close a
  WS-CSRF hole, and fail closed when the ownership store is absent.
- Warn when a CDP-attached session runs with the SSRF request guard
  off, and drop the unreachable CDP screencast teardown dead code.
- Read browser session launch config from a single canonical source
  (browser_navigate) so it is deterministic regardless of call order.
- Bound per-thread Chromium accumulation with idle-timeout eviction
  and an LRU max-sessions cap.
- Reset the Live reconnect counter on a successful open so the stream
  can't permanently stall after the cumulative attempt cap.

* fix(frontend): reduce long tool thread render stalls

Reuse stable historical message groups during streaming, defer heavy Markdown and browser previews, and lazy-decode message images.

* fix(browser): keep live control responsive during continuous input

Why: Manual browser control felt laggy — a physical click ran the remote
Playwright click three times and each non-move input synchronously awaited a
JPEG screenshot, so events queued behind capture (queue wait up to ~237ms).
The first async attempt used a trailing-edge debounce, which froze the visible
page until a wheel/keyboard gesture stopped ("scroll finishes, then it jumps").

What:
- Frontend forwards one `click` per physical click instead of also emitting
  `down`/`up`, so the remote page is not clicked twice per gesture.
- Backend detaches live-frame capture from input dispatch: non-move actions
  start a rate-limited background refresh loop (leading frame + bounded cadence)
  that keeps emitting frames while input continues and never blocks dispatch.
- Add regression tests: input dispatch no longer awaits the screenshot, rapid
  inputs coalesce, and continuous input keeps refreshing before it stops.

Scenarios: Verified in the live Browser panel — a single click completes in
~57ms (was blocked behind a 171ms capture), and a 1.14s sustained wheel gesture
renders ~7 frames throughout the scroll instead of one frame after it ends.

* fix(browser): harden worker and session lifecycle

* fix(browser): address latest review feedback

* fix(frontend): preserve optimistic new-chat message

* test(e2e): preserve mocked message run ids

* fix(browser): address capability review feedback

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
2026-07-21 11:46:33 +08:00

186 lines
5.0 KiB
TypeScript

import type { Message } from "@langchain/langgraph-sdk";
import { describe, expect, it, rs } from "@rstest/core";
import { createElement, type ComponentProps } from "react";
import { renderToStaticMarkup } from "react-dom/server";
import { MessageGroup } from "@/components/workspace/messages/message-group";
import { I18nContext } from "@/core/i18n/context";
rs.mock("@/components/workspace/artifacts", () => ({
useArtifacts: () => ({
artifacts: [],
setArtifacts: () => undefined,
selectedArtifact: null,
autoSelect: false,
select: () => undefined,
deselect: () => undefined,
open: false,
autoOpen: false,
setOpen: () => undefined,
}),
}));
describe("MessageGroup", () => {
it("renders assistant text attached to a tool-calling processing message", () => {
const html = renderGroup([
{
id: "ai-1",
type: "ai",
content: "The browser action failed, so I will try another approach.",
tool_calls: [
{
id: "call-1",
name: "web_search",
args: { query: "DeerFlow issue 4027" },
},
],
} as Message,
]);
expect(html).toContain(
"The browser action failed, so I will try another approach.",
);
expect(html).toContain("DeerFlow issue 4027");
});
it("keeps assistant text visible while older tool steps stay collapsed", () => {
const html = renderGroup([
{
id: "ai-1",
type: "ai",
content: "The first tool failed; I will try a narrower search.",
tool_calls: [
{
id: "call-1",
name: "web_search",
args: { query: "first hidden query" },
},
],
} as Message,
{
id: "tool-1",
type: "tool",
name: "web_search",
tool_call_id: "call-1",
content: "[]",
} as Message,
{
id: "ai-2",
type: "ai",
content: "The second approach should reveal the missing context.",
tool_calls: [
{
id: "call-2",
name: "bash",
args: {
description: "Inspect message rendering",
command: "rg assistantText frontend/src",
},
},
],
} as Message,
]);
expect(html).toContain(
"The first tool failed; I will try a narrower search.",
);
expect(html).toContain(
"The second approach should reveal the missing context.",
);
expect(html).not.toContain("first hidden query");
expect(html).toContain("Inspect message rendering");
expect(html).toContain("1 more step");
});
it("keeps tool-calling assistant text visible when reasoning is also present", () => {
const html = renderGroup([
{
id: "ai-1",
type: "ai",
content: "I found a likely cause, so I will inspect the renderer next.",
additional_kwargs: {
reasoning_content: "Check how processing groups convert messages.",
},
tool_calls: [
{
id: "call-1",
name: "bash",
args: {
description: "Inspect renderer conversion",
command: "sed -n '720,780p' message-group.tsx",
},
},
],
} as Message,
]);
expect(html).toContain(
"I found a likely cause, so I will inspect the renderer next.",
);
expect(html).toContain("Inspect renderer conversion");
expect(html).toContain("1 more step");
expect(html).not.toContain("Check how processing groups convert messages.");
});
it("defers browser screenshot previews while the thread is loading", () => {
const messages = [
{
id: "ai-1",
type: "ai",
content: "",
tool_calls: [
{
id: "call-1",
name: "browser_navigate",
args: { url: "https://github.com/bytedance/deer-flow" },
},
],
} as Message,
{
id: "tool-1",
type: "tool",
name: "browser_navigate",
tool_call_id: "call-1",
content: "Opened",
additional_kwargs: {
browser_view: {
screenshot: "/mnt/user-data/outputs/browser.png",
url: "https://github.com/bytedance/deer-flow",
},
},
} as Message,
];
const visibleHtml = renderGroup(messages, {
threadId: "thread-1",
deferBrowserPreviews: false,
});
const deferredHtml = renderGroup(messages, {
threadId: "thread-1",
deferBrowserPreviews: true,
});
expect(visibleHtml).toContain("<img");
expect(visibleHtml).toContain('decoding="async"');
expect(deferredHtml).not.toContain("<img");
});
});
function renderGroup(
messages: Message[],
props: Omit<ComponentProps<typeof MessageGroup>, "messages"> = {},
) {
return renderToStaticMarkup(
createElement(
I18nContext.Provider,
{
value: {
locale: "en-US",
setLocale: () => undefined,
},
},
createElement(MessageGroup, { ...props, messages }),
),
);
}