mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-07-27 16:37:55 +00:00
fa496c0c8d
* feat(browser): add agentic browser control
* fix(frontend): format browser view changes
* fix(browser): keep browser optional and isolate sidecar layout
* fix(browser): address PR review security and IME findings
- Nginx: add a browser-stream WebSocket location before the generic
/api/threads regex so Live upgrades instead of downgrading to HTTP
(both nginx.conf and nginx.local.conf).
- Ownership: require an existing owned thread for the WS stream and REST
navigate, and tear down the browser session on thread deletion so a
later caller cannot reuse a retained page/cookies by guessing the id.
- SSRF: enforce the URL policy at the browser request boundary via a
context-level route guard covering redirects, popups, iframes, and
subresources (skipped for CDP-attached Chrome).
- IME: skip key forwarding while a composition is active so confirming a
CJK candidate with Enter no longer submits the remote page form.
Adds regression tests for the request guard, session teardown on delete,
and the composing-Enter key decision.
* fix(frontend): smooth streaming in long tool threads
* Revert "fix(frontend): smooth streaming in long tool threads"
This reverts commit f0462516eabe77f138d4027ea1c714fb226683cf.
* fix(browser): address review security and lifecycle findings
- Reject cross-origin WebSocket upgrades on the live browser stream
(Origin allow-list reuse of CORS/same-origin helpers) to close a
WS-CSRF hole, and fail closed when the ownership store is absent.
- Warn when a CDP-attached session runs with the SSRF request guard
off, and drop the unreachable CDP screencast teardown dead code.
- Read browser session launch config from a single canonical source
(browser_navigate) so it is deterministic regardless of call order.
- Bound per-thread Chromium accumulation with idle-timeout eviction
and an LRU max-sessions cap.
- Reset the Live reconnect counter on a successful open so the stream
can't permanently stall after the cumulative attempt cap.
* fix(frontend): reduce long tool thread render stalls
Reuse stable historical message groups during streaming, defer heavy Markdown and browser previews, and lazy-decode message images.
* fix(browser): keep live control responsive during continuous input
Why: Manual browser control felt laggy — a physical click ran the remote
Playwright click three times and each non-move input synchronously awaited a
JPEG screenshot, so events queued behind capture (queue wait up to ~237ms).
The first async attempt used a trailing-edge debounce, which froze the visible
page until a wheel/keyboard gesture stopped ("scroll finishes, then it jumps").
What:
- Frontend forwards one `click` per physical click instead of also emitting
`down`/`up`, so the remote page is not clicked twice per gesture.
- Backend detaches live-frame capture from input dispatch: non-move actions
start a rate-limited background refresh loop (leading frame + bounded cadence)
that keeps emitting frames while input continues and never blocks dispatch.
- Add regression tests: input dispatch no longer awaits the screenshot, rapid
inputs coalesce, and continuous input keeps refreshing before it stops.
Scenarios: Verified in the live Browser panel — a single click completes in
~57ms (was blocked behind a 171ms capture), and a 1.14s sustained wheel gesture
renders ~7 frames throughout the scroll instead of one frame after it ends.
* fix(browser): harden worker and session lifecycle
* fix(browser): address latest review feedback
* fix(frontend): preserve optimistic new-chat message
* test(e2e): preserve mocked message run ids
* fix(browser): address capability review feedback
---------
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
73 lines
2.7 KiB
Python
73 lines
2.7 KiB
Python
"""Shared browser-control capability checks for Gateway surfaces."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
from deerflow.community.browser_automation.session import browser_multi_worker_error
|
|
from deerflow.config.app_config import AppConfig
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BrowserCapability:
|
|
"""Frontend/API availability for the agentic browser control surface."""
|
|
|
|
configured: bool
|
|
available: bool
|
|
reason: str | None = None
|
|
|
|
|
|
def _tool_config(config: AppConfig) -> Any | None:
|
|
get_tool_config = getattr(config, "get_tool_config", None)
|
|
if callable(get_tool_config) and callable(getattr(type(config), "get_tool_config", None)):
|
|
return get_tool_config("browser_navigate")
|
|
return next(
|
|
(tool for tool in (getattr(config, "tools", None) or []) if getattr(tool, "name", None) == "browser_navigate"),
|
|
None,
|
|
)
|
|
|
|
|
|
def _tool_extra(tool_cfg: Any) -> dict[str, Any]:
|
|
extra = getattr(tool_cfg, "model_extra", None)
|
|
return extra if isinstance(extra, dict) else {}
|
|
|
|
|
|
def browser_capability(config: AppConfig) -> BrowserCapability:
|
|
"""Return whether browser control can actually serve frontend requests."""
|
|
|
|
tool_cfg = _tool_config(config)
|
|
if tool_cfg is None:
|
|
return BrowserCapability(configured=False, available=False, reason="browser_navigate is not configured")
|
|
|
|
worker_error = browser_multi_worker_error()
|
|
if worker_error is not None:
|
|
return BrowserCapability(configured=True, available=False, reason=worker_error)
|
|
|
|
extra = _tool_extra(tool_cfg)
|
|
cdp_url = extra.get("cdp_url")
|
|
if isinstance(cdp_url, str) and cdp_url.strip() and extra.get("allow_unguarded_cdp") is not True:
|
|
return BrowserCapability(
|
|
configured=True,
|
|
available=False,
|
|
reason="cdp_url requires allow_unguarded_cdp: true because DeerFlow cannot enforce the SSRF request guard on a CDP-attached browser",
|
|
)
|
|
|
|
if importlib.util.find_spec("playwright") is None or importlib.util.find_spec("playwright.async_api") is None:
|
|
return BrowserCapability(
|
|
configured=True,
|
|
available=False,
|
|
reason="Playwright is not installed; install the backend browser extra and run `playwright install chromium`",
|
|
)
|
|
|
|
return BrowserCapability(configured=True, available=True)
|
|
|
|
|
|
def ensure_browser_runtime_available(config: AppConfig) -> None:
|
|
"""Fail startup when browser control is configured but cannot run."""
|
|
|
|
capability = browser_capability(config)
|
|
if capability.configured and not capability.available:
|
|
raise RuntimeError(capability.reason or "Browser automation is not available")
|