mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-07-26 16:07:53 +00:00
fa496c0c8d
* feat(browser): add agentic browser control
* fix(frontend): format browser view changes
* fix(browser): keep browser optional and isolate sidecar layout
* fix(browser): address PR review security and IME findings
- Nginx: add a browser-stream WebSocket location before the generic
/api/threads regex so Live upgrades instead of downgrading to HTTP
(both nginx.conf and nginx.local.conf).
- Ownership: require an existing owned thread for the WS stream and REST
navigate, and tear down the browser session on thread deletion so a
later caller cannot reuse a retained page/cookies by guessing the id.
- SSRF: enforce the URL policy at the browser request boundary via a
context-level route guard covering redirects, popups, iframes, and
subresources (skipped for CDP-attached Chrome).
- IME: skip key forwarding while a composition is active so confirming a
CJK candidate with Enter no longer submits the remote page form.
Adds regression tests for the request guard, session teardown on delete,
and the composing-Enter key decision.
* fix(frontend): smooth streaming in long tool threads
* Revert "fix(frontend): smooth streaming in long tool threads"
This reverts commit f0462516eabe77f138d4027ea1c714fb226683cf.
* fix(browser): address review security and lifecycle findings
- Reject cross-origin WebSocket upgrades on the live browser stream
(Origin allow-list reuse of CORS/same-origin helpers) to close a
WS-CSRF hole, and fail closed when the ownership store is absent.
- Warn when a CDP-attached session runs with the SSRF request guard
off, and drop the unreachable CDP screencast teardown dead code.
- Read browser session launch config from a single canonical source
(browser_navigate) so it is deterministic regardless of call order.
- Bound per-thread Chromium accumulation with idle-timeout eviction
and an LRU max-sessions cap.
- Reset the Live reconnect counter on a successful open so the stream
can't permanently stall after the cumulative attempt cap.
* fix(frontend): reduce long tool thread render stalls
Reuse stable historical message groups during streaming, defer heavy Markdown and browser previews, and lazy-decode message images.
* fix(browser): keep live control responsive during continuous input
Why: Manual browser control felt laggy — a physical click ran the remote
Playwright click three times and each non-move input synchronously awaited a
JPEG screenshot, so events queued behind capture (queue wait up to ~237ms).
The first async attempt used a trailing-edge debounce, which froze the visible
page until a wheel/keyboard gesture stopped ("scroll finishes, then it jumps").
What:
- Frontend forwards one `click` per physical click instead of also emitting
`down`/`up`, so the remote page is not clicked twice per gesture.
- Backend detaches live-frame capture from input dispatch: non-move actions
start a rate-limited background refresh loop (leading frame + bounded cadence)
that keeps emitting frames while input continues and never blocks dispatch.
- Add regression tests: input dispatch no longer awaits the screenshot, rapid
inputs coalesce, and continuous input keeps refreshing before it stops.
Scenarios: Verified in the live Browser panel — a single click completes in
~57ms (was blocked behind a 171ms capture), and a 1.14s sustained wheel gesture
renders ~7 frames throughout the scroll instead of one frame after it ends.
* fix(browser): harden worker and session lifecycle
* fix(browser): address latest review feedback
* fix(frontend): preserve optimistic new-chat message
* test(e2e): preserve mocked message run ids
* fix(browser): address capability review feedback
---------
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
172 lines
5.5 KiB
Python
172 lines
5.5 KiB
Python
"""Live end-to-end verification for the agentic browser tools.
|
|
|
|
NOT a unit test. Run manually with DEEPSEEK_API_KEY in the environment:
|
|
|
|
DEEPSEEK_API_KEY=sk-... PYTHONPATH=. uv run python tests/manual_browser_live_check.py
|
|
|
|
It:
|
|
1. serves a tiny local HTML form,
|
|
2. builds an isolated DeerFlow config (DeepSeek model + browser tool group),
|
|
3. runs a real agent turn that must navigate, type, submit, and read the result,
|
|
4. asserts the agent-visible tool trace shows the browser loop actually ran.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
import threading
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
from pathlib import Path
|
|
|
|
FORM_PAGE = """<!doctype html><html><head><title>DeerFlow Browser Test</title></head>
|
|
<body>
|
|
<h1>Sign-in demo</h1>
|
|
<form method="GET" action="/welcome">
|
|
<input type="text" name="username" placeholder="Username">
|
|
<button type="submit">Sign in</button>
|
|
</form>
|
|
</body></html>"""
|
|
|
|
|
|
def _welcome_page(username: str) -> str:
|
|
return f"""<!doctype html><html><head><title>Welcome</title></head>
|
|
<body><h1>Welcome, {username}!</h1><p>SECRET-TOKEN-4917</p></body></html>"""
|
|
|
|
|
|
class _Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *args): # silence server logs
|
|
pass
|
|
|
|
def do_GET(self):
|
|
from urllib.parse import parse_qs, urlparse
|
|
|
|
parsed = urlparse(self.path)
|
|
if parsed.path.startswith("/welcome"):
|
|
qs = parse_qs(parsed.query)
|
|
username = (qs.get("username") or ["friend"])[0]
|
|
body = _welcome_page(username)
|
|
else:
|
|
body = FORM_PAGE
|
|
data = body.encode()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
self.send_header("Content-Length", str(len(data)))
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
|
|
|
|
def _start_server() -> tuple[HTTPServer, int]:
|
|
server = HTTPServer(("127.0.0.1", 0), _Handler)
|
|
port = server.server_address[1]
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
return server, port
|
|
|
|
|
|
def _write_config(tmp: Path) -> Path:
|
|
config = f"""
|
|
config_version: 1
|
|
data_dir: {tmp / "data"}
|
|
|
|
models:
|
|
- name: deepseek-chat
|
|
display_name: DeepSeek Chat
|
|
use: deerflow.models.patched_deepseek:PatchedChatDeepSeek
|
|
model: deepseek-chat
|
|
api_key: $DEEPSEEK_API_KEY
|
|
timeout: 120.0
|
|
max_retries: 2
|
|
max_tokens: 4096
|
|
|
|
sandbox:
|
|
use: deerflow.sandbox.local:LocalSandboxProvider
|
|
|
|
tool_groups:
|
|
- name: browser
|
|
|
|
tools:
|
|
- name: browser_navigate
|
|
group: browser
|
|
use: deerflow.community.browser_automation.tools:browser_navigate_tool
|
|
headless: true
|
|
allow_private_addresses: true
|
|
- name: browser_snapshot
|
|
group: browser
|
|
use: deerflow.community.browser_automation.tools:browser_snapshot_tool
|
|
- name: browser_click
|
|
group: browser
|
|
use: deerflow.community.browser_automation.tools:browser_click_tool
|
|
- name: browser_type
|
|
group: browser
|
|
use: deerflow.community.browser_automation.tools:browser_type_tool
|
|
- name: browser_get_text
|
|
group: browser
|
|
use: deerflow.community.browser_automation.tools:browser_get_text_tool
|
|
- name: browser_close
|
|
group: browser
|
|
use: deerflow.community.browser_automation.tools:browser_close_tool
|
|
|
|
memory:
|
|
enabled: false
|
|
|
|
title:
|
|
enabled: false
|
|
"""
|
|
path = tmp / "config.yaml"
|
|
path.write_text(config)
|
|
return path
|
|
|
|
|
|
def main() -> int:
|
|
if not os.environ.get("DEEPSEEK_API_KEY"):
|
|
print("SKIP: DEEPSEEK_API_KEY not set")
|
|
return 0
|
|
|
|
server, port = _start_server()
|
|
base = f"http://127.0.0.1:{port}/"
|
|
tmpdir = Path(tempfile.mkdtemp(prefix="deerflow-browser-live-"))
|
|
try:
|
|
from deerflow.client import DeerFlowClient
|
|
|
|
config_path = _write_config(tmpdir)
|
|
# Make config resolution deterministic: get_available_tools() re-resolves
|
|
# via get_app_config(), which would otherwise pick up a project-root
|
|
# config.yaml. DEER_FLOW_CONFIG_PATH is resolution priority #2.
|
|
os.environ["DEER_FLOW_CONFIG_PATH"] = str(config_path)
|
|
client = DeerFlowClient(config_path=str(config_path))
|
|
|
|
prompt = (
|
|
f"Use the browser tools to complete this task. "
|
|
f"1) Navigate to {base} 2) type the username 'deerbot' into the username field "
|
|
f"3) click the Sign in button 4) read the resulting page's text. "
|
|
f"Then tell me the exact SECRET token shown on the welcome page."
|
|
)
|
|
|
|
tool_calls: list[str] = []
|
|
final_text = ""
|
|
for event in client.stream(prompt, thread_id="browser-live-check"):
|
|
if event.type == "messages-tuple":
|
|
data = event.data or {}
|
|
if data.get("type") == "tool":
|
|
name = data.get("name") or ""
|
|
if name.startswith("browser_"):
|
|
tool_calls.append(name)
|
|
elif data.get("type") == "ai":
|
|
final_text += data.get("content") or ""
|
|
|
|
print("Browser tool calls observed:", tool_calls)
|
|
print("Final answer:\n", final_text.strip()[:1000])
|
|
|
|
assert "browser_navigate" in tool_calls, "agent never navigated"
|
|
assert any(t in tool_calls for t in ("browser_type", "browser_click")), "agent never interacted"
|
|
assert "SECRET-TOKEN-4917" in final_text, "agent did not read the post-submit page content"
|
|
print("\nLIVE CHECK PASSED")
|
|
return 0
|
|
finally:
|
|
server.shutdown()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|