diff --git a/backend/packages/harness/deerflow/AGENTS.md b/backend/packages/harness/deerflow/AGENTS.md index 469047178..a895012e6 100644 --- a/backend/packages/harness/deerflow/AGENTS.md +++ b/backend/packages/harness/deerflow/AGENTS.md @@ -25,6 +25,15 @@ same helper for the embedded gate. trace id and not a DeerFlow `run_id`. Keep the existing subagent `trace_id` field separate: that short id is still only for subagent execution logs/status. +### Browser Progress Screenshots (`community/browser_automation/`) + +Hidden per-action browser progress frames use JPEG at quality 80 to keep their +storage and transfer cost bounded relative to lossless PNG. The explicit +`browser_screenshot` tool remains PNG because it creates a user-requested +artifact. New automatic capture entry points must reuse the shared progress +encoding definition in `tools.py` so the byte encoding and `.jpg` suffix cannot +drift. + ### Embedded Client (`packages/harness/deerflow/client.py`) `DeerFlowClient` provides direct in-process access to all DeerFlow capabilities without HTTP services. All return types align with the Gateway API response schemas, so consumer code works identically in HTTP and embedded modes. diff --git a/backend/packages/harness/deerflow/community/browser_automation/session.py b/backend/packages/harness/deerflow/community/browser_automation/session.py index fd0413f40..20e6a8221 100644 --- a/backend/packages/harness/deerflow/community/browser_automation/session.py +++ b/backend/packages/harness/deerflow/community/browser_automation/session.py @@ -22,7 +22,7 @@ import threading import time from collections.abc import Callable, Coroutine from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, Literal, TypeVar from urllib.parse import urlparse if TYPE_CHECKING: @@ -31,6 +31,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) T = TypeVar("T") +ScreenshotType = Literal["png", "jpeg", "webp"] # Element roles/tags treated as interactive when building a page snapshot. The # model addresses elements by the ``data-df-ref`` index this snapshot stamps, so @@ -533,9 +534,16 @@ class BrowserSession: text = await page.inner_text("body") return text[:max_chars] - async def _screenshot_bytes(self, full_page: bool) -> bytes: + async def _screenshot_bytes( + self, + full_page: bool, + image_type: ScreenshotType, + quality: int | None, + ) -> bytes: page = await self._ensure_page() - return await page.screenshot(full_page=full_page, type="png") + if quality is None: + return await page.screenshot(full_page=full_page, type=image_type) + return await page.screenshot(full_page=full_page, type=image_type, quality=quality) async def _live_frame(self) -> bytes: page = await self._ensure_page() @@ -786,9 +794,15 @@ class BrowserSession: with self._activity(): return await self._loop.run(self._get_text(max_chars)) - async def screenshot_bytes(self, full_page: bool = False) -> bytes: + async def screenshot_bytes( + self, + full_page: bool = False, + *, + image_type: ScreenshotType = "png", + quality: int | None = None, + ) -> bytes: with self._activity(): - return await self._loop.run(self._screenshot_bytes(full_page)) + return await self._loop.run(self._screenshot_bytes(full_page, image_type, quality)) async def live_frame(self) -> bytes: with self._activity(): diff --git a/backend/packages/harness/deerflow/community/browser_automation/tools.py b/backend/packages/harness/deerflow/community/browser_automation/tools.py index 5427486a1..7a2a10196 100644 --- a/backend/packages/harness/deerflow/community/browser_automation/tools.py +++ b/backend/packages/harness/deerflow/community/browser_automation/tools.py @@ -17,6 +17,7 @@ import asyncio import contextlib import logging import re +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Annotated @@ -32,7 +33,7 @@ from deerflow.config.paths import VIRTUAL_PATH_PREFIX from deerflow.constants import BROWSER_FRAMES_DIRNAME from deerflow.tools.types import Runtime -from .session import BrowserSession, BrowserSessionManager, PageSnapshot, get_browser_session_manager +from .session import BrowserSession, BrowserSessionManager, PageSnapshot, ScreenshotType, get_browser_session_manager logger = logging.getLogger(__name__) @@ -46,6 +47,16 @@ _FRAMES_VIRTUAL_PREFIX = f"{_OUTPUTS_VIRTUAL_PREFIX}/{_BROWSER_FRAMES_DIRNAME}" _SAFE_FILENAME_RE = re.compile(r"[^A-Za-z0-9._-]+") +@dataclass(frozen=True) +class _ScreenshotEncoding: + image_type: ScreenshotType + suffix: str + quality: int | None = None + + +_PROGRESS_SCREENSHOT_ENCODING = _ScreenshotEncoding(image_type="jpeg", suffix=".jpg", quality=80) + + def _get_tool_config(tool_name: str) -> dict: config = get_app_config().get_tool_config(tool_name) if config is None: @@ -167,7 +178,7 @@ def _tool_message(content: str, tool_call_id: str) -> Command: def _step_screenshot_name(action: str) -> str: stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S-%f") safe = _SAFE_FILENAME_RE.sub("_", action).strip("._-") or "step" - return f"browser-{safe}-{stamp}.png" + return f"browser-{safe}-{stamp}{_PROGRESS_SCREENSHOT_ENCODING.suffix}" async def _capture_step_screenshot(runtime: Runtime, session: BrowserSession, action: str) -> str | None: @@ -181,7 +192,11 @@ async def _capture_step_screenshot(runtime: Runtime, session: BrowserSession, ac if isinstance(outputs_path, str): return None try: - content = await session.screenshot_bytes(full_page=False) + content = await session.screenshot_bytes( + full_page=False, + image_type=_PROGRESS_SCREENSHOT_ENCODING.image_type, + quality=_PROGRESS_SCREENSHOT_ENCODING.quality, + ) name = _step_screenshot_name(action) frames_dir = outputs_path / _BROWSER_FRAMES_DIRNAME final_name = await asyncio.to_thread(_write_screenshot, frames_dir, name, content) @@ -244,7 +259,11 @@ async def navigate_and_capture(*, thread_id: str | None, url: str, outputs_path: snapshot = await session.navigate(url) screenshot_path: str | None = None try: - content = await session.screenshot_bytes(full_page=False) + content = await session.screenshot_bytes( + full_page=False, + image_type=_PROGRESS_SCREENSHOT_ENCODING.image_type, + quality=_PROGRESS_SCREENSHOT_ENCODING.quality, + ) name = _step_screenshot_name("navigate") frames_dir = outputs_path / _BROWSER_FRAMES_DIRNAME final_name = await asyncio.to_thread(_write_screenshot, frames_dir, name, content) diff --git a/backend/tests/test_browser_automation.py b/backend/tests/test_browser_automation.py index ba619ac80..1e2f37c38 100644 --- a/backend/tests/test_browser_automation.py +++ b/backend/tests/test_browser_automation.py @@ -90,7 +90,7 @@ class TestBrowserTools: outputs.mkdir() session = MagicMock() session.navigate = AsyncMock(return_value=_snapshot()) - session.screenshot_bytes = AsyncMock(return_value=b"\x89PNG\r\n\x1a\nshot") + session.screenshot_bytes = AsyncMock(return_value=b"\xff\xd8jpeg-shot") session.schedule_live_frames = MagicMock() ctx, _ = await self._patch_session(session) with ctx, patch.object(tools, "_get_tool_config", return_value={}): @@ -100,13 +100,13 @@ class TestBrowserTools: tool_call_id="t1", ) # Screenshot is captured, saved, exposed as an artifact + inline browser_view. - session.screenshot_bytes.assert_awaited_once() + session.screenshot_bytes.assert_awaited_once_with(full_page=False, image_type="jpeg", quality=80) session.schedule_live_frames.assert_called_once() artifact = result.update["artifacts"][0] assert artifact.startswith("/mnt/user-data/outputs/.browser-frames/browser-navigate-") - assert artifact.endswith(".png") - saved = list((outputs / ".browser-frames").glob("browser-navigate-*.png")) - assert saved and saved[0].read_bytes() == b"\x89PNG\r\n\x1a\nshot" + assert artifact.endswith(".jpg") + saved = list((outputs / ".browser-frames").glob("browser-navigate-*.jpg")) + assert saved and saved[0].read_bytes() == b"\xff\xd8jpeg-shot" meta = result.update["messages"][0].additional_kwargs["browser_view"] assert meta["screenshot"] == artifact assert meta["url"] == "https://example.com/" @@ -130,6 +130,29 @@ class TestBrowserTools: assert "artifacts" not in result.update assert result.update["messages"][0].additional_kwargs == {} + async def test_gateway_navigate_and_capture_uses_jpeg_progress_frame(self, tmp_path): + session = MagicMock() + session.navigate = AsyncMock(return_value=_snapshot()) + session.screenshot_bytes = AsyncMock(return_value=b"\xff\xd8gateway-jpeg") + lease = MagicMock() + lease.__enter__.return_value = session + manager = MagicMock() + manager.acquire_session.return_value = lease + + with ( + patch.object(tools, "_validate_url", return_value=None), + patch.object(tools, "_get_tool_config", return_value={}), + patch.object(tools, "get_browser_session_manager", return_value=manager), + ): + result = await tools.navigate_and_capture( + thread_id="thread-1", + url="https://example.com", + outputs_path=tmp_path, + ) + + session.screenshot_bytes.assert_awaited_once_with(full_page=False, image_type="jpeg", quality=80) + assert result["screenshot"].endswith(".jpg") + async def test_navigate_blocks_private_url(self): session = MagicMock() session.navigate = AsyncMock() @@ -212,6 +235,7 @@ class TestBrowserTools: artifact = result.update["artifacts"][0] assert artifact == "/mnt/user-data/outputs/Login_Page.png" assert (outputs / "Login_Page.png").read_bytes() == b"\x89PNG\r\n\x1a\npng-bytes" + session.screenshot_bytes.assert_awaited_once_with(full_page=False) async def test_screenshot_errors_without_outputs_path(self): session = MagicMock() @@ -318,6 +342,42 @@ async def test_live_frame_returns_jpeg_bytes_without_base64_expansion(): page.screenshot.assert_awaited_once_with(type="jpeg", quality=_LIVE_FRAME_JPEG_QUALITY) +@pytest.mark.asyncio +async def test_screenshot_bytes_defaults_to_png_without_quality(): + session = BrowserSession( + MagicMock(), + headless=True, + timeout_ms=1000, + viewport={"width": 1000, "height": 500}, + ) + page = MagicMock() + page.screenshot = AsyncMock(return_value=b"png") + session._ensure_page = AsyncMock(return_value=page) + + shot = await session._screenshot_bytes(full_page=False, image_type="png", quality=None) + + assert shot == b"png" + page.screenshot.assert_awaited_once_with(full_page=False, type="png") + + +@pytest.mark.asyncio +async def test_screenshot_bytes_forwards_jpeg_quality(): + session = BrowserSession( + MagicMock(), + headless=True, + timeout_ms=1000, + viewport={"width": 1000, "height": 500}, + ) + page = MagicMock() + page.screenshot = AsyncMock(return_value=b"jpeg") + session._ensure_page = AsyncMock(return_value=page) + + shot = await session._screenshot_bytes(full_page=False, image_type="jpeg", quality=80) + + assert shot == b"jpeg" + page.screenshot.assert_awaited_once_with(full_page=False, type="jpeg", quality=80) + + @pytest.mark.asyncio async def test_input_dispatch_does_not_wait_for_live_frame(): session = BrowserSession(