perf(browser): encode progress frames as JPEG (#4836)

backend/packages/harness/deerflow/AGENTS.md

@@ -25,6 +25,15 @@ same helper for the embedded gate.
 trace id and not a DeerFlow `run_id`. Keep the existing subagent `trace_id` field
 separate: that short id is still only for subagent execution logs/status.
 
+### Browser Progress Screenshots (`community/browser_automation/`)
+
+Hidden per-action browser progress frames use JPEG at quality 80 to keep their
+storage and transfer cost bounded relative to lossless PNG. The explicit
+`browser_screenshot` tool remains PNG because it creates a user-requested
+artifact. New automatic capture entry points must reuse the shared progress
+encoding definition in `tools.py` so the byte encoding and `.jpg` suffix cannot
+drift.
+
 ### Embedded Client (`packages/harness/deerflow/client.py`)
 
 `DeerFlowClient` provides direct in-process access to all DeerFlow capabilities without HTTP services. All return types align with the Gateway API response schemas, so consumer code works identically in HTTP and embedded modes.

backend/packages/harness/deerflow/community/browser_automation/session.py

@@ -22,7 +22,7 @@ import threading
 import time
 from collections.abc import Callable, Coroutine
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, TypeVar
+from typing import TYPE_CHECKING, Any, Literal, TypeVar
 from urllib.parse import urlparse
 
 if TYPE_CHECKING:
@@ -31,6 +31,7 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 T = TypeVar("T")
+ScreenshotType = Literal["png", "jpeg", "webp"]
 
 # Element roles/tags treated as interactive when building a page snapshot. The
 # model addresses elements by the ``data-df-ref`` index this snapshot stamps, so
@@ -533,9 +534,16 @@ class BrowserSession:
         text = await page.inner_text("body")
         return text[:max_chars]
 
-    async def _screenshot_bytes(self, full_page: bool) -> bytes:
+    async def _screenshot_bytes(
+        self,
+        full_page: bool,
+        image_type: ScreenshotType,
+        quality: int | None,
+    ) -> bytes:
         page = await self._ensure_page()
-        return await page.screenshot(full_page=full_page, type="png")
+        if quality is None:
+            return await page.screenshot(full_page=full_page, type=image_type)
+        return await page.screenshot(full_page=full_page, type=image_type, quality=quality)
 
     async def _live_frame(self) -> bytes:
         page = await self._ensure_page()
@@ -786,9 +794,15 @@ class BrowserSession:
         with self._activity():
             return await self._loop.run(self._get_text(max_chars))
 
-    async def screenshot_bytes(self, full_page: bool = False) -> bytes:
+    async def screenshot_bytes(
+        self,
+        full_page: bool = False,
+        *,
+        image_type: ScreenshotType = "png",
+        quality: int | None = None,
+    ) -> bytes:
         with self._activity():
-            return await self._loop.run(self._screenshot_bytes(full_page))
+            return await self._loop.run(self._screenshot_bytes(full_page, image_type, quality))
 
     async def live_frame(self) -> bytes:
         with self._activity():

backend/packages/harness/deerflow/community/browser_automation/tools.py

@@ -17,6 +17,7 @@ import asyncio
 import contextlib
 import logging
 import re
+from dataclasses import dataclass
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Annotated
@@ -32,7 +33,7 @@ from deerflow.config.paths import VIRTUAL_PATH_PREFIX
 from deerflow.constants import BROWSER_FRAMES_DIRNAME
 from deerflow.tools.types import Runtime
 
-from .session import BrowserSession, BrowserSessionManager, PageSnapshot, get_browser_session_manager
+from .session import BrowserSession, BrowserSessionManager, PageSnapshot, ScreenshotType, get_browser_session_manager
 
 logger = logging.getLogger(__name__)
 
@@ -46,6 +47,16 @@ _FRAMES_VIRTUAL_PREFIX = f"{_OUTPUTS_VIRTUAL_PREFIX}/{_BROWSER_FRAMES_DIRNAME}"
 _SAFE_FILENAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
 
 
+@dataclass(frozen=True)
+class _ScreenshotEncoding:
+    image_type: ScreenshotType
+    suffix: str
+    quality: int | None = None
+
+
+_PROGRESS_SCREENSHOT_ENCODING = _ScreenshotEncoding(image_type="jpeg", suffix=".jpg", quality=80)
+
+
 def _get_tool_config(tool_name: str) -> dict:
     config = get_app_config().get_tool_config(tool_name)
     if config is None:
@@ -167,7 +178,7 @@ def _tool_message(content: str, tool_call_id: str) -> Command:
 def _step_screenshot_name(action: str) -> str:
     stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S-%f")
     safe = _SAFE_FILENAME_RE.sub("_", action).strip("._-") or "step"
-    return f"browser-{safe}-{stamp}.png"
+    return f"browser-{safe}-{stamp}{_PROGRESS_SCREENSHOT_ENCODING.suffix}"
 
 
 async def _capture_step_screenshot(runtime: Runtime, session: BrowserSession, action: str) -> str | None:
@@ -181,7 +192,11 @@ async def _capture_step_screenshot(runtime: Runtime, session: BrowserSession, ac
     if isinstance(outputs_path, str):
         return None
     try:
-        content = await session.screenshot_bytes(full_page=False)
+        content = await session.screenshot_bytes(
+            full_page=False,
+            image_type=_PROGRESS_SCREENSHOT_ENCODING.image_type,
+            quality=_PROGRESS_SCREENSHOT_ENCODING.quality,
+        )
         name = _step_screenshot_name(action)
         frames_dir = outputs_path / _BROWSER_FRAMES_DIRNAME
         final_name = await asyncio.to_thread(_write_screenshot, frames_dir, name, content)
@@ -244,7 +259,11 @@ async def navigate_and_capture(*, thread_id: str | None, url: str, outputs_path:
         snapshot = await session.navigate(url)
         screenshot_path: str | None = None
         try:
-            content = await session.screenshot_bytes(full_page=False)
+            content = await session.screenshot_bytes(
+                full_page=False,
+                image_type=_PROGRESS_SCREENSHOT_ENCODING.image_type,
+                quality=_PROGRESS_SCREENSHOT_ENCODING.quality,
+            )
             name = _step_screenshot_name("navigate")
             frames_dir = outputs_path / _BROWSER_FRAMES_DIRNAME
             final_name = await asyncio.to_thread(_write_screenshot, frames_dir, name, content)

backend/tests/test_browser_automation.py

@@ -90,7 +90,7 @@ class TestBrowserTools:
         outputs.mkdir()
         session = MagicMock()
         session.navigate = AsyncMock(return_value=_snapshot())
-        session.screenshot_bytes = AsyncMock(return_value=b"\x89PNG\r\n\x1a\nshot")
+        session.screenshot_bytes = AsyncMock(return_value=b"\xff\xd8jpeg-shot")
         session.schedule_live_frames = MagicMock()
         ctx, _ = await self._patch_session(session)
         with ctx, patch.object(tools, "_get_tool_config", return_value={}):
@@ -100,13 +100,13 @@ class TestBrowserTools:
                 tool_call_id="t1",
             )
         # Screenshot is captured, saved, exposed as an artifact + inline browser_view.
-        session.screenshot_bytes.assert_awaited_once()
+        session.screenshot_bytes.assert_awaited_once_with(full_page=False, image_type="jpeg", quality=80)
         session.schedule_live_frames.assert_called_once()
         artifact = result.update["artifacts"][0]
         assert artifact.startswith("/mnt/user-data/outputs/.browser-frames/browser-navigate-")
-        assert artifact.endswith(".png")
+        assert artifact.endswith(".jpg")
-        saved = list((outputs / ".browser-frames").glob("browser-navigate-*.png"))
+        saved = list((outputs / ".browser-frames").glob("browser-navigate-*.jpg"))
-        assert saved and saved[0].read_bytes() == b"\x89PNG\r\n\x1a\nshot"
+        assert saved and saved[0].read_bytes() == b"\xff\xd8jpeg-shot"
         meta = result.update["messages"][0].additional_kwargs["browser_view"]
         assert meta["screenshot"] == artifact
         assert meta["url"] == "https://example.com/"
@@ -130,6 +130,29 @@ class TestBrowserTools:
         assert "artifacts" not in result.update
         assert result.update["messages"][0].additional_kwargs == {}
 
+    async def test_gateway_navigate_and_capture_uses_jpeg_progress_frame(self, tmp_path):
+        session = MagicMock()
+        session.navigate = AsyncMock(return_value=_snapshot())
+        session.screenshot_bytes = AsyncMock(return_value=b"\xff\xd8gateway-jpeg")
+        lease = MagicMock()
+        lease.__enter__.return_value = session
+        manager = MagicMock()
+        manager.acquire_session.return_value = lease
+
+        with (
+            patch.object(tools, "_validate_url", return_value=None),
+            patch.object(tools, "_get_tool_config", return_value={}),
+            patch.object(tools, "get_browser_session_manager", return_value=manager),
+        ):
+            result = await tools.navigate_and_capture(
+                thread_id="thread-1",
+                url="https://example.com",
+                outputs_path=tmp_path,
+            )
+
+        session.screenshot_bytes.assert_awaited_once_with(full_page=False, image_type="jpeg", quality=80)
+        assert result["screenshot"].endswith(".jpg")
+
     async def test_navigate_blocks_private_url(self):
         session = MagicMock()
         session.navigate = AsyncMock()
@@ -212,6 +235,7 @@ class TestBrowserTools:
         artifact = result.update["artifacts"][0]
         assert artifact == "/mnt/user-data/outputs/Login_Page.png"
         assert (outputs / "Login_Page.png").read_bytes() == b"\x89PNG\r\n\x1a\npng-bytes"
+        session.screenshot_bytes.assert_awaited_once_with(full_page=False)
 
     async def test_screenshot_errors_without_outputs_path(self):
         session = MagicMock()
@@ -318,6 +342,42 @@ async def test_live_frame_returns_jpeg_bytes_without_base64_expansion():
     page.screenshot.assert_awaited_once_with(type="jpeg", quality=_LIVE_FRAME_JPEG_QUALITY)
 
 
+@pytest.mark.asyncio
+async def test_screenshot_bytes_defaults_to_png_without_quality():
+    session = BrowserSession(
+        MagicMock(),
+        headless=True,
+        timeout_ms=1000,
+        viewport={"width": 1000, "height": 500},
+    )
+    page = MagicMock()
+    page.screenshot = AsyncMock(return_value=b"png")
+    session._ensure_page = AsyncMock(return_value=page)
+
+    shot = await session._screenshot_bytes(full_page=False, image_type="png", quality=None)
+
+    assert shot == b"png"
+    page.screenshot.assert_awaited_once_with(full_page=False, type="png")
+
+
+@pytest.mark.asyncio
+async def test_screenshot_bytes_forwards_jpeg_quality():
+    session = BrowserSession(
+        MagicMock(),
+        headless=True,
+        timeout_ms=1000,
+        viewport={"width": 1000, "height": 500},
+    )
+    page = MagicMock()
+    page.screenshot = AsyncMock(return_value=b"jpeg")
+    session._ensure_page = AsyncMock(return_value=page)
+
+    shot = await session._screenshot_bytes(full_page=False, image_type="jpeg", quality=80)
+
+    assert shot == b"jpeg"
+    page.screenshot.assert_awaited_once_with(full_page=False, type="jpeg", quality=80)
+
+
 @pytest.mark.asyncio
 async def test_input_dispatch_does_not_wait_for_live_frame():
     session = BrowserSession(