mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-08-10 06:49:00 +00:00
* fix(agents): stop persisting base64 image data in checkpoint state (#4138) The viewed_images state field stored full base64-encoded image data, which was duplicated across every subsequent checkpoint (O(n * steps) growth). A single 1MB image viewed early in a conversation would be re-stored in every checkpoint for the rest of the session. Changes: - ViewedImageData: replace base64 field with lightweight metadata (mime_type, size, actual_path) - view_image_tool: store only metadata in state, no base64 encoding - ViewImageMiddleware: read image files from disk on-demand in before_model and encode base64 temporarily for the model call - Update all tests to use the new metadata-only format This is the first step of #4138. The base64 data is no longer in persistent state, but the injected HumanMessage (with base64 content) still appears in the checkpoint for the step where it was injected. Checkpoint retention policies and large tool result dedup are separate follow-up items. * fix(agents): address review feedback on #4140 - view_image_tool: remove stale 'convert to base64' comment, replace with 'validate contents'; drop redundant image_size reassignment and add a TOCTOU guard that rejects files changed between stat() and read(). - view_image_middleware: extract _read_image_as_data_url helper that re-checks size against the recorded value AND the absolute cap (_MAX_IMAGE_BYTES). Document the trust assumption for actual_path (server-set, not client-settable) in the helper docstring. - view_image_middleware: abefore_model now runs the blocking read+encode via asyncio.to_thread to avoid stalling the event loop on up to 20MB images. - tests: add coverage for OSError during read, file-changed-since-view (TOCTOU), and size-exceeds-cap branches.
166 lines
5.6 KiB
Python
166 lines
5.6 KiB
Python
import base64
|
|
import importlib
|
|
import os
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from deerflow.tools.builtins.view_image_tool import view_image_tool
|
|
|
|
view_image_module = importlib.import_module("deerflow.tools.builtins.view_image_tool")
|
|
|
|
PNG_BYTES = base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==")
|
|
|
|
|
|
def _make_thread_data(tmp_path: Path) -> dict[str, str]:
|
|
user_data = tmp_path / "threads" / "thread-1" / "user-data"
|
|
workspace = user_data / "workspace"
|
|
uploads = user_data / "uploads"
|
|
outputs = user_data / "outputs"
|
|
for directory in (workspace, uploads, outputs):
|
|
directory.mkdir(parents=True)
|
|
|
|
return {
|
|
"workspace_path": str(workspace),
|
|
"uploads_path": str(uploads),
|
|
"outputs_path": str(outputs),
|
|
}
|
|
|
|
|
|
def _make_runtime(thread_data: dict[str, str]) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
state={"thread_data": thread_data},
|
|
context={"thread_id": "thread-1"},
|
|
config={},
|
|
)
|
|
|
|
|
|
def _message_content(result) -> str:
|
|
return result.update["messages"][0].content
|
|
|
|
|
|
def test_view_image_rejects_external_absolute_path(tmp_path: Path) -> None:
|
|
thread_data = _make_thread_data(tmp_path)
|
|
outside_image = tmp_path / "outside.png"
|
|
outside_image.write_bytes(PNG_BYTES)
|
|
|
|
result = view_image_tool.func(
|
|
runtime=_make_runtime(thread_data),
|
|
image_path=str(outside_image),
|
|
tool_call_id="tc-external",
|
|
)
|
|
|
|
assert "Only image paths under /mnt/user-data" in _message_content(result)
|
|
assert "viewed_images" not in result.update
|
|
|
|
|
|
def test_view_image_reads_virtual_uploads_path(tmp_path: Path) -> None:
|
|
thread_data = _make_thread_data(tmp_path)
|
|
image_path = Path(thread_data["uploads_path"]) / "sample.png"
|
|
image_path.write_bytes(PNG_BYTES)
|
|
|
|
result = view_image_tool.func(
|
|
runtime=_make_runtime(thread_data),
|
|
image_path="/mnt/user-data/uploads/sample.png",
|
|
tool_call_id="tc-uploads",
|
|
)
|
|
|
|
assert _message_content(result) == "Successfully read image"
|
|
viewed_image = result.update["viewed_images"]["/mnt/user-data/uploads/sample.png"]
|
|
assert viewed_image["mime_type"] == "image/png"
|
|
assert viewed_image["size"] == len(PNG_BYTES)
|
|
assert viewed_image["actual_path"] == str(image_path)
|
|
|
|
|
|
def test_view_image_rejects_spoofed_extension(tmp_path: Path) -> None:
|
|
thread_data = _make_thread_data(tmp_path)
|
|
image_path = Path(thread_data["uploads_path"]) / "not-really.png"
|
|
image_path.write_bytes(b"not an image")
|
|
|
|
result = view_image_tool.func(
|
|
runtime=_make_runtime(thread_data),
|
|
image_path="/mnt/user-data/uploads/not-really.png",
|
|
tool_call_id="tc-spoofed",
|
|
)
|
|
|
|
assert "contents do not match" in _message_content(result)
|
|
assert "viewed_images" not in result.update
|
|
|
|
|
|
def test_view_image_rejects_mismatched_magic_bytes(tmp_path: Path) -> None:
|
|
thread_data = _make_thread_data(tmp_path)
|
|
image_path = Path(thread_data["uploads_path"]) / "jpeg-named-png.png"
|
|
image_path.write_bytes(b"\xff\xd8\xff\xe0fake-jpeg")
|
|
|
|
result = view_image_tool.func(
|
|
runtime=_make_runtime(thread_data),
|
|
image_path="/mnt/user-data/uploads/jpeg-named-png.png",
|
|
tool_call_id="tc-mismatch",
|
|
)
|
|
|
|
assert "file extension indicates image/png" in _message_content(result)
|
|
assert "viewed_images" not in result.update
|
|
|
|
|
|
def test_view_image_rejects_oversized_image(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
thread_data = _make_thread_data(tmp_path)
|
|
image_path = Path(thread_data["uploads_path"]) / "sample.png"
|
|
image_path.write_bytes(PNG_BYTES)
|
|
monkeypatch.setattr(view_image_module, "_MAX_IMAGE_BYTES", len(PNG_BYTES) - 1)
|
|
|
|
result = view_image_tool.func(
|
|
runtime=_make_runtime(thread_data),
|
|
image_path="/mnt/user-data/uploads/sample.png",
|
|
tool_call_id="tc-oversized",
|
|
)
|
|
|
|
assert "Image file is too large" in _message_content(result)
|
|
assert "viewed_images" not in result.update
|
|
|
|
|
|
def test_view_image_sanitizes_read_errors(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
thread_data = _make_thread_data(tmp_path)
|
|
image_path = Path(thread_data["uploads_path"]) / "sample.png"
|
|
image_path.write_bytes(PNG_BYTES)
|
|
|
|
def _open(*args, **kwargs):
|
|
raise PermissionError(f"permission denied: {image_path}")
|
|
|
|
monkeypatch.setattr("builtins.open", _open)
|
|
|
|
result = view_image_tool.func(
|
|
runtime=_make_runtime(thread_data),
|
|
image_path="/mnt/user-data/uploads/sample.png",
|
|
tool_call_id="tc-read-error",
|
|
)
|
|
|
|
message = _message_content(result)
|
|
assert "Error reading image file" in message
|
|
assert str(image_path) not in message
|
|
assert str(Path(thread_data["uploads_path"])) not in message
|
|
assert "/mnt/user-data/uploads/sample.png" in message
|
|
assert "viewed_images" not in result.update
|
|
|
|
|
|
@pytest.mark.skipif(os.name == "nt", reason="symlink semantics differ on Windows")
|
|
def test_view_image_rejects_uploads_symlink_escape(tmp_path: Path) -> None:
|
|
thread_data = _make_thread_data(tmp_path)
|
|
outside_image = tmp_path / "outside-target.png"
|
|
outside_image.write_bytes(PNG_BYTES)
|
|
|
|
link_path = Path(thread_data["uploads_path"]) / "escape.png"
|
|
try:
|
|
link_path.symlink_to(outside_image)
|
|
except OSError as exc:
|
|
pytest.skip(f"symlink creation failed: {exc}")
|
|
|
|
result = view_image_tool.func(
|
|
runtime=_make_runtime(thread_data),
|
|
image_path="/mnt/user-data/uploads/escape.png",
|
|
tool_call_id="tc-symlink",
|
|
)
|
|
|
|
assert "path traversal" in _message_content(result)
|
|
assert "viewed_images" not in result.update
|