mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 21:26:39 +00:00
4fc0d8fb31
* fix(agents): bound viewed image context per model request Keep recent image payloads within a count and byte budget, and move re-viewed paths to the newest position. Add focused regressions for budget enforcement and state ordering. * test(agents): verify cumulative image budgets and re-view recovery * fix(agents): clarify image omission and read-attempt limits * docs(middleware): shorten image context guidance for CI budget --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
1296 lines
59 KiB
Python
1296 lines
59 KiB
Python
"""Unit tests for ViewImageMiddleware.
|
|
|
|
Tests cover the middleware's ability to inject image details (including base64
|
|
payloads) into the model request, triggered only when the previous assistant
|
|
turn contained `view_image` tool calls that have all been completed with
|
|
corresponding ToolMessages.
|
|
|
|
Covered behavior:
|
|
- `_get_last_assistant_message` returns the most recent AIMessage (or None).
|
|
- `_has_view_image_tool` only matches assistant messages with `view_image` tool calls.
|
|
- `_all_tools_completed` verifies every tool call id has a matching ToolMessage.
|
|
- `_create_image_details_message` produces correctly structured content blocks,
|
|
reading image files on-demand from disk (no base64 stored in state).
|
|
- `_should_inject_image_message` gates injection on all preconditions, including
|
|
deduplication when an image-details message is already in the request.
|
|
- `_inject` rebuilds the request's image context: it sweeps out any copy left
|
|
in the message list by an interrupted run before deciding whether to append a
|
|
freshly built one.
|
|
- `wrap_model_call` and `awrap_model_call` expose the same behavior sync/async,
|
|
handing the payload to the model without ever writing it to state — so no
|
|
checkpoint retains it, even if the run is interrupted mid-call.
|
|
"""
|
|
|
|
import base64
|
|
import hashlib
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
from langchain.agents import create_agent
|
|
from langchain.agents.middleware.types import ModelRequest
|
|
from langchain_core.callbacks import BaseCallbackHandler
|
|
from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel
|
|
from langchain_core.messages import AIMessage, AnyMessage, HumanMessage, SystemMessage, ToolMessage
|
|
from PIL import Image
|
|
from support.symlinks import symlink_or_skip
|
|
|
|
from deerflow.agents.middlewares.view_image_middleware import (
|
|
_IMAGE_CONTEXT_MESSAGE_MARKER_KEY,
|
|
ViewImageMiddleware,
|
|
)
|
|
from deerflow.agents.thread_state import merge_viewed_images
|
|
from deerflow.config.paths import Paths
|
|
from deerflow.storage import BlobRef
|
|
from deerflow.tools.builtins.view_image_tool import view_image_tool
|
|
|
|
|
|
@pytest.fixture
|
|
def budget_image_runtime(tmp_path, monkeypatch):
|
|
paths = Paths(tmp_path)
|
|
monkeypatch.setattr("deerflow.config.paths.get_paths", lambda: paths)
|
|
monkeypatch.setattr("deerflow.storage.get_blob_store_if_enabled", lambda: None)
|
|
outputs = paths.sandbox_outputs_dir("thread-test", user_id="user-test")
|
|
outputs.mkdir(parents=True)
|
|
payloads = {}
|
|
for index in range(5):
|
|
buffer = BytesIO()
|
|
Image.new("RGB", (2, 2), (index * 40, 80, 160)).save(buffer, format="PNG")
|
|
payloads[f"/mnt/user-data/outputs/{index}.png"] = buffer.getvalue()
|
|
(outputs / f"{index}.png").write_bytes(buffer.getvalue())
|
|
runtime = SimpleNamespace(
|
|
state={"thread_data": {"outputs_path": str(outputs)}},
|
|
context={"user_id": "user-test", "thread_id": "thread-test"},
|
|
config={},
|
|
)
|
|
return runtime, payloads
|
|
|
|
|
|
def _view_budget_image(runtime, image_path, call_id):
|
|
result = view_image_tool.func(runtime=runtime, image_path=image_path, tool_call_id=call_id)
|
|
assert result.update["messages"][0].content == "Successfully read image"
|
|
runtime.state["viewed_images"] = merge_viewed_images(runtime.state.get("viewed_images"), result.update["viewed_images"])
|
|
request = _model_request([AIMessage(content="", tool_calls=[_view_image_call(call_id, image_path)]), *result.update["messages"]])
|
|
return request.override(state={**runtime.state, "messages": request.messages}, runtime=runtime)
|
|
|
|
|
|
async def _capture_budget_request(request, use_async):
|
|
seen = []
|
|
|
|
def handler(prepared):
|
|
seen.append(prepared)
|
|
return AIMessage(content="ok")
|
|
|
|
async def async_handler(prepared):
|
|
return handler(prepared)
|
|
|
|
middleware = ViewImageMiddleware()
|
|
if use_async:
|
|
await middleware.awrap_model_call(request, async_handler)
|
|
else:
|
|
middleware.wrap_model_call(request, handler)
|
|
assert len(seen) == 1
|
|
assert _image_context_messages(request.messages) == []
|
|
assert _image_context_messages(request.state["messages"]) == []
|
|
return _image_urls(seen[0])
|
|
|
|
|
|
@pytest.mark.anyio
|
|
@pytest.mark.parametrize("use_async", [False, True])
|
|
@pytest.mark.parametrize("budget_delta", [0, -1], ids=["exact-fit", "one-byte-short"])
|
|
async def test_cumulative_image_bytes_reach_model_within_budget(budget_image_runtime, monkeypatch, use_async, budget_delta):
|
|
from deerflow.agents.middlewares import view_image_middleware as module
|
|
|
|
runtime, payloads = budget_image_runtime
|
|
selected_paths = list(payloads)[:3]
|
|
for index, path in enumerate(selected_paths):
|
|
request = _view_budget_image(runtime, path, f"view-{index}")
|
|
encoded = [base64.b64encode(payloads[path]).decode("ascii") for path in selected_paths]
|
|
budget = len(encoded[1]) + len(encoded[2]) + budget_delta
|
|
assert all(len(data) <= budget for data in encoded)
|
|
# Add attributes on the unfixed baseline too: RED must be behavioral,
|
|
# not an AttributeError caused by a missing budget constant.
|
|
monkeypatch.setattr(module, "_MAX_CONTEXT_IMAGES", 4, raising=False)
|
|
monkeypatch.setattr(module, "_MAX_CONTEXT_BASE64_BYTES", budget, raising=False)
|
|
reads = []
|
|
original_open = open
|
|
|
|
def record_open(file, *args, **kwargs):
|
|
if isinstance(file, (str, Path)) and Path(file).suffix == ".png":
|
|
reads.append(Path(file).name)
|
|
return original_open(file, *args, **kwargs)
|
|
|
|
monkeypatch.setattr("builtins.open", record_open)
|
|
urls = await _capture_budget_request(request, use_async)
|
|
expected = selected_paths[1:] if budget_delta == 0 else selected_paths[2:]
|
|
actual = [url.split(",", 1)[1] for url in urls]
|
|
assert sum(len(data) for data in actual) <= budget
|
|
assert [base64.b64decode(data) for data in actual] == [payloads[path] for path in expected]
|
|
assert reads == [Path(path).name for path in reversed(expected)]
|
|
|
|
|
|
@pytest.mark.anyio
|
|
@pytest.mark.parametrize("use_async", [False, True])
|
|
async def test_revisited_image_returns_to_model_context(budget_image_runtime, use_async):
|
|
runtime, payloads = budget_image_runtime
|
|
paths = list(payloads)
|
|
for index, path in enumerate(paths):
|
|
request = _view_budget_image(runtime, path, f"initial-{index}")
|
|
before = await _capture_budget_request(request, use_async)
|
|
assert [base64.b64decode(url.split(",", 1)[1]) for url in before] == [payloads[path] for path in paths[1:]]
|
|
saved = runtime.state["viewed_images"]
|
|
request = _view_budget_image(runtime, paths[0], "revisit-oldest")
|
|
after = await _capture_budget_request(request, use_async)
|
|
assert [base64.b64decode(url.split(",", 1)[1]) for url in after] == [payloads[path] for path in [*paths[2:], paths[0]]]
|
|
assert runtime.state["viewed_images"] == saved
|
|
assert list(saved) == paths
|
|
|
|
|
|
def _view_image_call(call_id: str = "call_1", path: str = "/mnt/user-data/uploads/img.png") -> dict:
|
|
return {"name": "view_image", "id": call_id, "args": {"image_path": path}}
|
|
|
|
|
|
def _other_tool_call(call_id: str = "call_other", name: str = "bash") -> dict:
|
|
return {"name": name, "id": call_id, "args": {"command": "ls"}}
|
|
|
|
|
|
def _model_request(messages: list[AnyMessage], viewed_images: dict | None = None) -> ModelRequest:
|
|
"""Build a real ModelRequest so `.override()` behaves as it does in the graph."""
|
|
return ModelRequest(
|
|
model=FakeMessagesListChatModel(responses=[AIMessage(content="ok")]),
|
|
messages=list(messages),
|
|
system_message=None,
|
|
tool_choice=None,
|
|
tools=[],
|
|
response_format=None,
|
|
state={"messages": list(messages), "viewed_images": viewed_images or {}},
|
|
runtime=MagicMock(),
|
|
model_settings={},
|
|
)
|
|
|
|
|
|
class _CaptureChatMessages(BaseCallbackHandler):
|
|
def __init__(self):
|
|
self.messages = []
|
|
|
|
def on_chat_model_start(self, serialized, messages, **kwargs):
|
|
self.messages = messages[0]
|
|
|
|
|
|
def _image_context_messages(messages: list[AnyMessage]) -> list[HumanMessage]:
|
|
return [message for message in messages if isinstance(message, HumanMessage) and message.id and message.id.startswith("view-image-context:")]
|
|
|
|
|
|
def _make_viewed_image(tmp_path, filename="img.png", mime_type="image/png", data=b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"):
|
|
"""Create a real image file and return viewed_images metadata dict."""
|
|
img_path = tmp_path / filename
|
|
img_path.write_bytes(data)
|
|
return {
|
|
"mime_type": mime_type,
|
|
"size": len(data),
|
|
"actual_path": str(img_path),
|
|
}
|
|
|
|
|
|
def _authorized_host_view(tmp_path, monkeypatch):
|
|
paths = Paths(tmp_path)
|
|
monkeypatch.setattr("deerflow.config.paths.get_paths", lambda: paths)
|
|
outputs = paths.sandbox_outputs_dir("thread-test", user_id="user-test")
|
|
outputs.mkdir(parents=True)
|
|
virtual_path = "/mnt/user-data/outputs/img.png"
|
|
return virtual_path, _make_viewed_image(outputs)
|
|
|
|
|
|
def _blob_backed_request(monkeypatch):
|
|
from deerflow.authz import sandbox_authz
|
|
|
|
image_bytes = b"\x89PNG\r\n\x1a\nshared-model-image"
|
|
ref = BlobRef(
|
|
sha256=hashlib.sha256(image_bytes).hexdigest(),
|
|
size=len(image_bytes),
|
|
kind="viewed-image",
|
|
content_type="image/png",
|
|
)
|
|
|
|
class SharedStore:
|
|
def get_bytes(self, requested: BlobRef) -> bytes:
|
|
assert requested == ref
|
|
return image_bytes
|
|
|
|
monkeypatch.setattr("deerflow.storage.get_blob_store_if_enabled", lambda: SharedStore())
|
|
monkeypatch.setattr(sandbox_authz, "authorize_sandbox_execution", lambda **kwargs: None)
|
|
virtual_path = "/mnt/user-data/outputs/shared.png"
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1", virtual_path)])
|
|
request = _model_request(
|
|
[assistant, ToolMessage(content="ok", tool_call_id="c1")],
|
|
{
|
|
virtual_path: {
|
|
"mime_type": "image/png",
|
|
"size": len(image_bytes),
|
|
"actual_path": "Z:/writer-only/shared.png",
|
|
"sha256": ref.sha256,
|
|
"blob_ref": ref.model_dump(exclude_none=True),
|
|
}
|
|
},
|
|
)
|
|
request.runtime.context = {"user_id": "user-test", "thread_id": "thread-test"}
|
|
return request, "data:image/png;base64," + base64.b64encode(image_bytes).decode("ascii")
|
|
|
|
|
|
class TestGetLastAssistantMessage:
|
|
def test_returns_none_on_empty_list(self):
|
|
mw = ViewImageMiddleware()
|
|
assert mw._get_last_assistant_message([]) is None
|
|
|
|
def test_returns_none_when_no_ai_message(self):
|
|
mw = ViewImageMiddleware()
|
|
messages = [
|
|
SystemMessage(content="sys"),
|
|
HumanMessage(content="hi"),
|
|
]
|
|
assert mw._get_last_assistant_message(messages) is None
|
|
|
|
def test_returns_most_recent_ai_message(self):
|
|
mw = ViewImageMiddleware()
|
|
older = AIMessage(content="older")
|
|
newer = AIMessage(content="newer")
|
|
messages = [HumanMessage(content="q"), older, HumanMessage(content="q2"), newer]
|
|
assert mw._get_last_assistant_message(messages) is newer
|
|
|
|
|
|
class TestHasViewImageTool:
|
|
def test_returns_false_when_tool_calls_attr_missing(self):
|
|
"""Exercise the `not hasattr(message, "tool_calls")` guard.
|
|
|
|
AIMessage always has a `tool_calls` attribute, so we use a plain
|
|
object that truly lacks the attribute to cover this branch.
|
|
"""
|
|
mw = ViewImageMiddleware()
|
|
msg = SimpleNamespace(content="just text") # no tool_calls attribute
|
|
assert not hasattr(msg, "tool_calls") # precondition
|
|
assert mw._has_view_image_tool(msg) is False
|
|
|
|
def test_returns_false_when_ai_message_has_no_tool_calls(self):
|
|
"""AIMessage without tool_calls kwarg defaults to an empty list."""
|
|
mw = ViewImageMiddleware()
|
|
msg = AIMessage(content="just text")
|
|
assert mw._has_view_image_tool(msg) is False
|
|
|
|
def test_returns_false_when_tool_calls_empty(self):
|
|
mw = ViewImageMiddleware()
|
|
msg = AIMessage(content="", tool_calls=[])
|
|
assert mw._has_view_image_tool(msg) is False
|
|
|
|
def test_returns_true_when_view_image_present(self):
|
|
mw = ViewImageMiddleware()
|
|
msg = AIMessage(content="", tool_calls=[_view_image_call()])
|
|
assert mw._has_view_image_tool(msg) is True
|
|
|
|
def test_returns_true_when_view_image_mixed_with_others(self):
|
|
mw = ViewImageMiddleware()
|
|
msg = AIMessage(
|
|
content="",
|
|
tool_calls=[_other_tool_call(), _view_image_call(call_id="call_vi")],
|
|
)
|
|
assert mw._has_view_image_tool(msg) is True
|
|
|
|
def test_returns_false_when_only_other_tools(self):
|
|
mw = ViewImageMiddleware()
|
|
msg = AIMessage(content="", tool_calls=[_other_tool_call()])
|
|
assert mw._has_view_image_tool(msg) is False
|
|
|
|
|
|
class TestAllToolsCompleted:
|
|
def test_returns_false_when_no_tool_calls(self):
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(content="", tool_calls=[])
|
|
assert mw._all_tools_completed([assistant], assistant) is False
|
|
|
|
def test_returns_true_when_all_completed(self):
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(
|
|
content="",
|
|
tool_calls=[_view_image_call("c1"), _view_image_call("c2", "/p2.png")],
|
|
)
|
|
messages = [
|
|
assistant,
|
|
ToolMessage(content="ok", tool_call_id="c1"),
|
|
ToolMessage(content="ok", tool_call_id="c2"),
|
|
]
|
|
assert mw._all_tools_completed(messages, assistant) is True
|
|
|
|
def test_returns_false_when_some_tool_call_unanswered(self):
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(
|
|
content="",
|
|
tool_calls=[_view_image_call("c1"), _view_image_call("c2", "/p2.png")],
|
|
)
|
|
messages = [assistant, ToolMessage(content="ok", tool_call_id="c1")]
|
|
assert mw._all_tools_completed(messages, assistant) is False
|
|
|
|
def test_returns_false_when_assistant_not_in_messages(self):
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1")])
|
|
# assistant is not part of the list, so messages.index() will raise and be caught
|
|
messages = [HumanMessage(content="hi")]
|
|
assert mw._all_tools_completed(messages, assistant) is False
|
|
|
|
def test_ignores_tool_messages_before_assistant(self):
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1")])
|
|
# A stale ToolMessage with matching id appears BEFORE the assistant turn.
|
|
# It should not count — only ToolMessages after the assistant close the call.
|
|
messages = [
|
|
ToolMessage(content="stale", tool_call_id="c1"),
|
|
assistant,
|
|
]
|
|
assert mw._all_tools_completed(messages, assistant) is False
|
|
|
|
|
|
class TestCreateImageDetailsMessage:
|
|
def test_keeps_recent_images_within_count_and_byte_budget(self, tmp_path, monkeypatch):
|
|
from deerflow.agents.middlewares import view_image_middleware as module
|
|
|
|
monkeypatch.setattr(module, "_MAX_CONTEXT_IMAGES", 2)
|
|
monkeypatch.setattr(module, "_MAX_CONTEXT_BASE64_BYTES", 16)
|
|
viewed = {f"/{name}.png": _make_viewed_image(tmp_path, f"{name}.png", data=b"123456") for name in ("old", "middle", "new")}
|
|
reads = []
|
|
|
|
def read(_cls, _state, image_path, *_args, **_kwargs):
|
|
reads.append(image_path)
|
|
return "data:image/png;base64,MTIzNDU2"
|
|
|
|
monkeypatch.setattr(ViewImageMiddleware, "_read_image_as_data_url", classmethod(read))
|
|
blocks = ViewImageMiddleware()._create_image_details_message({"viewed_images": viewed})
|
|
|
|
assert reads == ["/new.png", "/middle.png"]
|
|
assert [block["type"] for block in blocks].count("image_url") == 2
|
|
assert "Omitted viewed images: 1." in str(blocks)
|
|
|
|
@pytest.mark.parametrize("omitted_count", [1, 2])
|
|
def test_skips_oversized_image_without_reading_it(self, tmp_path, monkeypatch, omitted_count):
|
|
from deerflow.agents.middlewares import view_image_middleware as module
|
|
|
|
monkeypatch.setattr(module, "_MAX_CONTEXT_BASE64_BYTES", 8)
|
|
viewed = {f"/large-{i}.png": _make_viewed_image(tmp_path, f"large-{i}.png", data=b"1234567") for i in range(omitted_count)}
|
|
monkeypatch.setattr(
|
|
ViewImageMiddleware,
|
|
"_read_image_as_data_url",
|
|
lambda *_args, **_kwargs: pytest.fail("oversized image was read"),
|
|
)
|
|
blocks = ViewImageMiddleware()._create_image_details_message({"viewed_images": viewed})
|
|
assert all(block["type"] != "image_url" for block in blocks)
|
|
assert f"Omitted viewed images: {omitted_count}." in str(blocks)
|
|
|
|
def test_omission_notice_does_not_assume_omitted_image_is_older(self, tmp_path, monkeypatch):
|
|
from deerflow.agents.middlewares import view_image_middleware as module
|
|
|
|
monkeypatch.setattr(module, "_MAX_CONTEXT_BASE64_BYTES", 8)
|
|
viewed = {
|
|
"/old.png": _make_viewed_image(tmp_path, "old.png", data=b"123456"),
|
|
"/new.png": _make_viewed_image(tmp_path, "new.png", data=b"1234567"),
|
|
}
|
|
blocks = ViewImageMiddleware()._create_image_details_message({"viewed_images": viewed}, host_path_allowed=lambda _, actual: Path(actual).is_relative_to(tmp_path))
|
|
urls = [block["image_url"]["url"] for block in blocks if block["type"] == "image_url"]
|
|
assert [base64.b64decode(url.split(",", 1)[1]) for url in urls] == [b"123456"]
|
|
assert blocks[1] == {"type": "text", "text": "Omitted viewed images: 1. Call view_image again to inspect a specific image."}
|
|
|
|
def test_failed_reads_consume_slots_before_readable_older_image(self, tmp_path, monkeypatch):
|
|
viewed = {"/old.png": _make_viewed_image(tmp_path, "old.png")}
|
|
viewed.update({f"/stale-{i}.png": {"mime_type": "image/png", "size": 1} for i in range(4)})
|
|
middleware = ViewImageMiddleware()
|
|
original_read = middleware._read_image_as_data_url
|
|
reads = []
|
|
|
|
def record_read(state, image_path, *args, **kwargs):
|
|
reads.append(image_path)
|
|
return original_read(state, image_path, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(middleware, "_read_image_as_data_url", record_read)
|
|
blocks = middleware._create_image_details_message({"viewed_images": viewed}, host_path_allowed=lambda _, actual: Path(actual).is_relative_to(tmp_path))
|
|
assert reads == [f"/stale-{i}.png" for i in reversed(range(4))]
|
|
assert sum("file unavailable or changed" in block.get("text", "") for block in blocks) == 4
|
|
assert all(block["type"] != "image_url" for block in blocks)
|
|
assert "Omitted viewed images: 1." in str(blocks)
|
|
|
|
def test_returns_placeholder_when_no_images(self):
|
|
mw = ViewImageMiddleware()
|
|
state = {"viewed_images": {}}
|
|
blocks = mw._create_image_details_message(state)
|
|
assert blocks == [{"type": "text", "text": "No images have been viewed."}]
|
|
|
|
def test_returns_placeholder_when_state_missing_key(self):
|
|
mw = ViewImageMiddleware()
|
|
blocks = mw._create_image_details_message({})
|
|
assert blocks == [{"type": "text", "text": "No images have been viewed."}]
|
|
|
|
def test_builds_blocks_for_single_image(self, tmp_path):
|
|
mw = ViewImageMiddleware()
|
|
img_meta = _make_viewed_image(tmp_path, "cat.png")
|
|
state = {
|
|
"viewed_images": {
|
|
"/path/to/cat.png": img_meta,
|
|
}
|
|
}
|
|
blocks = mw._create_image_details_message(state, host_path_allowed=lambda _, actual: Path(actual).is_relative_to(tmp_path))
|
|
|
|
# header text + per-image description text + per-image image_url block
|
|
assert len(blocks) == 3
|
|
assert blocks[0] == {"type": "text", "text": "Here are the images you've viewed:"}
|
|
assert blocks[1]["type"] == "text"
|
|
assert "/path/to/cat.png" in blocks[1]["text"]
|
|
assert "image/png" in blocks[1]["text"]
|
|
assert blocks[2]["type"] == "image_url"
|
|
assert blocks[2]["image_url"]["url"].startswith("data:image/png;base64,")
|
|
|
|
def test_resolves_blob_ref_when_writer_host_copy_is_absent(self, tmp_path, monkeypatch):
|
|
image_bytes = b"\x89PNG\r\n\x1a\nshared-image"
|
|
ref = BlobRef(
|
|
sha256=hashlib.sha256(image_bytes).hexdigest(),
|
|
size=len(image_bytes),
|
|
kind="viewed-image",
|
|
content_type="image/png",
|
|
)
|
|
reads: list[BlobRef] = []
|
|
|
|
class SharedStore:
|
|
def get_bytes(self, requested: BlobRef) -> bytes:
|
|
reads.append(requested)
|
|
return image_bytes
|
|
|
|
monkeypatch.setattr("deerflow.storage.get_blob_store_if_enabled", lambda: SharedStore())
|
|
state = {
|
|
"viewed_images": {
|
|
"/mnt/user-data/outputs/shared.png": {
|
|
"mime_type": "image/png",
|
|
"size": len(image_bytes),
|
|
"actual_path": str(tmp_path / "writer-only" / "shared.png"),
|
|
"sha256": ref.sha256,
|
|
"blob_ref": ref.model_dump(exclude_none=True),
|
|
}
|
|
}
|
|
}
|
|
|
|
blocks = ViewImageMiddleware()._create_image_details_message(state)
|
|
|
|
image_blocks = [block for block in blocks if isinstance(block, dict) and block.get("type") == "image_url"]
|
|
assert len(image_blocks) == 1
|
|
assert image_blocks[0]["image_url"]["url"] == "data:image/png;base64," + base64.b64encode(image_bytes).decode("ascii")
|
|
assert reads == [ref]
|
|
|
|
def test_blob_store_factory_failure_falls_back_to_validated_host_copy(self, tmp_path, monkeypatch):
|
|
image_bytes = b"\x89PNG\r\n\x1a\nlocal-fallback"
|
|
img_path = tmp_path / "fallback.png"
|
|
img_path.write_bytes(image_bytes)
|
|
ref = BlobRef(
|
|
sha256=hashlib.sha256(image_bytes).hexdigest(),
|
|
size=len(image_bytes),
|
|
kind="viewed-image",
|
|
content_type="image/png",
|
|
)
|
|
|
|
def fail_to_resolve_store():
|
|
raise ValueError("misconfigured backend")
|
|
|
|
monkeypatch.setattr("deerflow.storage.get_blob_store_if_enabled", fail_to_resolve_store)
|
|
state = {
|
|
"viewed_images": {
|
|
"/mnt/user-data/outputs/fallback.png": {
|
|
"mime_type": "image/png",
|
|
"size": len(image_bytes),
|
|
"actual_path": str(img_path),
|
|
"sha256": ref.sha256,
|
|
"blob_ref": ref.model_dump(exclude_none=True),
|
|
}
|
|
}
|
|
}
|
|
|
|
blocks = ViewImageMiddleware()._create_image_details_message(
|
|
state,
|
|
host_path_allowed=lambda _virtual, actual: actual == str(img_path),
|
|
)
|
|
|
|
image_blocks = [block for block in blocks if isinstance(block, dict) and block.get("type") == "image_url"]
|
|
assert len(image_blocks) == 1
|
|
assert image_blocks[0]["image_url"]["url"] == "data:image/png;base64," + base64.b64encode(image_bytes).decode("ascii")
|
|
|
|
def test_raw_blob_read_failure_falls_back_to_validated_host_copy(self, tmp_path, monkeypatch):
|
|
image_bytes = b"\x89PNG\r\n\x1a\nlocal-fallback"
|
|
img_path = tmp_path / "fallback.png"
|
|
img_path.write_bytes(image_bytes)
|
|
ref = BlobRef(
|
|
sha256=hashlib.sha256(image_bytes).hexdigest(),
|
|
size=len(image_bytes),
|
|
kind="viewed-image",
|
|
content_type="image/png",
|
|
)
|
|
|
|
class FailingStore:
|
|
def get_bytes(self, requested: BlobRef) -> bytes:
|
|
raise RuntimeError("raw SDK failure")
|
|
|
|
monkeypatch.setattr("deerflow.storage.get_blob_store_if_enabled", lambda: FailingStore())
|
|
state = {
|
|
"viewed_images": {
|
|
"/mnt/user-data/outputs/fallback.png": {
|
|
"mime_type": "image/png",
|
|
"size": len(image_bytes),
|
|
"actual_path": str(img_path),
|
|
"sha256": ref.sha256,
|
|
"blob_ref": ref.model_dump(exclude_none=True),
|
|
}
|
|
}
|
|
}
|
|
|
|
blocks = ViewImageMiddleware()._create_image_details_message(
|
|
state,
|
|
host_path_allowed=lambda _virtual, actual: actual == str(img_path),
|
|
)
|
|
|
|
image_blocks = [block for block in blocks if isinstance(block, dict) and block.get("type") == "image_url"]
|
|
assert len(image_blocks) == 1
|
|
assert image_blocks[0]["image_url"]["url"] == "data:image/png;base64," + base64.b64encode(image_bytes).decode("ascii")
|
|
|
|
def test_malformed_blob_ref_falls_back_to_validated_host_copy(self, tmp_path):
|
|
image_bytes = b"\x89PNG\r\n\x1a\nlocal-fallback"
|
|
img_path = tmp_path / "fallback.png"
|
|
img_path.write_bytes(image_bytes)
|
|
state = {
|
|
"viewed_images": {
|
|
"/mnt/user-data/outputs/fallback.png": {
|
|
"mime_type": "image/png",
|
|
"size": len(image_bytes),
|
|
"actual_path": str(img_path),
|
|
"sha256": hashlib.sha256(image_bytes).hexdigest(),
|
|
"blob_ref": {"sha256": "not-a-digest", "size": len(image_bytes), "kind": "viewed-image"},
|
|
}
|
|
}
|
|
}
|
|
|
|
blocks = ViewImageMiddleware()._create_image_details_message(
|
|
state,
|
|
host_path_allowed=lambda _virtual, actual: actual == str(img_path),
|
|
)
|
|
|
|
assert any(isinstance(block, dict) and block.get("type") == "image_url" for block in blocks)
|
|
|
|
def test_blob_ref_must_match_checkpoint_metadata_before_read(self, tmp_path, monkeypatch):
|
|
image_bytes = b"\x89PNG\r\n\x1a\nshared-image"
|
|
ref = BlobRef(
|
|
sha256=hashlib.sha256(image_bytes).hexdigest(),
|
|
size=len(image_bytes),
|
|
kind="viewed-image",
|
|
content_type="image/png",
|
|
)
|
|
|
|
class StoreMustNotBeRead:
|
|
def get_bytes(self, requested: BlobRef) -> bytes:
|
|
raise AssertionError(f"unexpected blob read: {requested}")
|
|
|
|
monkeypatch.setattr("deerflow.storage.get_blob_store_if_enabled", lambda: StoreMustNotBeRead())
|
|
state = {
|
|
"viewed_images": {
|
|
"/mnt/user-data/outputs/shared.png": {
|
|
"mime_type": "image/png",
|
|
"size": len(image_bytes) + 1,
|
|
"actual_path": str(tmp_path / "missing.png"),
|
|
"sha256": ref.sha256,
|
|
"blob_ref": ref.model_dump(exclude_none=True),
|
|
}
|
|
}
|
|
}
|
|
|
|
blocks = ViewImageMiddleware()._create_image_details_message(state)
|
|
|
|
assert all(not (isinstance(block, dict) and block.get("type") == "image_url") for block in blocks)
|
|
|
|
def test_builds_blocks_for_multiple_images(self, tmp_path):
|
|
mw = ViewImageMiddleware()
|
|
img1 = _make_viewed_image(tmp_path, "a.png", data=b"\x89PNG\r\n\x1a\nfake-png")
|
|
img2 = _make_viewed_image(tmp_path, "b.jpg", mime_type="image/jpeg", data=b"\xff\xd8\xff\xe0fake-jpeg")
|
|
state = {
|
|
"viewed_images": {
|
|
"/a.png": img1,
|
|
"/b.jpg": img2,
|
|
}
|
|
}
|
|
blocks = mw._create_image_details_message(state, host_path_allowed=lambda _, actual: Path(actual).is_relative_to(tmp_path))
|
|
|
|
# 1 header + (1 description + 1 image_url) per image = 5 blocks
|
|
assert len(blocks) == 5
|
|
image_url_blocks = [b for b in blocks if isinstance(b, dict) and b.get("type") == "image_url"]
|
|
assert len(image_url_blocks) == 2
|
|
urls = {b["image_url"]["url"] for b in image_url_blocks}
|
|
assert any(u.startswith("data:image/png;base64,") for u in urls)
|
|
assert any(u.startswith("data:image/jpeg;base64,") for u in urls)
|
|
|
|
def test_omits_image_url_block_when_file_missing(self, tmp_path):
|
|
mw = ViewImageMiddleware()
|
|
state = {
|
|
"viewed_images": {
|
|
"/broken.png": {
|
|
"mime_type": "image/png",
|
|
"size": 0,
|
|
"actual_path": str(tmp_path / "nonexistent.png"),
|
|
},
|
|
}
|
|
}
|
|
blocks = mw._create_image_details_message(state)
|
|
# header + description + error text (file no longer available)
|
|
assert len(blocks) == 3
|
|
assert all(not (isinstance(b, dict) and b.get("type") == "image_url") for b in blocks)
|
|
|
|
def test_uses_unknown_mime_type_when_missing(self, tmp_path):
|
|
mw = ViewImageMiddleware()
|
|
img_meta = _make_viewed_image(tmp_path, "mystery.bin", mime_type="unknown")
|
|
state = {
|
|
"viewed_images": {
|
|
"/mystery.bin": img_meta,
|
|
}
|
|
}
|
|
blocks = mw._create_image_details_message(state, host_path_allowed=lambda _, actual: Path(actual).is_relative_to(tmp_path))
|
|
# The description block should mention unknown
|
|
description_blocks = [b for b in blocks if b.get("type") == "text" and "/mystery.bin" in b.get("text", "")]
|
|
assert len(description_blocks) == 1
|
|
assert "unknown" in description_blocks[0]["text"]
|
|
|
|
def test_omits_image_url_when_read_raises_oserror(self, tmp_path, monkeypatch):
|
|
"""A failure during on-demand read must not crash the middleware."""
|
|
img_meta = _make_viewed_image(tmp_path, "ok.png")
|
|
state = {
|
|
"viewed_images": {
|
|
"/ok.png": img_meta,
|
|
}
|
|
}
|
|
|
|
def _raise(*args, **kwargs):
|
|
raise OSError("disk error")
|
|
|
|
monkeypatch.setattr("builtins.open", _raise)
|
|
|
|
mw = ViewImageMiddleware()
|
|
blocks = mw._create_image_details_message(state)
|
|
# header + description + 'unavailable' text, no image_url block
|
|
assert all(not (isinstance(b, dict) and b.get("type") == "image_url") for b in blocks)
|
|
unavailable = [b for b in blocks if isinstance(b, dict) and b.get("type") == "text" and "unavailable" in b.get("text", "")]
|
|
assert len(unavailable) == 1
|
|
|
|
def test_omits_image_url_when_size_changes_between_view_and_inject(self, tmp_path):
|
|
"""Defense against TOCTOU growth: skip if current size differs from recorded size."""
|
|
img_meta = _make_viewed_image(tmp_path, "shrinking.png", data=b"original-larger-content")
|
|
# Grow the file after the metadata was written
|
|
img_meta_path = Path(img_meta["actual_path"])
|
|
img_meta_path.write_bytes(b"much-much-much-larger-content-bytes")
|
|
|
|
state = {"viewed_images": {"/shrinking.png": img_meta}}
|
|
mw = ViewImageMiddleware()
|
|
blocks = mw._create_image_details_message(state)
|
|
assert all(not (isinstance(b, dict) and b.get("type") == "image_url") for b in blocks)
|
|
|
|
def test_omits_image_url_when_size_exceeds_cap(self, tmp_path):
|
|
"""Records a small size but the actual file is large - the cap kicks in regardless."""
|
|
img_meta = _make_viewed_image(tmp_path, "huge.png", data=b"x" * 100)
|
|
img_meta_path = Path(img_meta["actual_path"])
|
|
# Grow past the cap (20 MB)
|
|
img_meta_path.write_bytes(b"y" * (21 * 1024 * 1024))
|
|
|
|
state = {"viewed_images": {"/huge.png": img_meta}}
|
|
mw = ViewImageMiddleware()
|
|
blocks = mw._create_image_details_message(state)
|
|
assert all(not (isinstance(b, dict) and b.get("type") == "image_url") for b in blocks)
|
|
|
|
|
|
class TestShouldInjectImageMessage:
|
|
def test_false_when_no_messages(self):
|
|
mw = ViewImageMiddleware()
|
|
assert mw._should_inject_image_message([]) is False
|
|
|
|
def test_false_when_no_assistant_message(self):
|
|
mw = ViewImageMiddleware()
|
|
assert mw._should_inject_image_message([HumanMessage(content="hello")]) is False
|
|
|
|
def test_false_when_no_view_image_tool_call(self):
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(content="", tool_calls=[_other_tool_call()])
|
|
messages = [assistant, ToolMessage(content="ok", tool_call_id="call_other")]
|
|
assert mw._should_inject_image_message(messages) is False
|
|
|
|
def test_false_when_tool_not_completed(self):
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1")])
|
|
assert mw._should_inject_image_message([assistant]) is False # no ToolMessage yet
|
|
|
|
def test_true_when_all_preconditions_met(self):
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1")])
|
|
messages = [assistant, ToolMessage(content="ok", tool_call_id="c1")]
|
|
assert mw._should_inject_image_message(messages) is True
|
|
|
|
def test_false_when_already_injected(self):
|
|
"""A checkpoint written by an older version (or by a run that died before
|
|
its `RemoveMessage` cleanup landed) can still carry image details. Do not
|
|
add a duplicate on top of one."""
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1")])
|
|
already_injected = HumanMessage(content="Here are the images you've viewed: /img.png")
|
|
messages = [
|
|
assistant,
|
|
ToolMessage(content="ok", tool_call_id="c1"),
|
|
already_injected,
|
|
]
|
|
assert mw._should_inject_image_message(messages) is False
|
|
|
|
def test_false_when_already_injected_with_list_content(self, tmp_path):
|
|
"""Deduplication must recognize the real injected payload shape.
|
|
|
|
An unmarked leftover carries `.content` as a *list* of dicts (text +
|
|
image_url blocks), not a plain string. This test reuses
|
|
`_create_image_details_message` output to reproduce the realistic shape
|
|
and confirms the marker is still detected via `str(msg.content)`.
|
|
"""
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1")])
|
|
viewed_images = {"/img.png": _make_viewed_image(tmp_path)}
|
|
# Build content the same way the middleware would.
|
|
real_injected_content = mw._create_image_details_message({"viewed_images": viewed_images})
|
|
# Sanity: this is a list of blocks, not a plain string.
|
|
assert isinstance(real_injected_content, list)
|
|
|
|
messages = [
|
|
assistant,
|
|
ToolMessage(content="ok", tool_call_id="c1"),
|
|
HumanMessage(content=real_injected_content),
|
|
]
|
|
assert mw._should_inject_image_message(messages) is False
|
|
|
|
def test_false_when_legacy_details_marker_present(self):
|
|
"""The middleware also recognizes the legacy 'Here are the details of the
|
|
images you've viewed' marker as an already-injected signal."""
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1")])
|
|
legacy = HumanMessage(content="Here are the details of the images you've viewed: ...")
|
|
messages = [
|
|
assistant,
|
|
ToolMessage(content="ok", tool_call_id="c1"),
|
|
legacy,
|
|
]
|
|
assert mw._should_inject_image_message(messages) is False
|
|
|
|
|
|
class TestInject:
|
|
def test_returns_request_unchanged_when_should_not_inject(self):
|
|
mw = ViewImageMiddleware()
|
|
request = _model_request([HumanMessage(content="hi")])
|
|
assert mw._inject(request) is request
|
|
|
|
def test_appends_image_context_message_to_request(self, tmp_path, monkeypatch):
|
|
mw = ViewImageMiddleware()
|
|
virtual_path, image_metadata = _authorized_host_view(tmp_path, monkeypatch)
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1", virtual_path)])
|
|
original = [assistant, ToolMessage(content="ok", tool_call_id="c1")]
|
|
request = _model_request(original, {virtual_path: image_metadata})
|
|
request.runtime.context = {"user_id": "user-test", "thread_id": "thread-test"}
|
|
|
|
injected_request = mw._inject(request)
|
|
|
|
assert injected_request is not request
|
|
# The payload is appended last, so it directly follows the tool results.
|
|
assert injected_request.messages[:-1] == original
|
|
injected = injected_request.messages[-1]
|
|
assert isinstance(injected, HumanMessage)
|
|
# Mixed-content payload: list of text + image_url blocks
|
|
assert isinstance(injected.content, list)
|
|
assert any(isinstance(b, dict) and b.get("type") == "image_url" for b in injected.content)
|
|
# Internal injection: must be hidden from the chat UI (and IM channels),
|
|
# like the other middleware-injected context messages.
|
|
assert injected.additional_kwargs.get("hide_from_ui") is True
|
|
assert injected.additional_kwargs.get(_IMAGE_CONTEXT_MESSAGE_MARKER_KEY) is True
|
|
assert injected.id is not None
|
|
assert injected.id.startswith("view-image-context:")
|
|
|
|
def test_replaces_a_stranded_payload_instead_of_stacking_a_second_one(self, tmp_path, monkeypatch):
|
|
"""A run that died during the model call can leave the old
|
|
before_model/after_model pair's message checkpointed. Rebuild it rather
|
|
than adding a second copy on top."""
|
|
mw = ViewImageMiddleware()
|
|
virtual_path, image_metadata = _authorized_host_view(tmp_path, monkeypatch)
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1", virtual_path)])
|
|
stranded = ViewImageMiddleware._create_image_context_message([{"type": "text", "text": "stale"}])
|
|
request = _model_request(
|
|
[assistant, ToolMessage(content="ok", tool_call_id="c1"), stranded],
|
|
{virtual_path: image_metadata},
|
|
)
|
|
request.runtime.context = {"user_id": "user-test", "thread_id": "thread-test"}
|
|
|
|
injected = _image_context_messages(mw._inject(request).messages)
|
|
|
|
assert len(injected) == 1
|
|
assert injected[0].id != stranded.id
|
|
assert any(isinstance(b, dict) and b.get("type") == "image_url" for b in injected[0].content)
|
|
|
|
def test_drops_a_stranded_payload_even_when_no_injection_is_warranted(self):
|
|
"""Otherwise the stale base64 would ride along in every later request for
|
|
the life of the thread -- the old `after_model` swept it, so dropping the
|
|
hook must not lose that."""
|
|
mw = ViewImageMiddleware()
|
|
stranded = ViewImageMiddleware._create_image_context_message([{"type": "text", "text": "stale"}])
|
|
request = _model_request([HumanMessage(content="hi"), stranded, AIMessage(content="done")])
|
|
|
|
prepared = mw._inject(request)
|
|
|
|
assert prepared is not request
|
|
assert _image_context_messages(prepared.messages) == []
|
|
assert [type(m) for m in prepared.messages] == [HumanMessage, AIMessage]
|
|
|
|
def test_never_drops_a_client_message_wearing_the_reserved_prefix(self):
|
|
"""The prefix alone is not enough — Gateway strips the server-owned
|
|
marker from client input, and both are required to match."""
|
|
mw = ViewImageMiddleware()
|
|
client_message = HumanMessage(id="view-image-context:client-supplied", content="client-authored", additional_kwargs={"hide_from_ui": True})
|
|
request = _model_request([client_message, AIMessage(content="done")])
|
|
|
|
assert mw._inject(request) is request
|
|
|
|
def test_does_not_mutate_the_incoming_request(self, tmp_path):
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1")])
|
|
request = _model_request(
|
|
[assistant, ToolMessage(content="ok", tool_call_id="c1")],
|
|
{"/img.png": _make_viewed_image(tmp_path)},
|
|
)
|
|
|
|
mw._inject(request)
|
|
|
|
assert _image_context_messages(request.messages) == []
|
|
|
|
|
|
class TestWrapModelCall:
|
|
def test_blob_backed_image_reaches_sync_model_without_writer_disk(self, monkeypatch):
|
|
request, expected = _blob_backed_request(monkeypatch)
|
|
seen: list[ModelRequest] = []
|
|
|
|
ViewImageMiddleware().wrap_model_call(
|
|
request,
|
|
lambda prepared: seen.append(prepared) or AIMessage(content="ok"),
|
|
)
|
|
|
|
assert _image_urls(seen[0]) == [expected]
|
|
|
|
def test_handler_receives_the_image_context_message(self, tmp_path):
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1")])
|
|
request = _model_request(
|
|
[assistant, ToolMessage(content="ok", tool_call_id="c1")],
|
|
{"/img.png": _make_viewed_image(tmp_path)},
|
|
)
|
|
seen: list[ModelRequest] = []
|
|
|
|
def handler(prepared: ModelRequest) -> AIMessage:
|
|
seen.append(prepared)
|
|
return AIMessage(content="I can see the image.")
|
|
|
|
result = mw.wrap_model_call(request, handler)
|
|
|
|
assert result.content == "I can see the image."
|
|
assert len(_image_context_messages(seen[0].messages)) == 1
|
|
|
|
def test_handler_receives_request_unchanged_when_not_warranted(self):
|
|
mw = ViewImageMiddleware()
|
|
request = _model_request([HumanMessage(content="hi")])
|
|
seen: list[ModelRequest] = []
|
|
|
|
mw.wrap_model_call(request, lambda prepared: seen.append(prepared) or AIMessage(content="ok"))
|
|
|
|
assert seen[0] is request
|
|
|
|
@pytest.mark.anyio
|
|
async def test_awrap_model_call_matches_sync_behavior(self, tmp_path):
|
|
mw = ViewImageMiddleware()
|
|
assistant = AIMessage(content="", tool_calls=[_view_image_call("c1")])
|
|
request = _model_request(
|
|
[assistant, ToolMessage(content="ok", tool_call_id="c1")],
|
|
{"/img.png": _make_viewed_image(tmp_path)},
|
|
)
|
|
seen: list[ModelRequest] = []
|
|
|
|
async def handler(prepared: ModelRequest) -> AIMessage:
|
|
seen.append(prepared)
|
|
return AIMessage(content="I can see the image.")
|
|
|
|
result = await mw.awrap_model_call(request, handler)
|
|
|
|
assert result.content == "I can see the image."
|
|
assert len(_image_context_messages(seen[0].messages)) == 1
|
|
|
|
@pytest.mark.anyio
|
|
async def test_blob_backed_image_reaches_async_model_without_writer_disk(self, monkeypatch):
|
|
from deerflow.authz import sandbox_authz
|
|
|
|
request, expected = _blob_backed_request(monkeypatch)
|
|
seen: list[ModelRequest] = []
|
|
|
|
async def authorize(**kwargs):
|
|
return None
|
|
|
|
monkeypatch.setattr(sandbox_authz, "authorize_sandbox_execution_async", authorize)
|
|
|
|
async def handler(prepared: ModelRequest) -> AIMessage:
|
|
seen.append(prepared)
|
|
return AIMessage(content="ok")
|
|
|
|
await ViewImageMiddleware().awrap_model_call(request, handler)
|
|
|
|
assert _image_urls(seen[0]) == [expected]
|
|
|
|
|
|
class TestGraphIntegration:
|
|
def _graph_and_capture(self):
|
|
capture = _CaptureChatMessages()
|
|
model = FakeMessagesListChatModel(
|
|
responses=[AIMessage(content="I can see the image.")],
|
|
callbacks=[capture],
|
|
)
|
|
return create_agent(model=model, tools=[], middleware=[ViewImageMiddleware()]), capture
|
|
|
|
def _input(self, tmp_path, monkeypatch):
|
|
virtual_path, image_metadata = _authorized_host_view(tmp_path, monkeypatch)
|
|
return {
|
|
"messages": [
|
|
AIMessage(content="", tool_calls=[_view_image_call("c1", virtual_path)]),
|
|
ToolMessage(content="ok", tool_call_id="c1"),
|
|
],
|
|
"viewed_images": {virtual_path: image_metadata},
|
|
}
|
|
|
|
def test_image_context_reaches_the_model_but_never_the_state(self, tmp_path, monkeypatch):
|
|
graph, capture = self._graph_and_capture()
|
|
|
|
result = graph.invoke(
|
|
self._input(tmp_path, monkeypatch),
|
|
config={"configurable": {"thread_id": "thread-test"}},
|
|
context={"user_id": "user-test"},
|
|
)
|
|
|
|
model_image_messages = _image_context_messages(capture.messages)
|
|
assert len(model_image_messages) == 1
|
|
assert any(block.get("type") == "image_url" for block in model_image_messages[0].content)
|
|
# Nothing is written back, so the payload is absent from every checkpoint
|
|
# rather than being added and then removed again.
|
|
assert _image_context_messages(result["messages"]) == []
|
|
|
|
@pytest.mark.anyio
|
|
async def test_async_graph_matches_sync_behavior(self, tmp_path, monkeypatch):
|
|
graph, capture = self._graph_and_capture()
|
|
|
|
result = await graph.ainvoke(
|
|
self._input(tmp_path, monkeypatch),
|
|
config={"configurable": {"thread_id": "thread-test"}},
|
|
context={"user_id": "user-test"},
|
|
)
|
|
|
|
assert len(_image_context_messages(capture.messages)) == 1
|
|
assert any(block.get("type") == "image_url" for block in _image_context_messages(capture.messages)[0].content)
|
|
assert _image_context_messages(result["messages"]) == []
|
|
|
|
def test_context_thread_id_cannot_override_configured_thread(self, tmp_path, monkeypatch):
|
|
graph, capture = self._graph_and_capture()
|
|
|
|
graph.invoke(
|
|
self._input(tmp_path, monkeypatch),
|
|
config={"configurable": {"thread_id": "another-thread"}},
|
|
context={"user_id": "user-test", "thread_id": "thread-test"},
|
|
)
|
|
|
|
model_image_messages = _image_context_messages(capture.messages)
|
|
assert len(model_image_messages) == 1
|
|
assert all(block.get("type") != "image_url" for block in model_image_messages[0].content)
|
|
|
|
def test_graph_preserves_normalized_client_message_with_reserved_prefix(self, tmp_path, monkeypatch):
|
|
from app.gateway.services import normalize_input
|
|
|
|
client_id = "view-image-context:client-supplied"
|
|
normalized = normalize_input(
|
|
{
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"id": client_id,
|
|
"content": "client-authored message",
|
|
"additional_kwargs": {
|
|
_IMAGE_CONTEXT_MESSAGE_MARKER_KEY: True,
|
|
"custom": "keep-me",
|
|
},
|
|
}
|
|
]
|
|
}
|
|
)
|
|
client_message = normalized["messages"][0]
|
|
assert _IMAGE_CONTEXT_MESSAGE_MARKER_KEY not in client_message.additional_kwargs
|
|
|
|
graph, capture = self._graph_and_capture()
|
|
graph_input = self._input(tmp_path, monkeypatch)
|
|
|
|
result = graph.invoke({**graph_input, "messages": [client_message, *graph_input["messages"]]}, context={"user_id": "user-test", "thread_id": "thread-test"})
|
|
|
|
assert any(message.id == client_id for message in capture.messages)
|
|
assert any(message.id != client_id and message.additional_kwargs.get(_IMAGE_CONTEXT_MESSAGE_MARKER_KEY) is True for message in _image_context_messages(capture.messages))
|
|
persisted_client = next(message for message in result["messages"] if message.id == client_id)
|
|
assert persisted_client.content == "client-authored message"
|
|
assert persisted_client.additional_kwargs == {"custom": "keep-me"}
|
|
assert all(message.additional_kwargs.get(_IMAGE_CONTEXT_MESSAGE_MARKER_KEY) is not True for message in result["messages"] if isinstance(message, HumanMessage))
|
|
|
|
|
|
def test_host_image_copy_is_scoped_to_current_user_and_thread(tmp_path, monkeypatch):
|
|
"""A stale viewed_images entry must not open another user's host copy."""
|
|
from deerflow.authz import sandbox_authz
|
|
|
|
monkeypatch.setattr(sandbox_authz, "authorize_sandbox_execution", lambda **kwargs: None)
|
|
paths = Paths(tmp_path)
|
|
monkeypatch.setattr("deerflow.config.paths.get_paths", lambda: paths)
|
|
monkeypatch.setattr("deerflow.sandbox.sandbox_provider.get_sandbox_provider", lambda: SimpleNamespace(get=lambda sandbox_id: None))
|
|
|
|
image_bytes = base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==")
|
|
virtual_path = "/mnt/user-data/outputs/canary.png"
|
|
owner_path = paths.sandbox_outputs_dir("thread-b", user_id="user-b") / "canary.png"
|
|
owner_path.parent.mkdir(parents=True)
|
|
owner_path.write_bytes(image_bytes)
|
|
metadata = {"mime_type": "image/png", "size": len(image_bytes), "actual_path": str(owner_path)}
|
|
messages = [AIMessage(content="", tool_calls=[_view_image_call("call-image", virtual_path)]), ToolMessage(content="Successfully read image", tool_call_id="call-image")]
|
|
|
|
def model_payloads(user_id: str, thread_id: str | None, actual_path: Path = owner_path) -> list[str]:
|
|
request = _model_request(messages, {virtual_path: {**metadata, "actual_path": str(actual_path)}})
|
|
request.runtime.context = {"user_id": user_id, "thread_id": thread_id}
|
|
prepared = ViewImageMiddleware()._inject(request)
|
|
return [block["image_url"]["url"] for message in _image_context_messages(prepared.messages) for block in message.content if isinstance(block, dict) and block.get("type") == "image_url"]
|
|
|
|
expected = "data:image/png;base64," + base64.b64encode(image_bytes).decode("ascii")
|
|
assert model_payloads("user-b", "thread-b") == [expected]
|
|
assert model_payloads("user-a", "thread-a") == []
|
|
assert model_payloads("user-b", "thread-a") == []
|
|
assert model_payloads("user-b", None) == []
|
|
|
|
alias_path = paths.sandbox_outputs_dir("thread-a", user_id="user-a") / "canary.png"
|
|
alias_path.parent.mkdir(parents=True)
|
|
symlink_or_skip(alias_path, owner_path)
|
|
assert model_payloads("user-a", "thread-a", alias_path) == []
|
|
|
|
|
|
def _custom_base_view_request(tmp_path, monkeypatch):
|
|
"""Use the real tool to record a host copy under a non-global Paths base."""
|
|
from deerflow.authz import sandbox_authz
|
|
from deerflow.tools.builtins.view_image_tool import _view_image_authorized
|
|
|
|
monkeypatch.setattr(sandbox_authz, "authorize_sandbox_execution", lambda **kwargs: None)
|
|
monkeypatch.setattr("deerflow.sandbox.sandbox_provider.get_sandbox_provider", lambda: SimpleNamespace(get=lambda sandbox_id: None))
|
|
|
|
paths = Paths(tmp_path / "custom-base")
|
|
virtual_path = "/mnt/user-data/outputs/canary.png"
|
|
image_bytes = base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==")
|
|
actual_path = paths.sandbox_outputs_dir("thread-test", user_id="user-test") / "canary.png"
|
|
actual_path.parent.mkdir(parents=True)
|
|
actual_path.write_bytes(image_bytes)
|
|
thread_data = {
|
|
"workspace_path": str(paths.sandbox_work_dir("thread-test", user_id="user-test")),
|
|
"uploads_path": str(paths.sandbox_uploads_dir("thread-test", user_id="user-test")),
|
|
"outputs_path": str(actual_path.parent),
|
|
}
|
|
tool_runtime = SimpleNamespace(
|
|
state={"thread_data": thread_data},
|
|
context={"user_id": "user-test", "thread_id": "thread-test"},
|
|
)
|
|
result = _view_image_authorized(tool_runtime, virtual_path, "call-image")
|
|
metadata = result.update["viewed_images"][virtual_path]
|
|
assert metadata["actual_path"] == str(actual_path)
|
|
|
|
messages = [
|
|
AIMessage(content="", tool_calls=[_view_image_call("call-image", virtual_path)]),
|
|
ToolMessage(content="Successfully read image", tool_call_id="call-image"),
|
|
]
|
|
request = _model_request(messages, {virtual_path: metadata})
|
|
request.state["thread_data"] = thread_data
|
|
request.runtime.context = tool_runtime.context
|
|
expected = "data:image/png;base64," + base64.b64encode(image_bytes).decode("ascii")
|
|
return request, expected
|
|
|
|
|
|
def _image_urls(request: ModelRequest) -> list[str]:
|
|
return [block["image_url"]["url"] for message in _image_context_messages(request.messages) for block in message.content if isinstance(block, dict) and block.get("type") == "image_url"]
|
|
|
|
|
|
def test_custom_base_host_copy_reaches_sync_model(tmp_path, monkeypatch):
|
|
request, expected = _custom_base_view_request(tmp_path, monkeypatch)
|
|
prepared: list[ModelRequest] = []
|
|
|
|
ViewImageMiddleware().wrap_model_call(request, lambda value: prepared.append(value) or AIMessage(content="ok"))
|
|
|
|
assert _image_urls(prepared[0]) == [expected]
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_custom_base_host_copy_reaches_async_model(tmp_path, monkeypatch):
|
|
request, expected = _custom_base_view_request(tmp_path, monkeypatch)
|
|
prepared: list[ModelRequest] = []
|
|
|
|
async def handler(value: ModelRequest) -> AIMessage:
|
|
prepared.append(value)
|
|
return AIMessage(content="ok")
|
|
|
|
await ViewImageMiddleware().awrap_model_call(request, handler)
|
|
|
|
assert _image_urls(prepared[0]) == [expected]
|
|
|
|
|
|
def test_custom_base_host_copy_still_requires_matching_user_and_thread(tmp_path, monkeypatch):
|
|
request, _ = _custom_base_view_request(tmp_path, monkeypatch)
|
|
middleware = ViewImageMiddleware()
|
|
|
|
for user_id, thread_id in (("other-user", "thread-test"), ("user-test", "other-thread"), ("user-test", None)):
|
|
request.runtime.context = {"user_id": user_id, "thread_id": thread_id}
|
|
assert _image_urls(middleware._inject(request)) == []
|
|
|
|
|
|
def test_custom_base_host_copy_rejects_unscoped_thread_data_root(tmp_path, monkeypatch):
|
|
request, _ = _custom_base_view_request(tmp_path, monkeypatch)
|
|
virtual_path = "/mnt/user-data/outputs/canary.png"
|
|
unscoped_path = tmp_path / "unscoped" / "canary.png"
|
|
unscoped_path.parent.mkdir()
|
|
unscoped_path.write_bytes(base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="))
|
|
request.state["thread_data"] = {**request.state["thread_data"], "outputs_path": str(unscoped_path.parent)}
|
|
request.state["viewed_images"][virtual_path] = {
|
|
**request.state["viewed_images"][virtual_path],
|
|
"actual_path": str(unscoped_path),
|
|
}
|
|
|
|
assert _image_urls(ViewImageMiddleware()._inject(request)) == []
|
|
|
|
|
|
def _text_blocks(blocks: list) -> list[str]:
|
|
return [block["text"] for block in blocks if isinstance(block, dict) and block.get("type") == "text"]
|
|
|
|
|
|
def test_injection_budget_keeps_newest_images_and_omits_oldest(tmp_path):
|
|
"""The image cap admits newest-first; omitted entries keep a placeholder bullet (see #6825)."""
|
|
viewed_images = {}
|
|
for i in range(4):
|
|
viewed_images[f"/mnt/user-data/outputs/img_{i}.png"] = _make_viewed_image(tmp_path, f"img_{i}.png", data=bytes([i]) + b"\x89PNG\r\n\x1a\n" + b"\x00" * 16)
|
|
state = {"viewed_images": viewed_images}
|
|
|
|
mw = ViewImageMiddleware(max_injected_images=2)
|
|
blocks = mw._create_image_details_message(state, host_path_allowed=lambda _, actual: Path(actual).is_relative_to(tmp_path))
|
|
|
|
image_urls = [b["image_url"]["url"] for b in blocks if isinstance(b, dict) and b.get("type") == "image_url"]
|
|
assert len(image_urls) == 2
|
|
|
|
# The two newest entries (img_2, img_3) are injected; the two oldest keep a placeholder.
|
|
# Walk the raw block list so each bullet's successor block is checked in place.
|
|
for i, block in enumerate(blocks):
|
|
if not (isinstance(block, dict) and block.get("type") == "text"):
|
|
continue
|
|
text = block["text"]
|
|
for omitted in ("img_0.png", "img_1.png"):
|
|
if omitted in text:
|
|
assert "omitted: image injection budget reached" in blocks[i + 1]["text"]
|
|
for injected in ("img_2.png", "img_3.png"):
|
|
if injected in text:
|
|
assert blocks[i + 1].get("type") == "image_url"
|
|
|
|
# Distinct payloads prove which entries made it.
|
|
assert base64.b64decode(image_urls[0].split(",", 1)[1]).startswith(bytes([2]))
|
|
assert base64.b64decode(image_urls[1].split(",", 1)[1]).startswith(bytes([3]))
|
|
|
|
|
|
def test_injection_byte_budget_drops_oldest_when_total_exceeded(tmp_path):
|
|
"""The byte cap evicts the oldest oversized entry while a newer small one still fits."""
|
|
big = b"\x89PNG\r\n\x1a\n" + b"\x00" * (900_000 - 8)
|
|
small = b"\x89PNG\r\n\x1a\n" + b"\x00" * (100_000 - 8)
|
|
state = {
|
|
"viewed_images": {
|
|
"/mnt/user-data/outputs/big_0.png": _make_viewed_image(tmp_path, "big_0.png", data=big),
|
|
"/mnt/user-data/outputs/big_1.png": _make_viewed_image(tmp_path, "big_1.png", data=big),
|
|
"/mnt/user-data/outputs/small_2.png": _make_viewed_image(tmp_path, "small_2.png", data=small),
|
|
}
|
|
}
|
|
|
|
mw = ViewImageMiddleware(max_injected_total_chars=2_000_000)
|
|
blocks = mw._create_image_details_message(state, host_path_allowed=lambda _, actual: Path(actual).is_relative_to(tmp_path))
|
|
|
|
image_urls = [b["image_url"]["url"] for b in blocks if isinstance(b, dict) and b.get("type") == "image_url"]
|
|
assert len(image_urls) == 2
|
|
for i, block in enumerate(blocks):
|
|
if not (isinstance(block, dict) and block.get("type") == "text"):
|
|
continue
|
|
if "big_0.png" in block["text"]:
|
|
assert "omitted: image injection budget reached" in blocks[i + 1]["text"]
|
|
if "small_2.png" in block["text"]:
|
|
assert blocks[i + 1].get("type") == "image_url"
|
|
|
|
|
|
def test_injection_budget_defaults_keep_small_threads_unbounded(tmp_path):
|
|
"""Defaults must not change threads whose image count and sizes stay small."""
|
|
viewed_images = {}
|
|
for i in range(3):
|
|
viewed_images[f"/mnt/user-data/outputs/img_{i}.png"] = _make_viewed_image(tmp_path, f"img_{i}.png")
|
|
state = {"viewed_images": viewed_images}
|
|
|
|
blocks = ViewImageMiddleware()._create_image_details_message(state, host_path_allowed=lambda _, actual: Path(actual).is_relative_to(tmp_path))
|
|
|
|
assert len([b for b in blocks if isinstance(b, dict) and b.get("type") == "image_url"]) == 3
|
|
assert not any("omitted" in t for t in _text_blocks(blocks))
|
|
|
|
|
|
def test_injection_budget_skips_reads_for_omitted_entries(tmp_path):
|
|
"""Entries over budget are never read from disk or sandbox."""
|
|
viewed_images = {}
|
|
for i in range(3):
|
|
viewed_images[f"/mnt/user-data/outputs/img_{i}.png"] = _make_viewed_image(tmp_path, f"img_{i}.png", data=bytes([i]) + b"\x89PNG\r\n\x1a\n" + b"\x00" * 16)
|
|
state = {"viewed_images": viewed_images}
|
|
|
|
mw = ViewImageMiddleware(max_injected_images=1)
|
|
calls = []
|
|
|
|
def _fake_read(*args, **kwargs):
|
|
calls.append(args)
|
|
return "data:image/png;base64,QUJD"
|
|
|
|
mw._read_image_as_data_url = _fake_read
|
|
blocks = mw._create_image_details_message(state)
|
|
|
|
assert len([b for b in blocks if isinstance(b, dict) and b.get("type") == "image_url"]) == 1
|
|
assert len(calls) == 1
|
|
|
|
|
|
def test_reviewing_omitted_image_rescues_it_from_budget(tmp_path):
|
|
"""Re-viewing an omitted image must move it to the newest slot so the
|
|
placeholder's recovery advice actually works (see #6829 review)."""
|
|
viewed_images = {}
|
|
for i in range(4):
|
|
viewed_images[f"/mnt/user-data/outputs/img_{i}.png"] = _make_viewed_image(tmp_path, f"img_{i}.png", data=bytes([i]) + b"\x89PNG\r\n\x1a\n" + b"\x00" * 16)
|
|
|
|
mw = ViewImageMiddleware(max_injected_images=2)
|
|
|
|
def host_ok(_, actual):
|
|
return Path(actual).is_relative_to(tmp_path)
|
|
|
|
blocks = mw._create_image_details_message({"viewed_images": viewed_images}, host_path_allowed=host_ok)
|
|
injected = [b for b in blocks if isinstance(b, dict) and b.get("type") == "image_url"]
|
|
assert len(injected) == 2 # img_2, img_3 admitted; img_0/img_1 omitted
|
|
|
|
# The model follows the placeholder advice and re-views img_0; the reducer
|
|
# re-inserts the updated key at the end, making it the newest entry.
|
|
updated = merge_viewed_images(
|
|
viewed_images,
|
|
{"/mnt/user-data/outputs/img_0.png": _make_viewed_image(tmp_path, "img_0.png", data=bytes([0]) + b"\x89PNG\r\n\x1a\n" + b"\x00" * 16)},
|
|
)
|
|
assert list(updated.keys())[-1] == "/mnt/user-data/outputs/img_0.png"
|
|
|
|
blocks = mw._create_image_details_message({"viewed_images": updated}, host_path_allowed=host_ok)
|
|
urls = [b["image_url"]["url"] for b in blocks if isinstance(b, dict) and b.get("type") == "image_url"]
|
|
assert len(urls) == 2
|
|
# Render order is oldest-first, so the admitted slots are img_3 then the
|
|
# re-viewed img_0; img_1/img_2 stay omitted.
|
|
payloads = [base64.b64decode(url.split(",", 1)[1]) for url in urls]
|
|
assert payloads[0].startswith(bytes([3]))
|
|
assert payloads[1].startswith(bytes([0]))
|