From 2055991eb2bf258cbd421d621668f2b5d55ff842 Mon Sep 17 00:00:00 2001 From: MsNBBc3R <152378630+MsNBBc3R@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:34:14 +0800 Subject: [PATCH] fix(video): preserve reference image MIME types (#7340) * fix(video): preserve reference image MIME types * fix(video): reject unrecognized reference image extensions * fix(tests): register video generation guidance --------- Co-authored-by: Willem Jiang --- README.md | 6 + backend/tests/AGENTS.md | 4 + backend/tests/test_agent_guidance_check.py | 1 + skills/public/video-generation/AGENTS.md | 11 ++ skills/public/video-generation/SKILL.md | 2 +- .../video-generation/scripts/generate.py | 11 +- .../test_video_generation_reference_mime.py | 132 ++++++++++++++++++ 7 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 skills/public/video-generation/AGENTS.md create mode 100644 tests/skills/test_video_generation_reference_mime.py diff --git a/README.md b/README.md index 5bc645b30e..48dd5ec450 100644 --- a/README.md +++ b/README.md @@ -1637,6 +1637,12 @@ Skills are what make DeerFlow do *almost anything*. A standard Agent Skill is a structured capability module — a Markdown file that defines a workflow, best practices, and references to supporting resources. DeerFlow ships with built-in skills for research, report generation, slide creation, web pages, image and video generation, and more. But the real power is extensibility: add your own skills, replace the built-in ones, or combine them into compound workflows. +The video-generation skill derives reference-image MIME metadata from `.png`, +`.jpg`, `.jpeg`, `.webp`, and `.gif` filename extensions (case-insensitive). A +submitted reference with a missing or unrecognized extension raises a clear error +before API submission. Filenames must match their contents; image bytes are not +validated or converted, and provider-specific format restrictions still apply. + The [music-generation skill](skills/public/music-generation/SKILL.md#output-handling) preserves existing output when the provider returns malformed, empty or unrecognized audio. Skills are loaded progressively — only when the task needs them, not all at once. This keeps the context window lean and makes DeerFlow work well even with token-sensitive models. diff --git a/backend/tests/AGENTS.md b/backend/tests/AGENTS.md index 5734c9e1b1..d6d62e31d6 100644 --- a/backend/tests/AGENTS.md +++ b/backend/tests/AGENTS.md @@ -1,5 +1,9 @@ # Backend Tests +When adding a scoped `AGENTS.md`, update `EXPECTED_GUIDANCE_PATHS` in +`test_agent_guidance_check.py` in the same change. The explicit inventory pins +guidance ownership and extends hard-budget checks to each approved guide. + `test_image_result_url_safety.py` drives GroundRoute through `ainvoke` and an offline HTTPX transport. Patch only the provider's `httpx` reference; preserve the shared client and exercise real configuration resolution and URL screening. diff --git a/backend/tests/test_agent_guidance_check.py b/backend/tests/test_agent_guidance_check.py index 110693d9ca..f7c365fe9a 100644 --- a/backend/tests/test_agent_guidance_check.py +++ b/backend/tests/test_agent_guidance_check.py @@ -10,6 +10,7 @@ CHECKER_PATH = REPO_ROOT / "scripts" / "check_agent_guidance.py" EXPECTED_GUIDANCE_PATHS = { "AGENTS.md", "skills/public/data-analysis/AGENTS.md", + "skills/public/video-generation/AGENTS.md", "backend/AGENTS.md", "backend/tests/AGENTS.md", "frontend/AGENTS.md", diff --git a/skills/public/video-generation/AGENTS.md b/skills/public/video-generation/AGENTS.md new file mode 100644 index 0000000000..cfa0e286c0 --- /dev/null +++ b/skills/public/video-generation/AGENTS.md @@ -0,0 +1,11 @@ +# Video Generation Maintenance + +[scripts/generate.py](scripts/generate.py) shares extension-based MIME inference between Gemini references +and MiniMax first-frame data URLs. Reject unmapped suffixes with a `ValueError` +that names the file and supported extensions. Preserve case-insensitive PNG, +JPEG, WebP and GIF metadata without decoding or transcoding images. + +Keep provider format acceptance separate from MIME recognition. The public-skill +reference-MIME regressions mock HTTP calls, verify request metadata and exact input +bytes for both providers, and reject unknown extensions before submission. +Preserve the layout of unrelated script code. diff --git a/skills/public/video-generation/SKILL.md b/skills/public/video-generation/SKILL.md index 242ec467b1..7e1640e8c9 100644 --- a/skills/public/video-generation/SKILL.md +++ b/skills/public/video-generation/SKILL.md @@ -51,7 +51,7 @@ python /mnt/skills/public/video-generation/scripts/generate.py \ Parameters: - `--prompt-file`: Absolute path to JSON prompt file (required) -- `--reference-images`: Absolute paths to reference image (optional) +- `--reference-images`: Absolute paths to reference images (optional). Images used by the provider must have a recognized filename extension: `.png`, `.jpg`, `.jpeg`, `.webp`, or `.gif` (case-insensitive). Missing or other extensions are rejected before submission. Filenames must match their contents: the script neither validates image contents nor converts bytes. Recognizing an extension does not guarantee that the selected provider accepts that format. - `--output-file`: Absolute path to output image file (required) - `--aspect-ratio`: Aspect ratio of the generated image (optional, default: 16:9) diff --git a/skills/public/video-generation/scripts/generate.py b/skills/public/video-generation/scripts/generate.py index 696decf5ec..182bda70a3 100644 --- a/skills/public/video-generation/scripts/generate.py +++ b/skills/public/video-generation/scripts/generate.py @@ -39,13 +39,18 @@ def _check_base_resp(payload: dict) -> None: def _guess_mime(image_path: str) -> str: ext = os.path.splitext(image_path)[1].lower() - return { + mime_types = { ".png": "image/png", ".webp": "image/webp", ".gif": "image/gif", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", - }.get(ext, "image/jpeg") + } + try: + return mime_types[ext] + except KeyError: + supported = ", ".join(mime_types) + raise ValueError(f"Unsupported reference image extension for {image_path!r}. Supported extensions: {supported}") from None def _to_data_url(image_path: str) -> str: @@ -143,7 +148,7 @@ def _generate_video_gemini(prompt: str, reference_images: list[str], output_file for reference_image in reference_images: with open(reference_image, "rb") as f: image_b64 = base64.b64encode(f.read()).decode("utf-8") - reference_payload.append({"image": {"mimeType": "image/jpeg", "bytesBase64Encoded": image_b64}, "referenceType": "asset"}) + reference_payload.append({"image": {"mimeType": _guess_mime(reference_image), "bytesBase64Encoded": image_b64}, "referenceType": "asset"}) if reference_payload: request_json["instances"][0]["referenceImages"] = reference_payload gemini_api_key = os.getenv("GEMINI_API_KEY") diff --git a/tests/skills/test_video_generation_reference_mime.py b/tests/skills/test_video_generation_reference_mime.py new file mode 100644 index 0000000000..0f11241947 --- /dev/null +++ b/tests/skills/test_video_generation_reference_mime.py @@ -0,0 +1,132 @@ +"""Verify extension-based reference metadata at offline provider boundaries.""" + +import base64 +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from skill_loader import FakeResp, load # noqa: E402 + +vid = load("video-generation") + + +def _mock_provider(monkeypatch, provider): + monkeypatch.setenv("VIDEO_GENERATION_PROVIDER", provider) + monkeypatch.setenv("GEMINI_API_KEY", "offline-test-key") + monkeypatch.setenv("MINIMAX_API_KEY", "offline-test-key") + monkeypatch.setenv("MINIMAX_API_HOST", "https://api.example.test") + captured = {"posts": [], "gets": []} + + def fake_post(url, *, json, **kwargs): + captured["posts"].append(json) + if provider == "gemini": + return FakeResp({"name": "operations/offline-test"}) + return FakeResp({"task_id": "offline-task", "base_resp": {"status_code": 0}}) + + def fake_get(url, **kwargs): + captured["gets"].append(url) + if url.endswith("/operations/offline-test"): + return FakeResp({"done": True, "response": {"generateVideoResponse": {"generatedSamples": [{"video": {"uri": "https://example.test/video.mp4"}}]}}}) + if url.endswith("/v1/query/video_generation"): + return FakeResp({"status": "Success", "file_id": "offline-file", "base_resp": {"status_code": 0}}) + if url.endswith("/v1/files/retrieve"): + return FakeResp({"file": {"download_url": "https://example.test/video.mp4"}, "base_resp": {"status_code": 0}}) + assert url == "https://example.test/video.mp4" + return FakeResp(content=b"offline-video-bytes") + + monkeypatch.setattr(vid.requests, "post", fake_post) + monkeypatch.setattr(vid.requests, "get", fake_get) + return captured + + +@pytest.mark.parametrize("provider", ["gemini", "minimax"]) +@pytest.mark.parametrize( + "images", + [ + pytest.param([("reference.png", b"\x89PNG\r\n\x1a\nreference", "image/png")], id="png"), + pytest.param([("reference.PNG", b"\x89PNG\r\n\x1a\nreference", "image/png")], id="uppercase-png"), + pytest.param([("reference.jpg", b"\xff\xd8\xffreference", "image/jpeg")], id="jpg"), + pytest.param([("reference.jpeg", b"\xff\xd8\xffreference", "image/jpeg")], id="jpeg"), + pytest.param([("reference.JPG", b"\xff\xd8\xffreference", "image/jpeg")], id="uppercase-jpg"), + pytest.param([("reference.webp", b"RIFF\x00\x00\x00\x00WEBPreference", "image/webp")], id="webp"), + pytest.param([("reference.WEBP", b"RIFF\x00\x00\x00\x00WEBPreference", "image/webp")], id="uppercase-webp"), + pytest.param([("reference.gif", b"GIF89areference", "image/gif")], id="gif"), + pytest.param([("reference.GIF", b"GIF89areference", "image/gif")], id="uppercase-gif"), + pytest.param( + [ + ("first.jpg", b"\xff\xd8\xfffirst", "image/jpeg"), + ("second.png", b"\x89PNG\r\n\x1a\nsecond", "image/png"), + ], + id="mixed-references", + ), + ], +) +def test_provider_preserves_reference_image_mime(monkeypatch, tmp_path, provider, images): + captured = _mock_provider(monkeypatch, provider) + prompt = tmp_path / "prompt.json" + prompt.write_text("a cat runs", encoding="utf-8") + reference_paths = [] + for filename, image_bytes, _ in images: + reference = tmp_path / filename + reference.write_bytes(image_bytes) + reference_paths.append(str(reference)) + output = tmp_path / "video.mp4" + + result = vid.generate_video(str(prompt), reference_paths, str(output)) + + assert len(captured["posts"]) == 1 + request = captured["posts"][0] + if provider == "gemini": + references = request["instances"][0]["referenceImages"] + assert len(references) == len(images) + for reference, (_, image_bytes, expected_mime) in zip(references, images, strict=True): + assert reference["referenceType"] == "asset" + assert reference["image"]["mimeType"] == expected_mime + assert base64.b64decode(reference["image"]["bytesBase64Encoded"]) == image_bytes + else: + # MiniMax uses only the first reference, as documented by the skill. + prefix, encoded = request["first_frame_image"].split(",", 1) + assert prefix == f"data:{images[0][2]};base64" + assert base64.b64decode(encoded) == images[0][1] + assert output.read_bytes() == b"offline-video-bytes" + assert "successfully" in result + + +@pytest.mark.parametrize("provider", ["gemini", "minimax"]) +@pytest.mark.parametrize("filename", ["reference", "reference.bmp", "reference.BMP", "reference.avif"]) +def test_unsupported_reference_extension_rejected_before_submission(monkeypatch, tmp_path, provider, filename): + captured = _mock_provider(monkeypatch, provider) + prompt = tmp_path / "prompt.json" + prompt.write_text("a cat runs", encoding="utf-8") + reference = tmp_path / filename + reference.write_bytes(b"unsupported-image-bytes") + output = tmp_path / "video.mp4" + + with pytest.raises(ValueError) as caught: + vid.generate_video(str(prompt), [str(reference)], str(output)) + + message = str(caught.value) + assert reference.name in message + for extension in (".png", ".jpg", ".jpeg", ".webp", ".gif"): + assert extension in message + assert captured == {"posts": [], "gets": []} + assert not output.exists() + + +def test_gemini_rejects_later_unsupported_reference_before_submission(monkeypatch, tmp_path): + captured = _mock_provider(monkeypatch, "gemini") + prompt = tmp_path / "prompt.json" + prompt.write_text("a cat runs", encoding="utf-8") + first = tmp_path / "first.png" + first.write_bytes(b"\x89PNG\r\n\x1a\nfirst") + unsupported = tmp_path / "second.bmp" + unsupported.write_bytes(b"unsupported-image-bytes") + output = tmp_path / "video.mp4" + + with pytest.raises(ValueError, match="second.bmp"): + vid.generate_video(str(prompt), [str(first), str(unsupported)], str(output)) + + assert captured == {"posts": [], "gets": []} + assert not output.exists()