fix(video): preserve reference image MIME types (#7340)

* fix(video): preserve reference image MIME types

* fix(video): reject unrecognized reference image extensions

* fix(tests): register video generation guidance

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>

README.md

@@ -1637,6 +1637,12 @@ Skills are what make DeerFlow do *almost anything*.
 
 A standard Agent Skill is a structured capability module — a Markdown file that defines a workflow, best practices, and references to supporting resources. DeerFlow ships with built-in skills for research, report generation, slide creation, web pages, image and video generation, and more. But the real power is extensibility: add your own skills, replace the built-in ones, or combine them into compound workflows.
 
+The video-generation skill derives reference-image MIME metadata from `.png`,
+`.jpg`, `.jpeg`, `.webp`, and `.gif` filename extensions (case-insensitive). A
+submitted reference with a missing or unrecognized extension raises a clear error
+before API submission. Filenames must match their contents; image bytes are not
+validated or converted, and provider-specific format restrictions still apply.
+
 The [music-generation skill](skills/public/music-generation/SKILL.md#output-handling) preserves existing output when the provider returns malformed, empty or unrecognized audio.
 
 Skills are loaded progressively — only when the task needs them, not all at once. This keeps the context window lean and makes DeerFlow work well even with token-sensitive models.

backend/tests/AGENTS.md

@@ -1,5 +1,9 @@
 # Backend Tests
 
+When adding a scoped `AGENTS.md`, update `EXPECTED_GUIDANCE_PATHS` in
+`test_agent_guidance_check.py` in the same change. The explicit inventory pins
+guidance ownership and extends hard-budget checks to each approved guide.
+
 `test_image_result_url_safety.py` drives GroundRoute through `ainvoke` and an
 offline HTTPX transport. Patch only the provider's `httpx` reference; preserve
 the shared client and exercise real configuration resolution and URL screening.

backend/tests/test_agent_guidance_check.py

@@ -10,6 +10,7 @@ CHECKER_PATH = REPO_ROOT / "scripts" / "check_agent_guidance.py"
 EXPECTED_GUIDANCE_PATHS = {
     "AGENTS.md",
     "skills/public/data-analysis/AGENTS.md",
+    "skills/public/video-generation/AGENTS.md",
     "backend/AGENTS.md",
     "backend/tests/AGENTS.md",
     "frontend/AGENTS.md",

skills/public/video-generation/AGENTS.md

@@ -0,0 +1,11 @@
+# Video Generation Maintenance
+
+[scripts/generate.py](scripts/generate.py) shares extension-based MIME inference between Gemini references
+and MiniMax first-frame data URLs. Reject unmapped suffixes with a `ValueError`
+that names the file and supported extensions. Preserve case-insensitive PNG,
+JPEG, WebP and GIF metadata without decoding or transcoding images.
+
+Keep provider format acceptance separate from MIME recognition. The public-skill
+reference-MIME regressions mock HTTP calls, verify request metadata and exact input
+bytes for both providers, and reject unknown extensions before submission.
+Preserve the layout of unrelated script code.

skills/public/video-generation/SKILL.md

@@ -51,7 +51,7 @@ python /mnt/skills/public/video-generation/scripts/generate.py \
 Parameters:
 
 - `--prompt-file`: Absolute path to JSON prompt file (required)
-- `--reference-images`: Absolute paths to reference image (optional)
+- `--reference-images`: Absolute paths to reference images (optional). Images used by the provider must have a recognized filename extension: `.png`, `.jpg`, `.jpeg`, `.webp`, or `.gif` (case-insensitive). Missing or other extensions are rejected before submission. Filenames must match their contents: the script neither validates image contents nor converts bytes. Recognizing an extension does not guarantee that the selected provider accepts that format.
 - `--output-file`: Absolute path to output image file (required)
 - `--aspect-ratio`: Aspect ratio of the generated image (optional, default: 16:9)
 

skills/public/video-generation/scripts/generate.py

@@ -39,13 +39,18 @@ def _check_base_resp(payload: dict) -> None:
 
 def _guess_mime(image_path: str) -> str:
     ext = os.path.splitext(image_path)[1].lower()
-    return {
+    mime_types = {
         ".png": "image/png",
         ".webp": "image/webp",
         ".gif": "image/gif",
         ".jpg": "image/jpeg",
         ".jpeg": "image/jpeg",
-    }.get(ext, "image/jpeg")
+    }
+    try:
+        return mime_types[ext]
+    except KeyError:
+        supported = ", ".join(mime_types)
+        raise ValueError(f"Unsupported reference image extension for {image_path!r}. Supported extensions: {supported}") from None
 
 
 def _to_data_url(image_path: str) -> str:
@@ -143,7 +148,7 @@ def _generate_video_gemini(prompt: str, reference_images: list[str], output_file
     for reference_image in reference_images:
         with open(reference_image, "rb") as f:
             image_b64 = base64.b64encode(f.read()).decode("utf-8")
-        reference_payload.append({"image": {"mimeType": "image/jpeg", "bytesBase64Encoded": image_b64}, "referenceType": "asset"})
+        reference_payload.append({"image": {"mimeType": _guess_mime(reference_image), "bytesBase64Encoded": image_b64}, "referenceType": "asset"})
     if reference_payload:
         request_json["instances"][0]["referenceImages"] = reference_payload
     gemini_api_key = os.getenv("GEMINI_API_KEY")

tests/skills/test_video_generation_reference_mime.py

@@ -0,0 +1,132 @@
+"""Verify extension-based reference metadata at offline provider boundaries."""
+
+import base64
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from skill_loader import FakeResp, load  # noqa: E402
+
+vid = load("video-generation")
+
+
+def _mock_provider(monkeypatch, provider):
+    monkeypatch.setenv("VIDEO_GENERATION_PROVIDER", provider)
+    monkeypatch.setenv("GEMINI_API_KEY", "offline-test-key")
+    monkeypatch.setenv("MINIMAX_API_KEY", "offline-test-key")
+    monkeypatch.setenv("MINIMAX_API_HOST", "https://api.example.test")
+    captured = {"posts": [], "gets": []}
+
+    def fake_post(url, *, json, **kwargs):
+        captured["posts"].append(json)
+        if provider == "gemini":
+            return FakeResp({"name": "operations/offline-test"})
+        return FakeResp({"task_id": "offline-task", "base_resp": {"status_code": 0}})
+
+    def fake_get(url, **kwargs):
+        captured["gets"].append(url)
+        if url.endswith("/operations/offline-test"):
+            return FakeResp({"done": True, "response": {"generateVideoResponse": {"generatedSamples": [{"video": {"uri": "https://example.test/video.mp4"}}]}}})
+        if url.endswith("/v1/query/video_generation"):
+            return FakeResp({"status": "Success", "file_id": "offline-file", "base_resp": {"status_code": 0}})
+        if url.endswith("/v1/files/retrieve"):
+            return FakeResp({"file": {"download_url": "https://example.test/video.mp4"}, "base_resp": {"status_code": 0}})
+        assert url == "https://example.test/video.mp4"
+        return FakeResp(content=b"offline-video-bytes")
+
+    monkeypatch.setattr(vid.requests, "post", fake_post)
+    monkeypatch.setattr(vid.requests, "get", fake_get)
+    return captured
+
+
+@pytest.mark.parametrize("provider", ["gemini", "minimax"])
+@pytest.mark.parametrize(
+    "images",
+    [
+        pytest.param([("reference.png", b"\x89PNG\r\n\x1a\nreference", "image/png")], id="png"),
+        pytest.param([("reference.PNG", b"\x89PNG\r\n\x1a\nreference", "image/png")], id="uppercase-png"),
+        pytest.param([("reference.jpg", b"\xff\xd8\xffreference", "image/jpeg")], id="jpg"),
+        pytest.param([("reference.jpeg", b"\xff\xd8\xffreference", "image/jpeg")], id="jpeg"),
+        pytest.param([("reference.JPG", b"\xff\xd8\xffreference", "image/jpeg")], id="uppercase-jpg"),
+        pytest.param([("reference.webp", b"RIFF\x00\x00\x00\x00WEBPreference", "image/webp")], id="webp"),
+        pytest.param([("reference.WEBP", b"RIFF\x00\x00\x00\x00WEBPreference", "image/webp")], id="uppercase-webp"),
+        pytest.param([("reference.gif", b"GIF89areference", "image/gif")], id="gif"),
+        pytest.param([("reference.GIF", b"GIF89areference", "image/gif")], id="uppercase-gif"),
+        pytest.param(
+            [
+                ("first.jpg", b"\xff\xd8\xfffirst", "image/jpeg"),
+                ("second.png", b"\x89PNG\r\n\x1a\nsecond", "image/png"),
+            ],
+            id="mixed-references",
+        ),
+    ],
+)
+def test_provider_preserves_reference_image_mime(monkeypatch, tmp_path, provider, images):
+    captured = _mock_provider(monkeypatch, provider)
+    prompt = tmp_path / "prompt.json"
+    prompt.write_text("a cat runs", encoding="utf-8")
+    reference_paths = []
+    for filename, image_bytes, _ in images:
+        reference = tmp_path / filename
+        reference.write_bytes(image_bytes)
+        reference_paths.append(str(reference))
+    output = tmp_path / "video.mp4"
+
+    result = vid.generate_video(str(prompt), reference_paths, str(output))
+
+    assert len(captured["posts"]) == 1
+    request = captured["posts"][0]
+    if provider == "gemini":
+        references = request["instances"][0]["referenceImages"]
+        assert len(references) == len(images)
+        for reference, (_, image_bytes, expected_mime) in zip(references, images, strict=True):
+            assert reference["referenceType"] == "asset"
+            assert reference["image"]["mimeType"] == expected_mime
+            assert base64.b64decode(reference["image"]["bytesBase64Encoded"]) == image_bytes
+    else:
+        # MiniMax uses only the first reference, as documented by the skill.
+        prefix, encoded = request["first_frame_image"].split(",", 1)
+        assert prefix == f"data:{images[0][2]};base64"
+        assert base64.b64decode(encoded) == images[0][1]
+    assert output.read_bytes() == b"offline-video-bytes"
+    assert "successfully" in result
+
+
+@pytest.mark.parametrize("provider", ["gemini", "minimax"])
+@pytest.mark.parametrize("filename", ["reference", "reference.bmp", "reference.BMP", "reference.avif"])
+def test_unsupported_reference_extension_rejected_before_submission(monkeypatch, tmp_path, provider, filename):
+    captured = _mock_provider(monkeypatch, provider)
+    prompt = tmp_path / "prompt.json"
+    prompt.write_text("a cat runs", encoding="utf-8")
+    reference = tmp_path / filename
+    reference.write_bytes(b"unsupported-image-bytes")
+    output = tmp_path / "video.mp4"
+
+    with pytest.raises(ValueError) as caught:
+        vid.generate_video(str(prompt), [str(reference)], str(output))
+
+    message = str(caught.value)
+    assert reference.name in message
+    for extension in (".png", ".jpg", ".jpeg", ".webp", ".gif"):
+        assert extension in message
+    assert captured == {"posts": [], "gets": []}
+    assert not output.exists()
+
+
+def test_gemini_rejects_later_unsupported_reference_before_submission(monkeypatch, tmp_path):
+    captured = _mock_provider(monkeypatch, "gemini")
+    prompt = tmp_path / "prompt.json"
+    prompt.write_text("a cat runs", encoding="utf-8")
+    first = tmp_path / "first.png"
+    first.write_bytes(b"\x89PNG\r\n\x1a\nfirst")
+    unsupported = tmp_path / "second.bmp"
+    unsupported.write_bytes(b"unsupported-image-bytes")
+    output = tmp_path / "video.mp4"
+
+    with pytest.raises(ValueError, match="second.bmp"):
+        vid.generate_video(str(prompt), [str(first), str(unsupported)], str(output))
+
+    assert captured == {"posts": [], "gets": []}
+    assert not output.exists()