mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-16 00:01:59 +00:00
* fix(gateway): reject artifact archives where a file collides with an implicit directory When a run outputs both a file and a deeper file path whose parent directory matches the first file's normalized name (e.g. 'Report' and 'report/slides/lesson.pptx', or Unicode NFC/NFD variations), the ZIP archive builder allowed both entries because it only checked for exact full path equality. On case-insensitive filesystems, extracting such archives fails because the file entry blocks directory creation or vice versa. This change collects all implicit directory prefixes from normalized casefolded entries and rejects the archive with HTTP 409 (ArtifactArchiveError) if any file entry name collides with an implicit directory, regardless of receipt order. Fixes #7977 * fix(gateway): address review feedback on test portability and loop readability
717 lines
23 KiB
Python
717 lines
23 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import io
|
|
import os
|
|
import threading
|
|
import zipfile
|
|
from pathlib import Path
|
|
from uuid import UUID
|
|
|
|
import pytest
|
|
from _router_auth_helpers import make_authed_test_app
|
|
from fastapi.testclient import TestClient
|
|
|
|
from app.gateway import artifact_archive
|
|
from app.gateway.auth.models import User
|
|
from app.gateway.routers import thread_runs
|
|
from deerflow.runtime.events.store.memory import MemoryRunEventStore
|
|
from deerflow.runtime.runs.manager import RunManager
|
|
from deerflow.runtime.runs.store.memory import MemoryRunStore
|
|
from deerflow.sandbox.exceptions import SandboxAuthorizationError
|
|
|
|
THREAD_ID = "thread-archive"
|
|
RUN_ID = "run-archive"
|
|
USER_ID = UUID("00000000-0000-0000-0000-000000000123")
|
|
ARCHIVE_URL = f"/api/threads/{THREAD_ID}/runs/{RUN_ID}/artifacts/archive"
|
|
|
|
|
|
class _FakePaths:
|
|
def __init__(self, outputs_dir: Path) -> None:
|
|
self._outputs_dir = outputs_dir
|
|
|
|
def sandbox_outputs_dir(self, _thread_id: str, *, user_id: str | None = None) -> Path:
|
|
return self._outputs_dir
|
|
|
|
def sandbox_user_data_dir(self, _thread_id: str, *, user_id: str | None = None) -> Path:
|
|
return self._outputs_dir.parent
|
|
|
|
|
|
def _user() -> User:
|
|
return User(
|
|
id=USER_ID,
|
|
email="archive-test@example.com",
|
|
password_hash="x",
|
|
system_role="user",
|
|
)
|
|
|
|
|
|
def _archive_app(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
outputs_dir: Path,
|
|
*,
|
|
paths: list[str] | None = None,
|
|
run_thread_id: str = THREAD_ID,
|
|
run_status: str = "success",
|
|
with_receipt: bool = True,
|
|
delivery_tool: str = "present_files",
|
|
) -> tuple[TestClient, MemoryRunStore, MemoryRunEventStore]:
|
|
run_store = MemoryRunStore()
|
|
event_store = MemoryRunEventStore()
|
|
run_manager = RunManager(store=run_store)
|
|
|
|
asyncio.run(
|
|
run_store.put(
|
|
RUN_ID,
|
|
thread_id=run_thread_id,
|
|
user_id=None,
|
|
status=run_status,
|
|
)
|
|
)
|
|
if with_receipt:
|
|
presented = paths or []
|
|
asyncio.run(
|
|
event_store.put(
|
|
thread_id=THREAD_ID,
|
|
run_id=RUN_ID,
|
|
event_type="run.delivery",
|
|
category="outputs",
|
|
content={
|
|
"presented": len(presented),
|
|
"paths": presented,
|
|
"by_tool": {delivery_tool: presented},
|
|
},
|
|
)
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
thread_runs,
|
|
"get_paths",
|
|
lambda: _FakePaths(outputs_dir),
|
|
raising=False,
|
|
)
|
|
|
|
app = make_authed_test_app(user_factory=_user)
|
|
app.state.run_store = run_store
|
|
app.state.run_event_store = event_store
|
|
app.state.run_manager = run_manager
|
|
app.include_router(thread_runs.router)
|
|
return TestClient(app), run_store, event_store
|
|
|
|
|
|
def test_archive_download_contains_only_presented_files(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
(outputs / "reports").mkdir(parents=True)
|
|
(outputs / "reports" / "summary.txt").write_text("summary", encoding="utf-8")
|
|
# Written as bytes on purpose: text mode translates "\n" to "\r\n" on
|
|
# Windows, so the archive would legitimately contain CRLF bytes while the
|
|
# assertion below compares against this literal LF spelling.
|
|
(outputs / "data.csv").write_bytes(b"a,b\n1,2\n")
|
|
(outputs / "not-presented.txt").write_text("secret", encoding="utf-8")
|
|
paths = [
|
|
"/mnt/user-data/outputs/reports/summary.txt",
|
|
"/mnt/user-data/outputs/data.csv",
|
|
"/mnt/user-data/outputs/data.csv",
|
|
]
|
|
client, _, _ = _archive_app(monkeypatch, outputs, paths=paths)
|
|
|
|
with client:
|
|
response = client.post(
|
|
ARCHIVE_URL,
|
|
json={"paths": ["/mnt/user-data/outputs/not-presented.txt"]},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.headers["content-type"] == "application/zip"
|
|
assert response.headers["cache-control"] == "private, no-store"
|
|
assert response.headers["x-content-type-options"] == "nosniff"
|
|
assert "attachment" in response.headers["content-disposition"]
|
|
with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
|
|
assert archive.namelist() == ["reports/summary.txt", "data.csv"]
|
|
assert archive.read("reports/summary.txt") == b"summary"
|
|
assert archive.read("data.csv") == b"a,b\n1,2\n"
|
|
assert "not-presented.txt" not in archive.namelist()
|
|
|
|
|
|
@pytest.mark.parametrize("delivery_tool", ["present_files", "present_shared_artifact"])
|
|
def test_archive_manifest_counts_only_verified_delivery_paths(tmp_path, monkeypatch, delivery_tool: str) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
client, _, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
paths=[
|
|
"/mnt/user-data/outputs/report.txt",
|
|
"/mnt/user-data/outputs/data.csv",
|
|
"/mnt/user-data/outputs/data.csv",
|
|
],
|
|
delivery_tool=delivery_tool,
|
|
)
|
|
|
|
with client:
|
|
response = client.get(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json() == {"file_count": 2}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"presented_path",
|
|
[
|
|
"/mnt/user-data/uploads/private.txt",
|
|
"/mnt/user-data/outputs/../uploads/private.txt",
|
|
"/mnt/user-data/outputs/.tool-results/raw.txt",
|
|
"/mnt/user-data/outputs/.browser-frames/frame.png",
|
|
],
|
|
)
|
|
def test_archive_rejects_paths_outside_public_outputs(
|
|
tmp_path,
|
|
monkeypatch,
|
|
presented_path: str,
|
|
) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
client, _, _ = _archive_app(monkeypatch, outputs, paths=[presented_path])
|
|
|
|
with client:
|
|
response = client.post(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 409
|
|
assert response.json()["detail"] == "The files listed by this response are not available for archive download"
|
|
|
|
|
|
def test_archive_rejects_a_directory_without_recursing(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
folder = outputs / "site"
|
|
folder.mkdir(parents=True)
|
|
(folder / "index.html").write_text("hidden descendant", encoding="utf-8")
|
|
client, _, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
paths=["/mnt/user-data/outputs/site"],
|
|
)
|
|
|
|
with client:
|
|
response = client.post(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 409
|
|
|
|
|
|
def test_archive_rejects_a_symlink(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
outside = tmp_path / "outside.txt"
|
|
outside.write_text("outside", encoding="utf-8")
|
|
link = outputs / "linked.txt"
|
|
try:
|
|
link.symlink_to(outside)
|
|
except OSError:
|
|
pytest.skip("symlinks are unavailable on this platform")
|
|
client, _, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
paths=["/mnt/user-data/outputs/linked.txt"],
|
|
)
|
|
|
|
with client:
|
|
response = client.post(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 409
|
|
|
|
|
|
def test_archive_rejects_a_symlinked_outputs_root(tmp_path, monkeypatch) -> None:
|
|
user_data = tmp_path / "user-data"
|
|
user_data.mkdir()
|
|
outside = tmp_path / "outside"
|
|
outside.mkdir()
|
|
(outside / "secret.txt").write_text("outside", encoding="utf-8")
|
|
outputs = user_data / "outputs"
|
|
try:
|
|
outputs.symlink_to(outside, target_is_directory=True)
|
|
except OSError:
|
|
pytest.skip("symlinks are unavailable on this platform")
|
|
client, _, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
paths=["/mnt/user-data/outputs/secret.txt"],
|
|
)
|
|
|
|
with client:
|
|
response = client.post(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 409
|
|
|
|
|
|
def test_archive_rejects_a_hard_linked_member(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
internal = outputs / ".tool-results"
|
|
internal.mkdir(parents=True)
|
|
secret = internal / "raw-secret.txt"
|
|
secret.write_text("internal", encoding="utf-8")
|
|
public_alias = outputs / "report.txt"
|
|
try:
|
|
public_alias.hardlink_to(secret)
|
|
except OSError:
|
|
pytest.skip("hard links are unavailable on this platform")
|
|
client, _, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
paths=["/mnt/user-data/outputs/report.txt"],
|
|
)
|
|
|
|
with client:
|
|
response = client.post(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 409
|
|
|
|
|
|
@pytest.mark.parametrize("junction_relative", [Path(), Path("site")])
|
|
def test_archive_rejects_junction_like_roots_and_components(
|
|
tmp_path,
|
|
monkeypatch,
|
|
junction_relative: Path,
|
|
) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
target = outputs / "site" / "report.txt"
|
|
target.parent.mkdir(parents=True)
|
|
target.write_text("report", encoding="utf-8")
|
|
junction = outputs / junction_relative
|
|
monkeypatch.setattr(Path, "is_junction", lambda self: self == junction)
|
|
|
|
with pytest.raises(artifact_archive.ArtifactArchiveError):
|
|
artifact_archive.build_artifact_archive(
|
|
outputs,
|
|
["/mnt/user-data/outputs/site/report.txt"],
|
|
user_data_dir=outputs.parent,
|
|
)
|
|
|
|
|
|
def test_archive_rejects_missing_or_nonterminal_delivery(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
client, _, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
run_status="running",
|
|
with_receipt=False,
|
|
)
|
|
|
|
with client:
|
|
response = client.post(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 409
|
|
|
|
|
|
@pytest.mark.parametrize("method", ["get", "post"])
|
|
def test_archive_hides_a_run_from_another_thread(tmp_path, monkeypatch, method: str) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
client, _, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
run_thread_id="another-thread",
|
|
paths=["/mnt/user-data/outputs/report.txt"],
|
|
)
|
|
|
|
with client:
|
|
response = getattr(client, method)(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 404
|
|
|
|
|
|
def test_archive_returns_503_when_sandbox_config_is_unavailable(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
client, _, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
paths=["/mnt/user-data/outputs/report.txt"],
|
|
)
|
|
|
|
async def unavailable():
|
|
raise SandboxAuthorizationError("Sandbox execution is temporarily unavailable")
|
|
|
|
monkeypatch.setattr(thread_runs, "safe_app_config_async", unavailable)
|
|
|
|
with client:
|
|
response = client.post(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 503
|
|
assert response.json()["detail"] == "Artifact archive is temporarily unavailable"
|
|
|
|
|
|
def test_archive_conflicts_with_an_active_run(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
(outputs / "report.txt").write_text("report", encoding="utf-8")
|
|
client, run_store, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
paths=["/mnt/user-data/outputs/report.txt"],
|
|
)
|
|
asyncio.run(
|
|
run_store.put(
|
|
"active-run",
|
|
thread_id=THREAD_ID,
|
|
user_id=None,
|
|
status="running",
|
|
)
|
|
)
|
|
|
|
with client:
|
|
response = client.post(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 409
|
|
|
|
|
|
def test_archive_rejects_when_the_worker_is_at_capacity(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
(outputs / "report.txt").write_text("report", encoding="utf-8")
|
|
client, _, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
paths=["/mnt/user-data/outputs/report.txt"],
|
|
)
|
|
slots = asyncio.Semaphore(1)
|
|
asyncio.run(slots.acquire())
|
|
monkeypatch.setattr(thread_runs, "_artifact_archive_slots", slots, raising=False)
|
|
|
|
with client:
|
|
response = client.post(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 429
|
|
|
|
|
|
def test_archive_rejects_casefolded_entry_collisions(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
upper = outputs / "Report.txt"
|
|
lower = outputs / "report.txt"
|
|
upper.write_text("upper", encoding="utf-8")
|
|
lower.write_text("lower", encoding="utf-8")
|
|
if upper.samefile(lower):
|
|
pytest.skip("case-sensitive filenames are unavailable on this filesystem")
|
|
client, _, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
paths=[
|
|
"/mnt/user-data/outputs/Report.txt",
|
|
"/mnt/user-data/outputs/report.txt",
|
|
],
|
|
)
|
|
|
|
with client:
|
|
response = client.post(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 409
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"order",
|
|
[
|
|
["/mnt/user-data/outputs/Report", "/mnt/user-data/outputs/report/slides/lesson.pptx"],
|
|
["/mnt/user-data/outputs/report/slides/lesson.pptx", "/mnt/user-data/outputs/Report"],
|
|
],
|
|
)
|
|
def test_archive_rejects_file_colliding_with_implicit_directory(tmp_path, monkeypatch, order) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
file_entry = outputs / "Report"
|
|
file_entry.write_text("file content", encoding="utf-8")
|
|
lower_report = outputs / "report"
|
|
if lower_report.exists() and file_entry.samefile(lower_report):
|
|
pytest.skip("case-sensitive filenames are unavailable on this filesystem")
|
|
nested_dir = outputs / "report" / "slides"
|
|
try:
|
|
nested_dir.mkdir(parents=True)
|
|
except (NotADirectoryError, FileExistsError):
|
|
pytest.skip("case-sensitive filenames are unavailable on this filesystem")
|
|
nested_entry = nested_dir / "lesson.pptx"
|
|
nested_entry.write_text("slides content", encoding="utf-8")
|
|
|
|
client, _, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
paths=order,
|
|
)
|
|
|
|
with client:
|
|
response = client.post(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 409
|
|
|
|
|
|
def test_archive_rejects_unicode_nfc_nfd_implicit_directory_collision(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
nfc_name = "Caf\u00e9"
|
|
nfd_dir = "Cafe\u0301"
|
|
file_entry = outputs / nfc_name
|
|
file_entry.write_text("cafe content", encoding="utf-8")
|
|
nfd_path = outputs / nfd_dir
|
|
if nfd_path.exists() and file_entry.samefile(nfd_path):
|
|
pytest.skip("normalization-sensitive filenames are unavailable on this filesystem")
|
|
nested_dir = outputs / nfd_dir
|
|
try:
|
|
nested_dir.mkdir(parents=True)
|
|
except (NotADirectoryError, FileExistsError):
|
|
pytest.skip("normalization-sensitive filenames are unavailable on this filesystem")
|
|
nested_entry = nested_dir / "menu.txt"
|
|
nested_entry.write_text("menu content", encoding="utf-8")
|
|
|
|
client, _, _ = _archive_app(
|
|
monkeypatch,
|
|
outputs,
|
|
paths=[
|
|
f"/mnt/user-data/outputs/{nfc_name}",
|
|
f"/mnt/user-data/outputs/{nfd_dir}/menu.txt",
|
|
],
|
|
)
|
|
|
|
with client:
|
|
response = client.post(ARCHIVE_URL)
|
|
|
|
assert response.status_code == 409
|
|
|
|
|
|
def test_archive_rechecks_size_after_open(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
artifact = outputs / "report.txt"
|
|
artifact.write_bytes(b"x")
|
|
original_member = artifact_archive._member
|
|
|
|
def grow_after_validation(*args, **kwargs):
|
|
member = original_member(*args, **kwargs)
|
|
artifact.write_bytes(b"12345")
|
|
return member
|
|
|
|
monkeypatch.setattr(artifact_archive, "MAX_FILE_BYTES", 4)
|
|
monkeypatch.setattr(artifact_archive, "MAX_TOTAL_BYTES", 8)
|
|
monkeypatch.setattr(artifact_archive, "_member", grow_after_validation)
|
|
|
|
with pytest.raises(artifact_archive.ArtifactArchiveError) as exc_info:
|
|
artifact_archive.build_artifact_archive(
|
|
outputs,
|
|
["/mnt/user-data/outputs/report.txt"],
|
|
user_data_dir=outputs.parent,
|
|
)
|
|
|
|
assert exc_info.value.status_code == 413
|
|
|
|
|
|
def test_archive_enforces_file_count_and_total_size_limits(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
paths = []
|
|
for name in ("one.txt", "two.txt"):
|
|
(outputs / name).write_bytes(b"x")
|
|
paths.append(f"/mnt/user-data/outputs/{name}")
|
|
|
|
monkeypatch.setattr(artifact_archive, "MAX_FILES", 1)
|
|
with pytest.raises(artifact_archive.ArtifactArchiveError) as count_error:
|
|
artifact_archive.build_artifact_archive(outputs, paths, user_data_dir=outputs.parent)
|
|
assert count_error.value.status_code == 413
|
|
|
|
monkeypatch.setattr(artifact_archive, "MAX_FILES", 2)
|
|
monkeypatch.setattr(artifact_archive, "MAX_TOTAL_BYTES", 1)
|
|
with pytest.raises(artifact_archive.ArtifactArchiveError) as size_error:
|
|
artifact_archive.build_artifact_archive(outputs, paths, user_data_dir=outputs.parent)
|
|
assert size_error.value.status_code == 413
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("relative_path", "reserved"),
|
|
[
|
|
(".ARTIFACT-EDIT-draft", set()),
|
|
("private-cache/result.txt", {"private-cache"}),
|
|
],
|
|
)
|
|
def test_archive_rejects_internal_output_names(
|
|
tmp_path,
|
|
relative_path: str,
|
|
reserved: set[str],
|
|
) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
target = outputs / relative_path
|
|
target.parent.mkdir(parents=True)
|
|
target.write_text("internal", encoding="utf-8")
|
|
|
|
with pytest.raises(artifact_archive.ArtifactArchiveError):
|
|
artifact_archive.build_artifact_archive(
|
|
outputs,
|
|
[f"/mnt/user-data/outputs/{relative_path}"],
|
|
user_data_dir=outputs.parent,
|
|
extra_reserved_dir_names=reserved,
|
|
)
|
|
|
|
|
|
# The swap the test performs is os.replace() while the archive still holds the
|
|
# source file open. Windows refuses that rename with WinError 5, so the race is
|
|
# only reproducible on POSIX; the same-size content change covered by
|
|
# test_archive_rejects_same_size_content_change_with_restored_mtime still runs
|
|
# on Windows and exercises the same mid-read revalidation.
|
|
@pytest.mark.skipif(os.name == "nt", reason="os.replace() cannot rename over a file the archive still holds open on Windows (WinError 5)")
|
|
def test_archive_rejects_a_path_replaced_during_read(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
artifact = outputs / "report.txt"
|
|
artifact.write_bytes(b"old-content")
|
|
replacement = outputs / "replacement.txt"
|
|
replacement.write_bytes(b"new-content")
|
|
original_read = artifact_archive.os.read
|
|
replaced = False
|
|
|
|
def replace_after_read(descriptor: int, count: int) -> bytes:
|
|
nonlocal replaced
|
|
data = original_read(descriptor, count)
|
|
if not replaced:
|
|
replaced = True
|
|
replacement.replace(artifact)
|
|
return data
|
|
|
|
monkeypatch.setattr(artifact_archive.os, "read", replace_after_read)
|
|
|
|
with pytest.raises(artifact_archive.ArtifactArchiveError):
|
|
artifact_archive.build_artifact_archive(
|
|
outputs,
|
|
["/mnt/user-data/outputs/report.txt"],
|
|
user_data_dir=outputs.parent,
|
|
)
|
|
|
|
|
|
def test_archive_rejects_same_size_content_change_with_restored_mtime(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
artifact = outputs / "report.bin"
|
|
chunk_size = artifact_archive._CHUNK_BYTES
|
|
artifact.write_bytes(b"A" * (chunk_size * 2))
|
|
original_mtime_ns = artifact.stat().st_mtime_ns
|
|
original_read = artifact_archive.os.read
|
|
reads = 0
|
|
|
|
def change_middle_chunk_then_restore(descriptor: int, count: int) -> bytes:
|
|
nonlocal reads
|
|
data = original_read(descriptor, count)
|
|
reads += 1
|
|
if reads == 1:
|
|
with artifact.open("r+b") as stream:
|
|
stream.seek(chunk_size)
|
|
stream.write(b"B" * chunk_size)
|
|
os.utime(artifact, ns=(artifact.stat().st_atime_ns, original_mtime_ns))
|
|
elif reads == 2:
|
|
with artifact.open("r+b") as stream:
|
|
stream.seek(chunk_size)
|
|
stream.write(b"A" * chunk_size)
|
|
os.utime(artifact, ns=(artifact.stat().st_atime_ns, original_mtime_ns))
|
|
return data
|
|
|
|
monkeypatch.setattr(artifact_archive.os, "read", change_middle_chunk_then_restore)
|
|
|
|
with pytest.raises(artifact_archive.ArtifactArchiveError):
|
|
artifact_archive.build_artifact_archive(
|
|
outputs,
|
|
["/mnt/user-data/outputs/report.bin"],
|
|
user_data_dir=outputs.parent,
|
|
)
|
|
|
|
|
|
def test_archive_deadline_applies_to_empty_files(tmp_path, monkeypatch) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
(outputs / "empty.txt").touch()
|
|
monkeypatch.setattr(artifact_archive, "BUILD_TIMEOUT_SECONDS", -1)
|
|
|
|
with pytest.raises(artifact_archive.ArtifactArchiveError) as exc_info:
|
|
artifact_archive.build_artifact_archive(
|
|
outputs,
|
|
["/mnt/user-data/outputs/empty.txt"],
|
|
user_data_dir=outputs.parent,
|
|
)
|
|
|
|
assert exc_info.value.status_code == 503
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"filename",
|
|
[
|
|
"report.txt.",
|
|
"C:report.txt",
|
|
"CON.txt",
|
|
"report<draft>.txt",
|
|
'report"draft.txt',
|
|
"report|draft.txt",
|
|
"report?.txt",
|
|
"report*.txt",
|
|
],
|
|
)
|
|
def test_archive_rejects_nonportable_zip_names(tmp_path, filename: str) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
target = outputs / filename
|
|
try:
|
|
target.write_text("report", encoding="utf-8")
|
|
except OSError:
|
|
pytest.skip("the platform cannot create this nonportable filename")
|
|
|
|
with pytest.raises(artifact_archive.ArtifactArchiveError):
|
|
artifact_archive.build_artifact_archive(
|
|
outputs,
|
|
[f"/mnt/user-data/outputs/{filename}"],
|
|
user_data_dir=outputs.parent,
|
|
)
|
|
|
|
|
|
def test_archive_allows_emoji_joiner_sequences(tmp_path) -> None:
|
|
outputs = tmp_path / "outputs"
|
|
outputs.mkdir()
|
|
filename = "report-🧑💻.txt"
|
|
(outputs / filename).write_text("report", encoding="utf-8")
|
|
|
|
result = artifact_archive.build_artifact_archive(
|
|
outputs,
|
|
[f"/mnt/user-data/outputs/{filename}"],
|
|
user_data_dir=outputs.parent,
|
|
)
|
|
|
|
with result.file, zipfile.ZipFile(result.file) as archive:
|
|
assert archive.namelist() == [filename]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_repeated_cancellation_keeps_the_archive_slot_until_worker_exit(monkeypatch) -> None:
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
|
|
def blocking_build(*_args, **_kwargs):
|
|
started.set()
|
|
release.wait(timeout=5)
|
|
return artifact_archive.ArtifactArchiveResult(io.BytesIO(), 0, 0, 0)
|
|
|
|
slots = asyncio.Semaphore(1)
|
|
monkeypatch.setattr(thread_runs, "_artifact_archive_slots", slots)
|
|
monkeypatch.setattr(thread_runs, "build_artifact_archive", blocking_build)
|
|
task = asyncio.create_task(
|
|
thread_runs._build_archive_without_abandoning_worker(
|
|
Path("unused"),
|
|
Path("unused-parent"),
|
|
[],
|
|
extra_reserved_dir_names=set(),
|
|
)
|
|
)
|
|
assert await asyncio.to_thread(started.wait, 1)
|
|
|
|
task.cancel()
|
|
await asyncio.sleep(0)
|
|
task.cancel()
|
|
done, _ = await asyncio.wait({task}, timeout=0.1)
|
|
|
|
try:
|
|
assert not done
|
|
assert slots.locked()
|
|
finally:
|
|
release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|