mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-12 14:00:35 +00:00
fix(uploads): keep dedupe counter before compound archive suffix (#8042)
Re-uploading a file whose name collides with a compound archive suffix (e.g. .tar.gz, .tar.bz2) previously inserted the dedupe counter inside the suffix: a second report.tar.gz became report.tar_1.gz, which broke file-type detection. Treat the compound suffix as a single extension so the counter lands before it (report_1.tar.gz), matching single-suffix behavior (report_1.tar, report_1.txt). - Add _COMPOUND_SUFFIXES and split stem/suffix as one unit, sorted longest-first with a closed, non-overlapping allowlist (.tar.br added). - Expand tests to all 13 suffixes plus counter-escalation and plain .tar. - Add a CHANGELOG [Unreleased] Fixed entry. Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
parent
54753636b1
commit
bfa2794c76
@ -58,6 +58,12 @@ This section accumulates work toward the **2.3.0** milestone
|
||||
they carry per-call environment variables, including Lark CLI commands.
|
||||
Commands within one subagent remain ordered and client cleanup drains them.
|
||||
|
||||
- **uploads:** Re-uploading a file whose name collides with a compound archive
|
||||
suffix no longer inserts the dedupe counter inside the suffix. A second
|
||||
`report.tar.gz` is now named `report_1.tar.gz` (matching `report_1.tar` and
|
||||
`report_1.txt`), instead of `report.tar_1.gz`, so the extension stays intact
|
||||
for file-type detection. ([#8042](https://github.com/bytedance/deer-flow/pull/8042))
|
||||
|
||||
- **gateway:** Deleting a thread no longer reports success while its
|
||||
conversation stays readable. When the checkpoint store failed during
|
||||
`DELETE /api/threads/{id}`, the route logged it at DEBUG, removed the
|
||||
|
||||
@ -42,6 +42,44 @@ UPLOAD_STAGING_MIN_AGE = timedelta(hours=24)
|
||||
|
||||
_MAX_FILENAME_BYTES = 255
|
||||
|
||||
# Multi-segment archive suffixes that must be treated as a single extension
|
||||
# when deduplicating, so ``report.tar.gz`` becomes ``report_1.tar.gz`` instead
|
||||
# of ``report.tar_1.gz``. Sorted longest-first so the longest match wins; the
|
||||
# list is deliberately closed (not every compression format is listed), and no
|
||||
# entry is a proper suffix of another, so a future addition (e.g. ``.tar.lzo``
|
||||
# beside ``.tar.lz``) stays safe regardless of position.
|
||||
_COMPOUND_SUFFIXES = (
|
||||
".tar.lzma",
|
||||
".tar.bz2",
|
||||
".tar.zst",
|
||||
".tar.lz4",
|
||||
".tar.br",
|
||||
".tar.gz",
|
||||
".tar.lz",
|
||||
".tar.xz",
|
||||
".tar.z",
|
||||
".tbz2",
|
||||
".tzst",
|
||||
".tgz",
|
||||
".txz",
|
||||
)
|
||||
|
||||
|
||||
def _split_stem_suffix(name: str) -> tuple[str, str]:
|
||||
"""Split *name* into ``(stem, suffix)``.
|
||||
|
||||
Unlike :meth:`pathlib.Path.stem`/:meth:`pathlib.Path.suffix`, which only
|
||||
strip the final extension, this keeps known compound archive suffixes
|
||||
(``.tar.gz``, ``.tgz``, ...) intact so a deduplication counter lands
|
||||
before the whole archive suffix.
|
||||
"""
|
||||
lowered = name.casefold()
|
||||
for compound in _COMPOUND_SUFFIXES:
|
||||
if lowered.endswith(compound):
|
||||
return name[: -len(compound)], name[-len(compound) :]
|
||||
path = Path(name)
|
||||
return path.stem, path.suffix
|
||||
|
||||
|
||||
def get_uploads_dir(thread_id: str, *, user_id: str | None = None) -> Path:
|
||||
"""Return the uploads directory path for a thread (no side effects)."""
|
||||
@ -122,7 +160,7 @@ def claim_unique_filename(name: str, seen: set[str]) -> str:
|
||||
if name.casefold() not in claimed:
|
||||
seen.add(name)
|
||||
return name
|
||||
stem, suffix = Path(name).stem, Path(name).suffix
|
||||
stem, suffix = _split_stem_suffix(name)
|
||||
counter = 1
|
||||
while True:
|
||||
tag = f"_{counter}"
|
||||
|
||||
@ -200,6 +200,62 @@ class TestDeduplicateFilename:
|
||||
seen = {"data.txt"}
|
||||
assert claim_unique_filename("data.txt", seen) == "data_1.txt"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "expected"),
|
||||
[
|
||||
("report.tar.gz", "report_1.tar.gz"),
|
||||
("report.tar.bz2", "report_1.tar.bz2"),
|
||||
("report.tar.xz", "report_1.tar.xz"),
|
||||
("report.tar.zst", "report_1.tar.zst"),
|
||||
("report.tar.lz", "report_1.tar.lz"),
|
||||
("report.tar.lz4", "report_1.tar.lz4"),
|
||||
("report.tar.lzma", "report_1.tar.lzma"),
|
||||
("report.tar.z", "report_1.tar.z"),
|
||||
("report.tar.br", "report_1.tar.br"),
|
||||
("report.tgz", "report_1.tgz"),
|
||||
("report.tbz2", "report_1.tbz2"),
|
||||
("report.txz", "report_1.txz"),
|
||||
("report.tzst", "report_1.tzst"),
|
||||
],
|
||||
)
|
||||
def test_compound_archive_suffix_keeps_counter_before_suffix(
|
||||
self,
|
||||
name: str,
|
||||
expected: str,
|
||||
):
|
||||
"""The ``_N`` counter lands before the whole archive suffix, not
|
||||
inside it (``report.tar_1.gz`` is wrong)."""
|
||||
seen = {name}
|
||||
assert claim_unique_filename(name, seen) == expected
|
||||
assert expected in seen
|
||||
|
||||
def test_plain_tar_suffix_keeps_original_dedupe_shape(self):
|
||||
"""A plain ``.tar`` (single suffix, not in the compound list) still
|
||||
dedupes as ``report_1.tar``."""
|
||||
seen = {"report.tar"}
|
||||
assert claim_unique_filename("report.tar", seen) == "report_1.tar"
|
||||
assert "report_1.tar" in seen
|
||||
|
||||
def test_compound_suffix_counter_escalation(self):
|
||||
"""A second collision re-enters the compound split, producing
|
||||
``report_2.tar.gz`` rather than mangling the suffix."""
|
||||
seen = {"report.tar.gz", "report_1.tar.gz"}
|
||||
assert claim_unique_filename("report.tar.gz", seen) == "report_2.tar.gz"
|
||||
assert "report_2.tar.gz" in seen
|
||||
|
||||
def test_compound_suffix_collision_is_case_insensitive(self):
|
||||
seen = {"Report.Tar.Gz"}
|
||||
assert claim_unique_filename("report.tar.gz", seen) == "report_1.tar.gz"
|
||||
|
||||
def test_compound_suffix_max_length_stays_within_filename_limit(self):
|
||||
# 250-byte stem + 7-byte ".tar.gz" = 257 bytes; dedupe must truncate
|
||||
# the stem so the counter + suffix still fit in 255 bytes.
|
||||
name = "a" * 250 + ".tar.gz"
|
||||
seen = {name}
|
||||
deduped = claim_unique_filename(name, seen)
|
||||
assert deduped.endswith(".tar.gz")
|
||||
assert len(deduped.encode("utf-8")) <= 255
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# validate_path_traversal
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user