From bfa2794c7677d22d2a8097c55ffaeb2d00979c24 Mon Sep 17 00:00:00 2001 From: hdu_zhou <94913492+zuizhuai@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:52:01 +0800 Subject: [PATCH] fix(uploads): keep dedupe counter before compound archive suffix (#8042) Re-uploading a file whose name collides with a compound archive suffix (e.g. .tar.gz, .tar.bz2) previously inserted the dedupe counter inside the suffix: a second report.tar.gz became report.tar_1.gz, which broke file-type detection. Treat the compound suffix as a single extension so the counter lands before it (report_1.tar.gz), matching single-suffix behavior (report_1.tar, report_1.txt). - Add _COMPOUND_SUFFIXES and split stem/suffix as one unit, sorted longest-first with a closed, non-overlapping allowlist (.tar.br added). - Expand tests to all 13 suffixes plus counter-escalation and plain .tar. - Add a CHANGELOG [Unreleased] Fixed entry. Co-authored-by: Willem Jiang --- CHANGELOG.md | 6 ++ .../harness/deerflow/uploads/manager.py | 40 ++++++++++++- backend/tests/test_uploads_manager.py | 56 +++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62b6dbaf1a..407a1cd76e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,12 @@ This section accumulates work toward the **2.3.0** milestone they carry per-call environment variables, including Lark CLI commands. Commands within one subagent remain ordered and client cleanup drains them. +- **uploads:** Re-uploading a file whose name collides with a compound archive + suffix no longer inserts the dedupe counter inside the suffix. A second + `report.tar.gz` is now named `report_1.tar.gz` (matching `report_1.tar` and + `report_1.txt`), instead of `report.tar_1.gz`, so the extension stays intact + for file-type detection. ([#8042](https://github.com/bytedance/deer-flow/pull/8042)) + - **gateway:** Deleting a thread no longer reports success while its conversation stays readable. When the checkpoint store failed during `DELETE /api/threads/{id}`, the route logged it at DEBUG, removed the diff --git a/backend/packages/harness/deerflow/uploads/manager.py b/backend/packages/harness/deerflow/uploads/manager.py index ad55503b1c..5de08fa30c 100644 --- a/backend/packages/harness/deerflow/uploads/manager.py +++ b/backend/packages/harness/deerflow/uploads/manager.py @@ -42,6 +42,44 @@ UPLOAD_STAGING_MIN_AGE = timedelta(hours=24) _MAX_FILENAME_BYTES = 255 +# Multi-segment archive suffixes that must be treated as a single extension +# when deduplicating, so ``report.tar.gz`` becomes ``report_1.tar.gz`` instead +# of ``report.tar_1.gz``. Sorted longest-first so the longest match wins; the +# list is deliberately closed (not every compression format is listed), and no +# entry is a proper suffix of another, so a future addition (e.g. ``.tar.lzo`` +# beside ``.tar.lz``) stays safe regardless of position. +_COMPOUND_SUFFIXES = ( + ".tar.lzma", + ".tar.bz2", + ".tar.zst", + ".tar.lz4", + ".tar.br", + ".tar.gz", + ".tar.lz", + ".tar.xz", + ".tar.z", + ".tbz2", + ".tzst", + ".tgz", + ".txz", +) + + +def _split_stem_suffix(name: str) -> tuple[str, str]: + """Split *name* into ``(stem, suffix)``. + + Unlike :meth:`pathlib.Path.stem`/:meth:`pathlib.Path.suffix`, which only + strip the final extension, this keeps known compound archive suffixes + (``.tar.gz``, ``.tgz``, ...) intact so a deduplication counter lands + before the whole archive suffix. + """ + lowered = name.casefold() + for compound in _COMPOUND_SUFFIXES: + if lowered.endswith(compound): + return name[: -len(compound)], name[-len(compound) :] + path = Path(name) + return path.stem, path.suffix + def get_uploads_dir(thread_id: str, *, user_id: str | None = None) -> Path: """Return the uploads directory path for a thread (no side effects).""" @@ -122,7 +160,7 @@ def claim_unique_filename(name: str, seen: set[str]) -> str: if name.casefold() not in claimed: seen.add(name) return name - stem, suffix = Path(name).stem, Path(name).suffix + stem, suffix = _split_stem_suffix(name) counter = 1 while True: tag = f"_{counter}" diff --git a/backend/tests/test_uploads_manager.py b/backend/tests/test_uploads_manager.py index 9936a0fe80..c5932bfcd4 100644 --- a/backend/tests/test_uploads_manager.py +++ b/backend/tests/test_uploads_manager.py @@ -200,6 +200,62 @@ class TestDeduplicateFilename: seen = {"data.txt"} assert claim_unique_filename("data.txt", seen) == "data_1.txt" + @pytest.mark.parametrize( + ("name", "expected"), + [ + ("report.tar.gz", "report_1.tar.gz"), + ("report.tar.bz2", "report_1.tar.bz2"), + ("report.tar.xz", "report_1.tar.xz"), + ("report.tar.zst", "report_1.tar.zst"), + ("report.tar.lz", "report_1.tar.lz"), + ("report.tar.lz4", "report_1.tar.lz4"), + ("report.tar.lzma", "report_1.tar.lzma"), + ("report.tar.z", "report_1.tar.z"), + ("report.tar.br", "report_1.tar.br"), + ("report.tgz", "report_1.tgz"), + ("report.tbz2", "report_1.tbz2"), + ("report.txz", "report_1.txz"), + ("report.tzst", "report_1.tzst"), + ], + ) + def test_compound_archive_suffix_keeps_counter_before_suffix( + self, + name: str, + expected: str, + ): + """The ``_N`` counter lands before the whole archive suffix, not + inside it (``report.tar_1.gz`` is wrong).""" + seen = {name} + assert claim_unique_filename(name, seen) == expected + assert expected in seen + + def test_plain_tar_suffix_keeps_original_dedupe_shape(self): + """A plain ``.tar`` (single suffix, not in the compound list) still + dedupes as ``report_1.tar``.""" + seen = {"report.tar"} + assert claim_unique_filename("report.tar", seen) == "report_1.tar" + assert "report_1.tar" in seen + + def test_compound_suffix_counter_escalation(self): + """A second collision re-enters the compound split, producing + ``report_2.tar.gz`` rather than mangling the suffix.""" + seen = {"report.tar.gz", "report_1.tar.gz"} + assert claim_unique_filename("report.tar.gz", seen) == "report_2.tar.gz" + assert "report_2.tar.gz" in seen + + def test_compound_suffix_collision_is_case_insensitive(self): + seen = {"Report.Tar.Gz"} + assert claim_unique_filename("report.tar.gz", seen) == "report_1.tar.gz" + + def test_compound_suffix_max_length_stays_within_filename_limit(self): + # 250-byte stem + 7-byte ".tar.gz" = 257 bytes; dedupe must truncate + # the stem so the counter + suffix still fit in 255 bytes. + name = "a" * 250 + ".tar.gz" + seen = {name} + deduped = claim_unique_filename(name, seen) + assert deduped.endswith(".tar.gz") + assert len(deduped.encode("utf-8")) <= 255 + # --------------------------------------------------------------------------- # validate_path_traversal