From eb5d1e1880f221051957f797d8119b075348fcb3 Mon Sep 17 00:00:00 2001 From: Hyeonsang Cho Date: Sat, 31 Oct 2026 20:16:50 +0900 Subject: [PATCH] fix(projects): keep referenced originals when the trash sweep walks a symlinked tree (#7732) * fix(projects): keep referenced originals when the trash sweep walks a symlinked tree The storage reconciliation protects each row's namespace by its resolved path (project_document_path resolves symlinks) but walked the unresolved documents directory. With a symlinked .deer-flow (no DEER_FLOW_HOME), users/, users/{id}/ or projects/, no walked directory matched a protected namespace, so the startup sweep and the lazy trash-listing sweep unlinked live document originals older than 24 hours as orphans. Walk each project's resolved documents directory so both sides compare the same paths. * docs(changelog): reference #7732 for the trash sweep symlink fix --------- Co-authored-by: Willem Jiang --- CHANGELOG.md | 8 +++++ CHANGELOG_zh.md | 6 ++++ .../harness/deerflow/projects/trash.py | 8 ++++- backend/tests/test_project_trash.py | 36 +++++++++++++++++++ 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efcce2433e..c9e6c2d7ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,13 @@ This section accumulates work toward the **2.3.0** milestone ### Fixed +- **projects:** The trash retention sweep no longer deletes live document + originals when the data directory, `users/`, a user's directory or a + user's `projects/` is a symlink. It protected each document by its resolved + path but walked the unresolved tree, so no document matched and every + original older than 24 hours was removed as an orphan, at Gateway startup + and from `GET /api/trash/documents`. ([#7732]) + - **dev:** `scripts/dev_multi_instance.sh logs all` now prints the nginx logs as well as both Gateway logs; it printed only the Gateway logs, the same as `logs` with no argument. `logs nginx` and `logs all` also include `nginx.log`, @@ -15949,3 +15956,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7723]: https://github.com/bytedance/deer-flow/pull/7723 [#7727]: https://github.com/bytedance/deer-flow/pull/7727 [#7729]: https://github.com/bytedance/deer-flow/pull/7729 +[#7732]: https://github.com/bytedance/deer-flow/pull/7732 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index cd994c6d08..b240372e8e 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -20,6 +20,11 @@ ### 修复 +- **项目：** 当数据目录、`users/`、某个用户目录或用户的 `projects/` 是符号链接时， + 回收站保留期清扫不再删除仍在使用的文档原件。此前它按解析后的路径保护每个文档， + 却遍历未解析的目录树，导致没有任何文档能匹配上，所有超过 24 小时的原件都被当作 + 孤立文件删除，Gateway 启动时和 `GET /api/trash/documents` 都会触发。([#7732]) + - **开发：** `scripts/dev_multi_instance.sh logs all` 现在除两个 Gateway 日志外也会输出 nginx 日志；此前它只输出 Gateway 日志，与不带参数的 `logs` 相同。`logs nginx` 与 `logs all` 还会包含 `nginx.log`（nginx 启动命令自身的输出），此前没有任何选项会输出它。 @@ -12713,3 +12718,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#7723]: https://github.com/bytedance/deer-flow/pull/7723 [#7727]: https://github.com/bytedance/deer-flow/pull/7727 [#7729]: https://github.com/bytedance/deer-flow/pull/7729 +[#7732]: https://github.com/bytedance/deer-flow/pull/7732 diff --git a/backend/packages/harness/deerflow/projects/trash.py b/backend/packages/harness/deerflow/projects/trash.py index d976c765b2..b39c7dea6a 100644 --- a/backend/packages/harness/deerflow/projects/trash.py +++ b/backend/packages/harness/deerflow/projects/trash.py @@ -200,6 +200,8 @@ def _sweep_project_documents_dir(documents_dir: Path, *, protected: set[Path], c 24-hour guard; a row's namespace (active or trashed, even after restore re-pointed it to another project) is protected in full (§8.3). Empty parents are rmdir'd best-effort. Nothing younger is ever collected. + ``documents_dir`` must be resolved, like the protected namespaces it is + compared against. """ staging = documents_dir / ".staging" if staging.is_dir(): @@ -266,7 +268,11 @@ def _reconcile_storage(paths: Paths, *, user_id: str | None, rows: list[dict], g for project_dir in sorted(projects_root.iterdir()): documents_dir = project_dir / "documents" if documents_dir.is_dir(): - _sweep_project_documents_dir(documents_dir, protected=protected_by_user.get(uid, set()), cutoff_ts=cutoff_ts, report=report) + # Protected namespaces come from ``project_document_path``, which + # resolves symlinks. Walking an unresolved root through a + # symlinked ``.deer-flow``, ``users/`` or ``projects/`` would match + # none of them and collect live originals as orphans. + _sweep_project_documents_dir(documents_dir.resolve(), protected=protected_by_user.get(uid, set()), cutoff_ts=cutoff_ts, report=report) def _parse_iso(value: object) -> datetime | None: diff --git a/backend/tests/test_project_trash.py b/backend/tests/test_project_trash.py index f5499c0136..a7162c328b 100644 --- a/backend/tests/test_project_trash.py +++ b/backend/tests/test_project_trash.py @@ -526,6 +526,42 @@ class TestRetentionSweep: assert not ghost_ns.parent.exists() assert documents_dir.is_dir() + @pytest.mark.parametrize("sweep_user", [_USER, None], ids=["lazy", "startup"]) + @pytest.mark.parametrize("linked", [".deer-flow", "users", f"users/{_USER}", f"users/{_USER}/projects"]) + async def test_symlinked_storage_ancestor_keeps_referenced_originals(self, env, monkeypatch, tmp_path, linked, sweep_user): + """§8.3: a row protects its namespace even when the data root, the + users tree or a user's projects tree is a symlink. Protected + namespaces are resolved, so the walk must be too; an unreferenced + file past the guard is still collected through the link.""" + target = tmp_path / "elsewhere" + target.mkdir() + if linked == ".deer-flow": + # Without DEER_FLOW_HOME the data root is the unresolved + # ``/.deer-flow``. + checkout = tmp_path / "checkout" + checkout.mkdir() + monkeypatch.delenv("DEER_FLOW_HOME") + monkeypatch.setenv("DEER_FLOW_PROJECT_ROOT", str(checkout)) + link = checkout / ".deer-flow" + else: + link = env.paths.base_dir / linked + link.parent.mkdir(parents=True, exist_ok=True) + symlink_or_skip(link, target, target_is_directory=True) + project = await env.projects.create(name="P", user_id=_USER) + row = await _add(env, project["id"], name="keep.txt", data=b"keep") + kept = original_file_path(env.paths, user_id=_USER, row=row) + _age(kept, hours=25) + orphan = env.paths.project_documents_dir(_USER, project["id"]) / "ab" / ("ab" * 32) / "ghost" / "original" / "old.bin" + orphan.parent.mkdir(parents=True) + orphan.write_bytes(b"x") + _age(orphan, hours=25) + + report = await run_trash_retention_sweep(env.docs, env.paths, retention_days=30, user_id=sweep_user) + + assert kept.read_bytes() == b"keep" + assert report.orphans_removed == 1 + assert not orphan.exists() + async def test_row_reconciliation_detects_logs_and_never_deletes(self, env, caplog): """§8.3/§15.17: a row whose content vanished past the 24h guard is surfaced as content_missing (log + report) but the row is retained —