diff --git a/README.md b/README.md index a844f5f58f..bc78d1f7fa 100644 --- a/README.md +++ b/README.md @@ -2452,6 +2452,11 @@ follow ordinary upload filename validation. Names containing NUL, Windows reserved device names (such as `CON.txt`), or trailing dots are rejected with `400` before bytes are staged. +On native Windows, the document shelf uses extended-length filesystem paths +so deep workspace directories and long filenames can be uploaded, downloaded, +restored, and purged without enabling the system-wide long-path setting. +Filesystem limits on individual path components still apply. + Runs on member threads also receive a bounded `` index rendered per run from the pinned snapshot (capped by `projects.shelf_index_max_entries` and `projects.shelf_index_max_bytes`), and the agent can page the shelf and read diff --git a/backend/packages/harness/deerflow/config/AGENTS.md b/backend/packages/harness/deerflow/config/AGENTS.md index 247a1bccc3..2ed1ebda12 100644 --- a/backend/packages/harness/deerflow/config/AGENTS.md +++ b/backend/packages/harness/deerflow/config/AGENTS.md @@ -1,5 +1,14 @@ ### Configuration System +`Paths.user_projects_dir()` uses `extended_length_path()` on native Windows. +All document paths, including staging and retention walks, inherit the same +extended drive/UNC namespace even when the root itself is short. Persisted +`stored_relpath` values and Docker mount paths retain their existing spelling. +`project_document_path()` still resolves symlinks and checks confinement using +the same namespace for both the user projects root and the document path. +Do not prefix only paths already exceeding MAX_PATH: appended filenames and +derived companions can cross the limit later. + Operator prompt overlays: `lead_prompt_overlay` on AppConfig and `subagents.agents..prompt_overlay` accept literal `prepend`/`append` strings. The per-assembly snapshot owns these settings; no run-context override exists. diff --git a/backend/packages/harness/deerflow/config/paths.py b/backend/packages/harness/deerflow/config/paths.py index 3723f04969..e8508c6808 100644 --- a/backend/packages/harness/deerflow/config/paths.py +++ b/backend/packages/harness/deerflow/config/paths.py @@ -6,6 +6,7 @@ import shutil from pathlib import Path, PureWindowsPath from deerflow.config.runtime_paths import runtime_home +from deerflow.utils.host_paths import extended_length_path from deerflow.utils.thread_id import validate_thread_id # Virtual path prefix seen by agents inside the sandbox @@ -366,7 +367,7 @@ class Paths: def user_projects_dir(self, user_id: str) -> Path: """Host path root for one user's project shelves: ``users/{user_id}/projects/``.""" - return self.user_dir(user_id) / "projects" + return extended_length_path(self.user_dir(user_id) / "projects") def user_project_dir(self, user_id: str, project_id: str) -> Path: """Host path for one project: ``users/{user_id}/projects/{project_id}/``.""" diff --git a/backend/packages/harness/deerflow/projects/trash.py b/backend/packages/harness/deerflow/projects/trash.py index fdafe15f6a..fb0f93c841 100644 --- a/backend/packages/harness/deerflow/projects/trash.py +++ b/backend/packages/harness/deerflow/projects/trash.py @@ -31,6 +31,7 @@ from typing import TYPE_CHECKING from deerflow.config.paths import Paths from deerflow.projects.documents import _content_intact, check_document_content, converted_markdown_path, original_file_path from deerflow.utils.file_io import await_drained, run_file_io +from deerflow.utils.host_paths import extended_length_path from deerflow.utils.time import coerce_iso if TYPE_CHECKING: @@ -233,7 +234,7 @@ def _reconcile_storage(paths: Paths, *, user_id: str | None, rows: list[dict], g if user_id is not None: user_ids = [user_id] else: - users_root = paths.base_dir / "users" + users_root = extended_length_path(paths.base_dir / "users") user_ids = sorted(entry.name for entry in users_root.iterdir() if entry.is_dir()) if users_root.is_dir() else [] for uid in user_ids: projects_root = paths.user_projects_dir(uid) diff --git a/backend/packages/harness/deerflow/utils/AGENTS.md b/backend/packages/harness/deerflow/utils/AGENTS.md index 3143816145..b254801cf0 100644 --- a/backend/packages/harness/deerflow/utils/AGENTS.md +++ b/backend/packages/harness/deerflow/utils/AGENTS.md @@ -82,6 +82,13 @@ lives in `tests/test_file_conversion_cancellation.py`. ### Host Path Portability +`host_paths.extended_length_path()` provides lexical Windows filesystem +spelling (absolute `\\?\` drive paths or `\\?\UNC\` shares); it is a no-op +on POSIX. It does not replace symlink resolution or confinement checks and +must not be used for persisted relative paths or Docker mount sources. +Existing extended (`\\?\`) and device (`\\.\`, including named pipes) +namespaces pass through unchanged before normalization. + `host_paths.py` rejects Windows device names for host-visible creation paths on every platform, including the `COM`/`LPT` aliases with superscript ¹, ² and ³. The console aliases `CONIN$` and `CONOUT$` are reserved too; match them diff --git a/backend/packages/harness/deerflow/utils/host_paths.py b/backend/packages/harness/deerflow/utils/host_paths.py index f998a7f707..b65553a74b 100644 --- a/backend/packages/harness/deerflow/utils/host_paths.py +++ b/backend/packages/harness/deerflow/utils/host_paths.py @@ -2,6 +2,32 @@ from __future__ import annotations +import os +from pathlib import Path + + +def extended_length_path(path: Path) -> Path: + """Use Windows' extended namespace for local filesystem access. + + Prefix before appending children: even a short root can contain a long + document filename. Normalize ordinary paths before adding the prefix; + extended paths bypass Win32's ordinary path normalization. This is lexical + only: callers must still resolve symlinks and enforce their own confinement. + Existing extended/device namespaces (including named pipes) are returned + unchanged, without normalizing their segments. + POSIX paths are unchanged. Do not use this spelling for Docker mount sources + or persisted relative paths. + """ + if os.name != "nt": + return path + if str(path).startswith(("\\\\?\\", "\\\\.\\")): + return path + absolute = os.path.abspath(path) + if absolute.startswith("\\\\"): + return Path("\\\\?\\UNC\\" + absolute[2:]) + return Path("\\\\?\\" + absolute) + + # Windows also treats the ISO-8859-1 superscript digits ¹, ² and ³ as device numbers. # https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file # CONIN$ and CONOUT$ open the console input and output buffers, not ordinary files. diff --git a/backend/tests/blocking_io/test_project_documents.py b/backend/tests/blocking_io/test_project_documents.py index c05c66a736..fb08765e5a 100644 --- a/backend/tests/blocking_io/test_project_documents.py +++ b/backend/tests/blocking_io/test_project_documents.py @@ -20,7 +20,6 @@ from __future__ import annotations import functools import json -import os from types import SimpleNamespace import pytest @@ -36,10 +35,6 @@ from deerflow.utils.file_io import run_file_io as _real_run_file_io pytestmark = [ pytest.mark.asyncio, pytest.mark.allow_blocking_io, - pytest.mark.skipif( - os.name == "nt", - reason="project-document anchors exceed Windows MAX_PATH (WinError 206); see #6353", - ), ] _USER = "u1" diff --git a/backend/tests/blocking_io/test_project_documents_promotion.py b/backend/tests/blocking_io/test_project_documents_promotion.py index 98b94ad15b..e3e34f643f 100644 --- a/backend/tests/blocking_io/test_project_documents_promotion.py +++ b/backend/tests/blocking_io/test_project_documents_promotion.py @@ -13,7 +13,6 @@ an offload drops the dispatch and turns the anchor red. from __future__ import annotations import functools -import os from types import SimpleNamespace import pytest @@ -64,10 +63,6 @@ async def _shelf_document(env) -> dict: return result[0] -@pytest.mark.skipif( - os.name == "nt", - reason="attach staging copy/read exceed Windows MAX_PATH (WinError 206); see #6353", -) async def test_attach_staging_copy_dispatches_off_the_loop(tmp_path, monkeypatch) -> None: """The under-lock source copy (existence/size check + ``shutil.copyfile``) must go through ``run_file_io`` while the row lock is held.""" @@ -87,10 +82,6 @@ async def test_attach_staging_copy_dispatches_off_the_loop(tmp_path, monkeypatch await close_engine() -@pytest.mark.skipif( - os.name == "nt", - reason="attach staging copy/read exceed Windows MAX_PATH (WinError 206); see #6353", -) async def test_attach_staged_read_dispatches_off_the_loop(tmp_path, monkeypatch) -> None: """``read_file_chunks`` (attach ingestion source + from-thread promote source) must dispatch the open, every chunk read, and the close through diff --git a/backend/tests/blocking_io/test_project_trash.py b/backend/tests/blocking_io/test_project_trash.py index 7a40948b6c..2126c9db98 100644 --- a/backend/tests/blocking_io/test_project_trash.py +++ b/backend/tests/blocking_io/test_project_trash.py @@ -11,7 +11,6 @@ anchor red (mutation-verified style of this suite). from __future__ import annotations import functools -import os from types import SimpleNamespace import pytest @@ -25,10 +24,6 @@ from deerflow.utils.file_io import run_file_io as _real_run_file_io pytestmark = [ pytest.mark.asyncio, pytest.mark.allow_blocking_io, - pytest.mark.skipif( - os.name == "nt", - reason="project-document anchors exceed Windows MAX_PATH (WinError 206); see #6353", - ), ] _USER = "u1" diff --git a/backend/tests/test_extended_length_paths.py b/backend/tests/test_extended_length_paths.py new file mode 100644 index 0000000000..440be37afc --- /dev/null +++ b/backend/tests/test_extended_length_paths.py @@ -0,0 +1,72 @@ +"""Windows filesystem spelling and document confinement regressions.""" + +import ntpath +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from deerflow.config.paths import Paths +from deerflow.utils import host_paths + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + (r"C:\shelf\..\documents", r"\\?\C:\documents"), + ("C:/shelf/documents", r"\\?\C:\shelf\documents"), + (r"\\server\share\shelf", r"\\?\UNC\server\share\shelf"), + (r"\\?\C:\shelf", r"\\?\C:\shelf"), + (r"\\?\UNC\server\share\shelf", r"\\?\UNC\server\share\shelf"), + (r"\\.\C:\shelf", r"\\.\C:\shelf"), + (r"\\.\pipe\deerflow", r"\\.\pipe\deerflow"), + (r"\\?\C:\shelf\..\documents", r"\\?\C:\shelf\..\documents"), + ], +) +def test_windows_extended_spelling(monkeypatch, raw, expected): + # Patch this module's OS binding, never the process-wide os.name (pathlib + # and pytest need to keep using the actual host's path implementation). + monkeypatch.setattr(host_paths, "os", SimpleNamespace(name="nt", path=ntpath)) + assert str(host_paths.extended_length_path(Path(raw))) == expected + + +def test_posix_path_is_unchanged(monkeypatch): + monkeypatch.setattr(host_paths, "os", SimpleNamespace(name="posix")) + path = Path("relative/documents") + assert host_paths.extended_length_path(path) is path + + +@pytest.mark.skipif(os.name != "nt", reason="native Windows relative-path normalization") +def test_windows_relative_path_is_absolutized(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + actual = host_paths.extended_length_path(Path("shelf/../documents")) + assert str(actual) == "\\\\?\\" + str(tmp_path / "documents") + + +def test_document_traversal_is_still_rejected(tmp_path): + paths = Paths(base_dir=tmp_path) + with pytest.raises(ValueError, match="path traversal"): + paths.project_document_path("alice", "../outside") + with pytest.raises(ValueError, match="path traversal"): + paths.project_document_path("alice", str(tmp_path / "outside")) + + +@pytest.mark.parametrize("outside_root", [False, True]) +def test_document_symlink_confinement(tmp_path, outside_root): + paths = Paths(base_dir=tmp_path) + base = paths.user_projects_dir("alice") + base.mkdir(parents=True) + target = tmp_path / "outside" if outside_root else base / "target" + target.mkdir() + try: + (base / "link").symlink_to(target, target_is_directory=True) + except OSError: + if os.name == "nt": + pytest.skip("Windows symlink privilege is not available") + raise + if outside_root: + with pytest.raises(ValueError, match="path traversal"): + paths.project_document_path("alice", "link/content") + else: + assert paths.project_document_path("alice", "link/content") == (target / "content").resolve() diff --git a/backend/tests/test_trash_router.py b/backend/tests/test_trash_router.py index a98715a244..0b6d37ff65 100644 --- a/backend/tests/test_trash_router.py +++ b/backend/tests/test_trash_router.py @@ -8,6 +8,7 @@ and the lazy retention sweep through the real HTTP stack. from __future__ import annotations +import os from datetime import UTC, datetime, timedelta from pathlib import Path from types import SimpleNamespace @@ -131,6 +132,52 @@ def _original_path(app: FastAPI, row: dict, *, user_id: str = "user-a") -> Path: return original_file_path(get_paths(), user_id=user_id, row=row) +@pytest.mark.parametrize("filename", ["notes.txt", "a" * 251 + ".txt"]) +def test_long_workspace_document_lifecycle(tmp_path, monkeypatch, filename): + """Real router/file operations must work beyond Windows' legacy MAX_PATH.""" + from deerflow.config.paths import get_paths + from deerflow.projects.trash import run_trash_retention_sweep + + home = tmp_path / ("nested-" + "a" * 73) / ("nested-" + "b" * 73) / ("nested-" + "c" * 73) + assert len(str(home)) > 260 + monkeypatch.setenv("DEER_FLOW_HOME", str(home)) + app = _build_app(tmp_path) + with TestClient(app) as client: + origin = _create_project(client, name="Origin") + target = _create_project(client, name="Target") + doc = _upload(client, origin["id"], filename, b"long-path content") + row = _get_row(app, doc["id"]) + original = _original_path(app, row) + assert original.read_bytes() == b"long-path content" + assert not row["stored_relpath"].startswith("\\\\") + listing = client.get(f"/api/projects/{origin['id']}/documents").json() + assert listing["documents"][0]["content_missing"] is False + download = client.get(f"/api/projects/{origin['id']}/documents/{doc['id']}/content", params={"download": "true"}) + assert download.status_code == 200 + assert download.content == b"long-path content" + staging = get_paths().project_documents_dir("user-a", origin["id"]) / ".staging" / "old-orphan" + staging.write_bytes(b"abandoned upload") + os.utime(staging, (0, 0)) + + async def startup_sweep(): + return await run_trash_retention_sweep(app.state.project_document_repo, get_paths(), retention_days=30, user_id=None) + + report = anyio.run(startup_sweep) + assert report.staging_removed == 1 + assert not staging.exists() + assert original.read_bytes() == b"long-path content" + _trash(client, origin["id"], doc["id"]) + restored = client.post(f"/api/trash/documents/{doc['id']}/restore", json={"project_id": target["id"]}) + assert restored.status_code == 200 + assert restored.json()["outcome"] == "restored" + assert _original_path(app, _get_row(app, doc["id"])) == original + assert client.get(f"/api/projects/{target['id']}/documents/{doc['id']}/content").content == b"long-path content" + _trash(client, target["id"], doc["id"]) + assert client.post(f"/api/trash/documents/{doc['id']}/purge").status_code == 204 + assert not original.exists() + assert _get_row(app, doc["id"]) is None + + class TestTrashListing: def test_list_empty_then_populated_with_origin_display_fields(self, tmp_path): app = _build_app(tmp_path)