mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
cb6ee72132
`Paths.delete_thread_dir()` called `shutil.rmtree()` directly. Windows
refuses to unlink a file that carries the read-only attribute
(`WinError 5`) where POSIX only needs a writable parent, so one read-only
file under `user-data/` made the whole thread undeletable: `DELETE
/api/threads/{id}` answered 500 `Failed to delete local thread data`,
`rmtree` had already removed everything it walked before that entry, and
every later retry failed on the same file.
Retry at the call site: clear the read-only attribute tree-wide and call
`rmtree()` once more. Only `PermissionError` is retried, the retry still
raises when it cannot help, so a file another process holds open stays a
real error instead of becoming a silent skip. A `onexc` handler cannot do
this portably: POSIX `_rmtree_safe_fd()` catches the handler's own
`PermissionError`, re-dispatches it as a `scandir` error and leaves the
deletion to fail later as `ENOTEMPTY`.
Co-authored-by: Aling33 <along308@users.noreply.github.com>
338 lines
14 KiB
Python
338 lines
14 KiB
Python
"""Tests for user-scoped path resolution in Paths."""
|
|
|
|
import os
|
|
import stat
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from deerflow.config.paths import Paths
|
|
|
|
|
|
@pytest.fixture
|
|
def paths(tmp_path: Path) -> Paths:
|
|
return Paths(tmp_path)
|
|
|
|
|
|
class TestValidateUserId:
|
|
def test_valid_user_id(self, paths: Paths):
|
|
d = paths.user_dir("u-abc-123")
|
|
assert d == paths.base_dir / "users" / "u-abc-123"
|
|
|
|
def test_rejects_path_traversal(self, paths: Paths):
|
|
with pytest.raises(ValueError, match="Invalid user_id"):
|
|
paths.user_dir("../escape")
|
|
|
|
def test_rejects_slash(self, paths: Paths):
|
|
with pytest.raises(ValueError, match="Invalid user_id"):
|
|
paths.user_dir("foo/bar")
|
|
|
|
def test_rejects_empty(self, paths: Paths):
|
|
with pytest.raises(ValueError, match="Invalid user_id"):
|
|
paths.user_dir("")
|
|
|
|
def test_rejects_trailing_newline(self, paths: Paths):
|
|
with pytest.raises(ValueError, match="Invalid user_id"):
|
|
paths.user_dir("alice\n")
|
|
|
|
|
|
class TestMakeSafeUserId:
|
|
def test_already_safe_id_is_unchanged(self):
|
|
from deerflow.config.paths import make_safe_user_id
|
|
|
|
assert make_safe_user_id("ou_abc-123") == "ou_abc-123"
|
|
assert make_safe_user_id("123456") == "123456"
|
|
|
|
def test_unsafe_chars_are_sanitized_with_stable_suffix(self):
|
|
from deerflow.config.paths import make_safe_user_id
|
|
|
|
result = make_safe_user_id("user@example.com")
|
|
# Sanitized prefix plus a stable digest of the original.
|
|
assert result.startswith("user-example-com-")
|
|
assert len(result.rsplit("-", 1)[1]) == 16
|
|
assert result == "user-example-com-b4c9a289323b21a0"
|
|
assert make_safe_user_id("user@example.com") == result
|
|
|
|
def test_sanitized_id_passes_validation(self, paths: Paths):
|
|
from deerflow.config.paths import make_safe_user_id
|
|
|
|
safe = make_safe_user_id("用户/../etc")
|
|
# Must be usable as a filesystem-scoped bucket without raising.
|
|
assert paths.user_dir(safe) == paths.base_dir / "users" / safe
|
|
|
|
def test_distinct_unsafe_ids_do_not_collide(self):
|
|
from deerflow.config.paths import make_safe_user_id
|
|
|
|
assert make_safe_user_id("a.b") != make_safe_user_id("a/b")
|
|
|
|
def test_empty_id_rejected(self):
|
|
from deerflow.config.paths import make_safe_user_id
|
|
|
|
with pytest.raises(ValueError, match="non-empty"):
|
|
make_safe_user_id("")
|
|
|
|
|
|
class TestValidateIntegrationId:
|
|
def test_accepts_dotted_integration_id(self):
|
|
from deerflow.config.paths import _validate_integration_id
|
|
|
|
assert _validate_integration_id("lark-cli") == "lark-cli"
|
|
assert _validate_integration_id("some.integration") == "some.integration"
|
|
|
|
@pytest.mark.parametrize("integration_id", [".", ".."])
|
|
def test_rejects_dot_and_dotdot(self, integration_id):
|
|
from deerflow.config.paths import _validate_integration_id
|
|
|
|
with pytest.raises(ValueError, match="Invalid integration_id"):
|
|
_validate_integration_id(integration_id)
|
|
|
|
def test_rejects_trailing_newline(self):
|
|
from deerflow.config.paths import _validate_integration_id
|
|
|
|
with pytest.raises(ValueError, match="Invalid integration_id"):
|
|
_validate_integration_id("lark-cli\n")
|
|
|
|
@pytest.mark.parametrize("integration_id", [".", ".."])
|
|
def test_host_integration_config_dir_rejects_dot_traversal(self, paths: Paths, integration_id):
|
|
with pytest.raises(ValueError, match="Invalid integration_id"):
|
|
paths.host_user_integration_config_dir("alice", integration_id)
|
|
|
|
@pytest.mark.parametrize("integration_id", [".", ".."])
|
|
def test_host_integration_data_dir_rejects_dot_traversal(self, paths: Paths, integration_id):
|
|
with pytest.raises(ValueError, match="Invalid integration_id"):
|
|
paths.host_user_integration_data_dir("alice", integration_id)
|
|
|
|
|
|
class TestValidateProjectId:
|
|
def test_rejects_trailing_newline(self, paths: Paths):
|
|
with pytest.raises(ValueError, match="Invalid project_id"):
|
|
paths.user_project_dir("alice", "project\n")
|
|
|
|
|
|
class TestUserDir:
|
|
def test_user_dir(self, paths: Paths):
|
|
assert paths.user_dir("alice") == paths.base_dir / "users" / "alice"
|
|
|
|
def test_prepare_user_dir_migrates_unique_legacy_unsafe_bucket(self, paths: Paths):
|
|
from deerflow.config.paths import make_safe_user_id
|
|
|
|
raw = "user@example.com"
|
|
safe = make_safe_user_id(raw)
|
|
legacy_dir = paths.base_dir / "users" / "user-example-com-63a710569261a24b"
|
|
legacy_dir.mkdir(parents=True)
|
|
(legacy_dir / "memory.json").write_text('{"legacy": true}\n', encoding="utf-8")
|
|
|
|
assert paths.prepare_user_dir_for_raw_id(raw) == safe
|
|
|
|
current_dir = paths.user_dir(safe)
|
|
assert current_dir.exists()
|
|
assert not legacy_dir.exists()
|
|
assert (current_dir / "memory.json").read_text(encoding="utf-8") == '{"legacy": true}\n'
|
|
|
|
def test_prepare_user_dir_never_migrates_another_users_bucket(self, paths: Paths):
|
|
"""A different raw ID with the same sanitized prefix has a different legacy digest."""
|
|
import hashlib
|
|
|
|
from deerflow.config.paths import make_safe_user_id
|
|
|
|
users_dir = paths.base_dir / "users"
|
|
other_legacy = users_dir / f"a-b-{hashlib.sha1(b'a/b').hexdigest()[:16]}"
|
|
other_legacy.mkdir(parents=True)
|
|
arbitrary_16_hex = users_dir / "a-b-1111111111111111"
|
|
arbitrary_16_hex.mkdir(parents=True)
|
|
|
|
assert paths.prepare_user_dir_for_raw_id("a.b") == make_safe_user_id("a.b")
|
|
|
|
assert not paths.user_dir(make_safe_user_id("a.b")).exists()
|
|
assert other_legacy.exists()
|
|
assert arbitrary_16_hex.exists()
|
|
|
|
|
|
class TestUserMemoryFile:
|
|
def test_user_memory_file(self, paths: Paths):
|
|
assert paths.user_memory_file("bob") == paths.base_dir / "users" / "bob" / "memory.json"
|
|
|
|
|
|
class TestUserAgentMemoryFile:
|
|
def test_user_agent_memory_file(self, paths: Paths):
|
|
expected = paths.base_dir / "users" / "bob" / "agents" / "myagent" / "memory.json"
|
|
assert paths.user_agent_memory_file("bob", "myagent") == expected
|
|
|
|
def test_user_agent_memory_file_lowercases_name(self, paths: Paths):
|
|
expected = paths.base_dir / "users" / "bob" / "agents" / "myagent" / "memory.json"
|
|
assert paths.user_agent_memory_file("bob", "MyAgent") == expected
|
|
|
|
|
|
class TestUserAgentDir:
|
|
def test_user_agents_dir(self, paths: Paths):
|
|
assert paths.user_agents_dir("alice") == paths.base_dir / "users" / "alice" / "agents"
|
|
|
|
def test_user_agent_dir(self, paths: Paths):
|
|
assert paths.user_agent_dir("alice", "code-reviewer") == paths.base_dir / "users" / "alice" / "agents" / "code-reviewer"
|
|
|
|
def test_user_agent_dir_lowercases_name(self, paths: Paths):
|
|
assert paths.user_agent_dir("alice", "CodeReviewer") == paths.base_dir / "users" / "alice" / "agents" / "codereviewer"
|
|
|
|
def test_user_agent_dir_validates_user_id(self, paths: Paths):
|
|
with pytest.raises(ValueError, match="Invalid user_id"):
|
|
paths.user_agent_dir("../escape", "myagent")
|
|
|
|
|
|
class TestUserThreadDir:
|
|
def test_user_thread_dir(self, paths: Paths):
|
|
expected = paths.base_dir / "users" / "u1" / "threads" / "t1"
|
|
assert paths.thread_dir("t1", user_id="u1") == expected
|
|
|
|
def test_thread_dir_no_user_id_falls_back_to_legacy(self, paths: Paths):
|
|
expected = paths.base_dir / "threads" / "t1"
|
|
assert paths.thread_dir("t1") == expected
|
|
|
|
|
|
class TestUserSandboxDirs:
|
|
def test_sandbox_work_dir(self, paths: Paths):
|
|
expected = paths.base_dir / "users" / "u1" / "threads" / "t1" / "user-data" / "workspace"
|
|
assert paths.sandbox_work_dir("t1", user_id="u1") == expected
|
|
|
|
def test_sandbox_uploads_dir(self, paths: Paths):
|
|
expected = paths.base_dir / "users" / "u1" / "threads" / "t1" / "user-data" / "uploads"
|
|
assert paths.sandbox_uploads_dir("t1", user_id="u1") == expected
|
|
|
|
def test_sandbox_outputs_dir(self, paths: Paths):
|
|
expected = paths.base_dir / "users" / "u1" / "threads" / "t1" / "user-data" / "outputs"
|
|
assert paths.sandbox_outputs_dir("t1", user_id="u1") == expected
|
|
|
|
def test_sandbox_user_data_dir(self, paths: Paths):
|
|
expected = paths.base_dir / "users" / "u1" / "threads" / "t1" / "user-data"
|
|
assert paths.sandbox_user_data_dir("t1", user_id="u1") == expected
|
|
|
|
def test_acp_workspace_dir(self, paths: Paths):
|
|
expected = paths.base_dir / "users" / "u1" / "threads" / "t1" / "acp-workspace"
|
|
assert paths.acp_workspace_dir("t1", user_id="u1") == expected
|
|
|
|
def test_legacy_sandbox_work_dir(self, paths: Paths):
|
|
expected = paths.base_dir / "threads" / "t1" / "user-data" / "workspace"
|
|
assert paths.sandbox_work_dir("t1") == expected
|
|
|
|
|
|
class TestHostPathsWithUserId:
|
|
def test_host_thread_dir_with_user_id(self, paths: Paths):
|
|
result = paths.host_thread_dir("t1", user_id="u1")
|
|
assert "users" in result
|
|
assert "u1" in result
|
|
assert "threads" in result
|
|
assert "t1" in result
|
|
|
|
def test_host_thread_dir_legacy(self, paths: Paths):
|
|
result = paths.host_thread_dir("t1")
|
|
assert "threads" in result
|
|
assert "t1" in result
|
|
assert "users" not in result
|
|
|
|
def test_host_sandbox_user_data_dir_with_user_id(self, paths: Paths):
|
|
result = paths.host_sandbox_user_data_dir("t1", user_id="u1")
|
|
assert "users" in result
|
|
assert "user-data" in result
|
|
|
|
def test_host_sandbox_work_dir_with_user_id(self, paths: Paths):
|
|
result = paths.host_sandbox_work_dir("t1", user_id="u1")
|
|
assert "workspace" in result
|
|
|
|
def test_host_sandbox_uploads_dir_with_user_id(self, paths: Paths):
|
|
result = paths.host_sandbox_uploads_dir("t1", user_id="u1")
|
|
assert "uploads" in result
|
|
|
|
def test_host_sandbox_outputs_dir_with_user_id(self, paths: Paths):
|
|
result = paths.host_sandbox_outputs_dir("t1", user_id="u1")
|
|
assert "outputs" in result
|
|
|
|
def test_host_acp_workspace_dir_with_user_id(self, paths: Paths):
|
|
result = paths.host_acp_workspace_dir("t1", user_id="u1")
|
|
assert "acp-workspace" in result
|
|
|
|
|
|
class TestEnsureAndDeleteWithUserId:
|
|
def test_ensure_thread_dirs_creates_user_scoped(self, paths: Paths):
|
|
paths.ensure_thread_dirs("t1", user_id="u1")
|
|
assert paths.sandbox_work_dir("t1", user_id="u1").is_dir()
|
|
assert paths.sandbox_uploads_dir("t1", user_id="u1").is_dir()
|
|
assert paths.sandbox_outputs_dir("t1", user_id="u1").is_dir()
|
|
assert paths.acp_workspace_dir("t1", user_id="u1").is_dir()
|
|
|
|
def test_delete_thread_dir_removes_user_scoped(self, paths: Paths):
|
|
paths.ensure_thread_dirs("t1", user_id="u1")
|
|
assert paths.thread_dir("t1", user_id="u1").exists()
|
|
paths.delete_thread_dir("t1", user_id="u1")
|
|
assert not paths.thread_dir("t1", user_id="u1").exists()
|
|
|
|
def test_delete_thread_dir_idempotent(self, paths: Paths):
|
|
paths.delete_thread_dir("nonexistent", user_id="u1") # should not raise
|
|
|
|
def test_delete_thread_dir_retries_read_only_files(self, paths: Paths, monkeypatch):
|
|
"""Windows refuses a read-only file even though the parent is writable."""
|
|
paths.ensure_thread_dirs("t1", user_id="u1")
|
|
artifact = paths.sandbox_outputs_dir("t1", user_id="u1") / "report.pdf"
|
|
artifact.write_text("quarterly")
|
|
artifact.chmod(0o444)
|
|
real_unlink = os.unlink
|
|
|
|
def windows_unlink(path, *args, **kwargs):
|
|
# POSIX deletes through ``os.unlink(name, dir_fd=...)``, so the refusal
|
|
# has to key on that call shape to fire anywhere but Windows.
|
|
dir_fd = kwargs.get("dir_fd")
|
|
try:
|
|
if dir_fd is not None:
|
|
mode = os.stat(path, dir_fd=dir_fd, follow_symlinks=False).st_mode
|
|
else:
|
|
mode = os.stat(path).st_mode
|
|
except FileNotFoundError:
|
|
return real_unlink(path, *args, **kwargs)
|
|
if stat.S_ISREG(mode) and not mode & stat.S_IWRITE:
|
|
raise PermissionError(13, "Access is denied", str(path))
|
|
return real_unlink(path, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(os, "unlink", windows_unlink)
|
|
paths.delete_thread_dir("t1", user_id="u1")
|
|
assert not paths.thread_dir("t1", user_id="u1").exists()
|
|
|
|
def test_delete_thread_dir_keeps_raising_when_the_file_stays_locked(self, paths: Paths, monkeypatch):
|
|
"""A file still held open is a real failure; the retry must not swallow it."""
|
|
paths.ensure_thread_dirs("t1", user_id="u1")
|
|
(paths.sandbox_uploads_dir("t1", user_id="u1") / "notes.txt").write_text("x")
|
|
|
|
def locked_unlink(path, *args, **kwargs):
|
|
raise PermissionError(13, "The process cannot access the file", str(path))
|
|
|
|
monkeypatch.setattr(os, "unlink", locked_unlink)
|
|
with pytest.raises(PermissionError):
|
|
paths.delete_thread_dir("t1", user_id="u1")
|
|
assert paths.thread_dir("t1", user_id="u1").exists()
|
|
|
|
def test_ensure_thread_dirs_legacy_still_works(self, paths: Paths):
|
|
paths.ensure_thread_dirs("t1")
|
|
assert paths.sandbox_work_dir("t1").is_dir()
|
|
|
|
def test_user_scoped_and_legacy_are_independent(self, paths: Paths):
|
|
paths.ensure_thread_dirs("t1", user_id="u1")
|
|
paths.ensure_thread_dirs("t1")
|
|
# Both exist independently
|
|
assert paths.thread_dir("t1", user_id="u1").exists()
|
|
assert paths.thread_dir("t1").exists()
|
|
# Delete one doesn't affect the other
|
|
paths.delete_thread_dir("t1", user_id="u1")
|
|
assert not paths.thread_dir("t1", user_id="u1").exists()
|
|
assert paths.thread_dir("t1").exists()
|
|
|
|
|
|
class TestResolveVirtualPathWithUserId:
|
|
def test_resolve_virtual_path_with_user_id(self, paths: Paths):
|
|
paths.ensure_thread_dirs("t1", user_id="u1")
|
|
result = paths.resolve_virtual_path("t1", "/mnt/user-data/workspace/file.txt", user_id="u1")
|
|
expected_base = paths.sandbox_user_data_dir("t1", user_id="u1").resolve()
|
|
assert str(result).startswith(str(expected_base))
|
|
|
|
def test_resolve_virtual_path_legacy(self, paths: Paths):
|
|
paths.ensure_thread_dirs("t1")
|
|
result = paths.resolve_virtual_path("t1", "/mnt/user-data/workspace/file.txt")
|
|
expected_base = paths.sandbox_user_data_dir("t1").resolve()
|
|
assert str(result).startswith(str(expected_base))
|