mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-14 16:08:41 +00:00
fix: mount upload conversions read-only
This commit is contained in:
hetaoBackend
parent
920dc247d2
commit
83e908bfe3
@ -44,6 +44,7 @@ from deerflow.integrations.lark_cli import LARK_CLI_SANDBOX_CONFIG_DIR, LARK_CLI
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.sandbox.sandbox import Sandbox
|
||||
from deerflow.sandbox.sandbox_provider import SandboxProvider
|
||||
from deerflow.uploads.layout import UPLOAD_CONVERSIONS_DIRNAME, ensure_conversion_dir
|
||||
|
||||
from .aio_sandbox import AioSandbox
|
||||
from .backend import SandboxBackend, wait_for_sandbox_ready, wait_for_sandbox_ready_async
|
||||
@ -847,11 +848,17 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
|
||||
paths = get_paths()
|
||||
effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id)
|
||||
paths.ensure_thread_dirs(thread_id, user_id=effective_user_id)
|
||||
ensure_conversion_dir(paths.sandbox_uploads_dir(thread_id, user_id=effective_user_id))
|
||||
|
||||
return [
|
||||
(paths.host_sandbox_work_dir(thread_id, user_id=effective_user_id), f"{VIRTUAL_PATH_PREFIX}/workspace", False),
|
||||
(paths.host_sandbox_uploads_dir(thread_id, user_id=effective_user_id), f"{VIRTUAL_PATH_PREFIX}/uploads", False),
|
||||
(paths.host_sandbox_outputs_dir(thread_id, user_id=effective_user_id), f"{VIRTUAL_PATH_PREFIX}/outputs", False),
|
||||
(
|
||||
join_host_path(paths.host_sandbox_user_data_dir(thread_id, user_id=effective_user_id), UPLOAD_CONVERSIONS_DIRNAME),
|
||||
f"{VIRTUAL_PATH_PREFIX}/{UPLOAD_CONVERSIONS_DIRNAME}",
|
||||
True,
|
||||
),
|
||||
# ACP workspace: read-only inside the sandbox (lead agent reads results;
|
||||
# the ACP subprocess writes from the host side, not from within the container).
|
||||
(paths.host_acp_workspace_dir(thread_id, user_id=effective_user_id), "/mnt/acp-workspace", True),
|
||||
|
||||
@ -6,6 +6,7 @@ from pathlib import Path
|
||||
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping
|
||||
from deerflow.sandbox.sandbox import Sandbox
|
||||
from deerflow.sandbox.sandbox_provider import SandboxProvider
|
||||
from deerflow.uploads.layout import UPLOAD_CONVERSIONS_DIRNAME, ensure_conversion_dir
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -42,7 +43,8 @@ class LocalSandboxProvider(SandboxProvider):
|
||||
|
||||
The provider now produces a fresh ``LocalSandbox`` per ``thread_id`` whose
|
||||
``path_mappings`` include thread-scoped entries for
|
||||
``/mnt/user-data/{workspace,uploads,outputs}`` and ``/mnt/acp-workspace``,
|
||||
``/mnt/user-data/{workspace,uploads,outputs}``, the read-only generated
|
||||
``/mnt/user-data/.upload-conversions`` namespace, and ``/mnt/acp-workspace``,
|
||||
mirroring how :class:`AioSandboxProvider` bind-mounts those paths into its
|
||||
docker container. The legacy ``acquire()`` / ``acquire(None)`` call still
|
||||
returns a generic singleton with id ``"local"`` for callers (and tests)
|
||||
@ -292,6 +294,7 @@ class LocalSandboxProvider(SandboxProvider):
|
||||
paths = get_paths()
|
||||
effective_user_id = LocalSandboxProvider._effective_acquire_user_id(user_id)
|
||||
paths.ensure_thread_dirs(thread_id, user_id=effective_user_id)
|
||||
conversion_dir = ensure_conversion_dir(paths.sandbox_uploads_dir(thread_id, user_id=effective_user_id))
|
||||
|
||||
mappings = [
|
||||
# Aggregate parent mapping so ``ls /mnt/user-data`` and other
|
||||
@ -319,6 +322,11 @@ class LocalSandboxProvider(SandboxProvider):
|
||||
local_path=str(paths.sandbox_outputs_dir(thread_id, user_id=effective_user_id)),
|
||||
read_only=False,
|
||||
),
|
||||
PathMapping(
|
||||
container_path=f"{_USER_DATA_VIRTUAL_PREFIX}/{UPLOAD_CONVERSIONS_DIRNAME}",
|
||||
local_path=str(conversion_dir),
|
||||
read_only=True,
|
||||
),
|
||||
PathMapping(
|
||||
container_path=_ACP_WORKSPACE_VIRTUAL_PREFIX,
|
||||
local_path=str(paths.acp_workspace_dir(thread_id, user_id=effective_user_id)),
|
||||
|
||||
@ -159,6 +159,20 @@ def test_get_thread_mounts_includes_user_data_dirs(tmp_path, monkeypatch):
|
||||
assert "/mnt/user-data/outputs" in container_paths
|
||||
|
||||
|
||||
def test_get_thread_mounts_includes_upload_conversions_read_only(tmp_path, monkeypatch):
|
||||
"""Generated upload conversions must be readable but immutable in AIO."""
|
||||
aio_mod = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox_provider")
|
||||
monkeypatch.setattr(aio_mod, "get_paths", lambda: Paths(base_dir=tmp_path))
|
||||
monkeypatch.setattr(aio_mod, "get_effective_user_id", lambda: None)
|
||||
|
||||
mounts = aio_mod.AioSandboxProvider._get_thread_mounts("thread-4")
|
||||
container_paths = {container_path: (host_path, read_only) for host_path, container_path, read_only in mounts}
|
||||
|
||||
expected_host = tmp_path / "threads" / "thread-4" / "user-data" / ".upload-conversions"
|
||||
assert container_paths["/mnt/user-data/.upload-conversions"] == (str(expected_host), True)
|
||||
assert expected_host.is_dir()
|
||||
|
||||
|
||||
def test_get_thread_mounts_uses_explicit_user_id(tmp_path, monkeypatch):
|
||||
"""Channel runs must mount the same user bucket used for artifact delivery."""
|
||||
aio_mod = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox_provider")
|
||||
@ -171,6 +185,7 @@ def test_get_thread_mounts_uses_explicit_user_id(tmp_path, monkeypatch):
|
||||
assert container_paths["/mnt/user-data/workspace"] == str(tmp_path / "users" / "ou-user" / "threads" / "thread-4" / "user-data" / "workspace")
|
||||
assert container_paths["/mnt/user-data/uploads"] == str(tmp_path / "users" / "ou-user" / "threads" / "thread-4" / "user-data" / "uploads")
|
||||
assert container_paths["/mnt/user-data/outputs"] == str(tmp_path / "users" / "ou-user" / "threads" / "thread-4" / "user-data" / "outputs")
|
||||
assert container_paths["/mnt/user-data/.upload-conversions"] == str(tmp_path / "users" / "ou-user" / "threads" / "thread-4" / "user-data" / ".upload-conversions")
|
||||
|
||||
|
||||
def test_get_lark_cli_runtime_mounts_uses_user_auth_dirs(tmp_path, monkeypatch):
|
||||
@ -298,6 +313,7 @@ def test_get_thread_mounts_preserves_windows_host_path_style(tmp_path, monkeypat
|
||||
assert container_paths["/mnt/user-data/workspace"] == r"C:\Users\demo\deer-flow\backend\.deer-flow\threads\thread-10\user-data\workspace"
|
||||
assert container_paths["/mnt/user-data/uploads"] == r"C:\Users\demo\deer-flow\backend\.deer-flow\threads\thread-10\user-data\uploads"
|
||||
assert container_paths["/mnt/user-data/outputs"] == r"C:\Users\demo\deer-flow\backend\.deer-flow\threads\thread-10\user-data\outputs"
|
||||
assert container_paths["/mnt/user-data/.upload-conversions"] == r"C:\Users\demo\deer-flow\backend\.deer-flow\threads\thread-10\user-data\.upload-conversions"
|
||||
assert container_paths["/mnt/acp-workspace"] == r"C:\Users\demo\deer-flow\backend\.deer-flow\threads\thread-10\acp-workspace"
|
||||
|
||||
|
||||
|
||||
@ -544,6 +544,34 @@ class TestMultipleMounts:
|
||||
|
||||
|
||||
class TestLocalSandboxProviderMounts:
|
||||
def test_thread_mappings_mount_upload_conversions_read_only(self, tmp_path):
|
||||
from deerflow.config.paths import Paths
|
||||
|
||||
paths = Paths(base_dir=tmp_path / "home")
|
||||
config = SimpleNamespace(
|
||||
skills=SimpleNamespace(
|
||||
container_path="/mnt/skills",
|
||||
get_skills_path=lambda: tmp_path / "skills",
|
||||
use="deerflow.skills.storage.local_skill_storage:LocalSkillStorage",
|
||||
)
|
||||
)
|
||||
|
||||
with (
|
||||
patch("deerflow.config.get_app_config", return_value=config),
|
||||
patch("deerflow.config.paths.get_paths", return_value=paths),
|
||||
):
|
||||
mappings = LocalSandboxProvider._build_thread_path_mappings("thread-a", user_id="alice")
|
||||
|
||||
conversion_mapping = next(mapping for mapping in mappings if mapping.container_path == "/mnt/user-data/.upload-conversions")
|
||||
assert conversion_mapping.local_path == str(paths.sandbox_user_data_dir("thread-a", user_id="alice") / ".upload-conversions")
|
||||
assert conversion_mapping.read_only is True
|
||||
assert Path(conversion_mapping.local_path).is_dir()
|
||||
|
||||
sandbox = LocalSandbox("test", mappings)
|
||||
with pytest.raises(OSError) as exc_info:
|
||||
sandbox.write_file("/mnt/user-data/.upload-conversions/forbidden.md", "content")
|
||||
assert exc_info.value.errno == errno.EROFS
|
||||
|
||||
def test_thread_mappings_mount_per_user_integration_projections(self, tmp_path):
|
||||
from deerflow.config.paths import Paths
|
||||
|
||||
|
||||
@ -555,7 +555,7 @@ git commit -m "fix: retain upload leases through adapter sync"
|
||||
- Modify: `backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py`
|
||||
- Modify: `backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py`
|
||||
- Test: `backend/tests/test_local_sandbox_provider_mounts.py`
|
||||
- Test: `backend/tests/test_aio_sandbox_thread_mounts.py`
|
||||
- Test: `backend/tests/test_aio_sandbox_provider.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `UPLOAD_CONVERSIONS_DIRNAME`, `ensure_conversion_dir`, and existing `join_host_path`.
|
||||
@ -573,7 +573,7 @@ def test_local_thread_mappings_mount_conversions_read_only(paths_config):
|
||||
assert Path(conversion.local_path).is_dir()
|
||||
|
||||
|
||||
def test_aio_thread_mounts_include_read_only_conversions(paths_config):
|
||||
def test_get_thread_mounts_includes_upload_conversions_read_only(tmp_path, monkeypatch):
|
||||
mounts = AioSandboxProvider._get_thread_mounts("thread-1", user_id="user-1")
|
||||
assert any(container == "/mnt/user-data/.upload-conversions" and read_only for _, container, read_only in mounts)
|
||||
```
|
||||
@ -582,7 +582,7 @@ def test_aio_thread_mounts_include_read_only_conversions(paths_config):
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
uv run pytest tests/test_local_sandbox_provider_mounts.py tests/test_aio_sandbox_thread_mounts.py -k "conversion" -q
|
||||
uv run pytest tests/test_local_sandbox_provider_mounts.py tests/test_aio_sandbox_provider.py -k "conversion" -q
|
||||
```
|
||||
|
||||
Expected: neither provider exposes the explicit mapping.
|
||||
@ -605,13 +605,13 @@ AIO adds:
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
uv run pytest tests/test_local_sandbox_provider_mounts.py tests/test_aio_sandbox_thread_mounts.py tests/test_local_sandbox_virtual_path_contract.py -q
|
||||
uv run pytest tests/test_local_sandbox_provider_mounts.py tests/test_aio_sandbox_provider.py tests/test_local_sandbox_virtual_path_contract.py -q
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Commit sandbox visibility**
|
||||
|
||||
```bash
|
||||
git add backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py backend/tests/test_local_sandbox_provider_mounts.py backend/tests/test_aio_sandbox_thread_mounts.py
|
||||
git add backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py backend/tests/test_local_sandbox_provider_mounts.py backend/tests/test_aio_sandbox_provider.py
|
||||
git commit -m "fix: mount upload conversions read-only"
|
||||
```
|
||||
|
||||
@ -642,7 +642,7 @@ use `<truncated-primary>.<full-sha256>.md`. State that `.upload-*.part` is reser
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
uv run pytest tests/test_uploads_manager.py tests/test_upload_conversion.py tests/test_file_conversion.py tests/test_uploads_router.py tests/test_client.py tests/test_channel_file_attachments.py tests/test_dingtalk_channel.py tests/test_wechat_channel.py tests/test_local_sandbox_provider_mounts.py tests/test_aio_sandbox_thread_mounts.py tests/test_uploads_middleware_core_logic.py tests/test_list_uploaded_files_tool.py tests/blocking_io/test_upload_conversion.py tests/blocking_io/test_uploads_router.py tests/blocking_io/test_channels_ingest.py tests/blocking_io/test_feishu_receive_file.py tests/blocking_io/test_dingtalk_receive_file.py -q
|
||||
uv run pytest tests/test_uploads_manager.py tests/test_upload_conversion.py tests/test_file_conversion.py tests/test_uploads_router.py tests/test_client.py tests/test_channel_file_attachments.py tests/test_dingtalk_channel.py tests/test_wechat_channel.py tests/test_local_sandbox_provider_mounts.py tests/test_aio_sandbox_provider.py tests/test_uploads_middleware_core_logic.py tests/test_list_uploaded_files_tool.py tests/blocking_io/test_upload_conversion.py tests/blocking_io/test_uploads_router.py tests/blocking_io/test_channels_ingest.py tests/blocking_io/test_feishu_receive_file.py tests/blocking_io/test_dingtalk_receive_file.py -q
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Run formatting, lint, and the full backend suite**
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user